Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ingestion): port the ingestion framework to rust (#127607)

Frank Hamand185bac0d5a2b
diff --git a/nodejs/src/ingestion/framework/benchmarks/framework-bench.ts b/nodejs/src/ingestion/framework/benchmarks/framework-bench.ts
new file mode 100644
index 000000000000..f84d29cab24c
--- /dev/null
+++ b/nodejs/src/ingestion/framework/benchmarks/framework-bench.ts
@@ -0,0 +1,214 @@
+// The workloads and sizes match `rust/ingestion-framework/benches/framework.rs`, so the two can be compared.
+import {
+ newBatchingPipeline,
+ newChunkPipelineBuilder,
+ newPipelineBuilder,
+ perElement,
+} from '~/ingestion/framework/builders'
+import { createOkContext } from '~/ingestion/framework/helpers'
+import { isOkResult, ok } from '~/ingestion/framework/results'
+import { ProcessingStep } from '~/ingestion/framework/steps'
+
+const SIZES = [1, 100, 10_000]
+const MIN_BENCH_MS = Number(process.env.BENCH_MS ?? 2000)
+
+type Ctx = Record<string, never>
+
+function inc(name: string): ProcessingStep<number, number> {
+ const step: ProcessingStep<number, number> = (v) => Promise.resolve(ok(v + 1))
+ Object.defineProperty(step, 'name', { value: name })
+ return step
+}
+
+function incChunk(name: string): (values: number[]) => Promise<ReturnType<typeof ok<number>>[]> {
+ const step = (values: number[]): Promise<ReturnType<typeof ok<number>>[]> =>
+ Promise.resolve(values.map((v) => ok(v + 1)))
+ Object.defineProperty(step, 'name', { value: name })
+ return step
+}
+
+function inputs(n: number): ReturnType<typeof createOkContext<number, Ctx>>[] {
+ const out = new Array(n)
+ for (let i = 0; i < n; i++) {
+ out[i] = createOkContext<number, Ctx>(i, {})
+ }
+ return out
+}
+
+function objectInputs(n: number): ReturnType<typeof createOkContext<{ value: number }, Ctx>>[] {
+ const out = new Array(n)
+ for (let i = 0; i < n; i++) {
+ out[i] = createOkContext<{ value: number }, Ctx>({ value: i }, {})
+ }
+ return out
+}
+
+async function drain(pipeline: { next(): Promise<unknown[] | null> }): Promise<number> {
+ let count = 0
+ let chunk = await pipeline.next()
+ while (chunk !== null) {
+ count += chunk.length
+ chunk = await pipeline.next()
+ }
+ return count
+}
+
+const stepPipeline = newPipelineBuilder<number, Ctx>()
+ .pipe(inc('s1'))
+ .pipe(inc('s2'))
+ .pipe(inc('s3'))
+ .pipe(inc('s4'))
+ .pipe(inc('s5'))
+ .build()
+
+const WORKLOADS: Record<string, (n: number) => Promise<void>> = {
+ step_pipeline_5_steps: async (n) => {
+ for (const input of inputs(n)) {
+ await stepPipeline.process(input)
+ }
+ },
+ pipe_chunk_5_steps: async (n) => {
+ const pipeline = newChunkPipelineBuilder<number, Ctx>()
+ .pipeChunk(incChunk('c1'))
+ .pipeChunk(incChunk('c2'))
+ .pipeChunk(incChunk('c3'))
+ .pipeChunk(incChunk('c4'))
+ .pipeChunk(incChunk('c5'))
+ .build()
+ pipeline.feed(inputs(n))
+ assertCount(await drain(pipeline), n)
+ },
+ concurrently_max50: async (n) => {
+ const pipeline = newChunkPipelineBuilder<number, Ctx>()
+ .concurrently((b) => b.pipe(inc('s1')).pipe(inc('s2')), { maxConcurrency: 50 })
+ .build()
+ pipeline.feed(inputs(n))
+ assertCount(await drain(pipeline), n)
+ },
+ per_element_max50: async (n) => {
+ const pipeline = newChunkPipelineBuilder<number, Ctx>()
+ .pipeChunk(perElement(inc('s1'), { maxConcurrency: 50 }))
+ .pipeChunk(perElement(inc('s2'), { maxConcurrency: 50 }))
+ .build()
+ pipeline.feed(inputs(n))
+ assertCount(await drain(pipeline), n)
+ },
+ concurrently_per_group_16: async (n) => {
+ const pipeline = newChunkPipelineBuilder<number, Ctx>()
+ .concurrentlyPerGroup(
+ (v) => v % 16,
+ (g) => g.sequentially((b) => b.pipe(inc('s1')).pipe(inc('s2')))
+ )
+ .build()
+ pipeline.feed(inputs(n))
+ assertCount(await drain(pipeline), n)
+ },
+ batching_full: async (n) => {
+ // feed() spreads `batchId` into each value, so the values are objects with a numeric field.
+ type V = { value: number; batchId?: number }
+ const pipeline = newBatchingPipeline<V, V, Ctx>(
+ (b) =>
+ b.pipe(function before(input) {
+ return Promise.resolve(ok({ elements: input.elements, batchContext: input.batchContext }))
+ }),
+ (b) =>
+ b
+ .pipeChunk(function prepare(values: V[]) {
+ return Promise.resolve(
+ values.map((v) => {
+ v.value += 1
+ return ok(v)
+ })
+ )
+ })
+ .concurrentlyPerGroup(
+ (v: V) => v.value % 16,
+ (g) =>
+ g.sequentially((s) =>
+ s.pipe(function work(v: V) {
+ v.value += 1
+ return Promise.resolve(ok(v))
+ })
+ ),
+ { maxConcurrency: 50 }
+ )
+ .gather(),
+ (b) =>
+ b.pipe(function after(input) {
+ return Promise.resolve(ok(input))
+ }),
+ { concurrentBatches: 1 }
+ )
+ const fed = await pipeline.feed(objectInputs(n), {})
+ if (!fed.ok) {
+ throw new Error(`feed rejected: ${fed.reason}`)
+ }
+ const done = await pipeline.next()
+ assertCount(done!.elements.length, n)
+ let sum = 0
+ for (const element of done!.elements) {
+ if (!isOkResult(element.result)) {
+ throw new Error('batching_full produced a non-ok result')
+ }
+ sum += element.result.value.value
+ }
+ assertCount(sum, batchingSum(n))
+ },
+}
+
+// Each input `i` passes two `+ 1` steps, so the outputs sum to `0 + 1 + ... + (n - 1) + 2n`.
+function batchingSum(n: number): number {
+ return (n * (n - 1)) / 2 + 2 * n
+}
+
+function assertCount(actual: number, expected: number): void {
+ if (actual !== expected) {
+ throw new Error(`expected ${expected} results, got ${actual}`)
+ }
+}
+
+async function bench(name: string, n: number, run: (n: number) => Promise<void>): Promise<void> {
+ const warmupUntil = performance.now() + Math.min(500, MIN_BENCH_MS / 4)
+ while (performance.now() < warmupUntil) {
+ await run(n)
+ }
+ let iterations = 0
+ const started = process.hrtime.bigint()
+ const deadline = performance.now() + MIN_BENCH_MS
+ while (performance.now() < deadline) {
+ await run(n)
+ iterations++
+ }
+ const elapsedNs = Number(process.hrtime.bigint() - started)
+ const nsPerElement = elapsedNs / (iterations * n)
+ console.log(
+ JSON.stringify({
+ impl: 'node',
+ workload: name,
+ size: n,
+ iterations,
+ ns_per_element: Math.round(nsPerElement),
+ elements_per_sec: Math.round(1e9 / nsPerElement),
+ })
+ )
+}
+
+async function main(): Promise<void> {
+ const only = process.argv[2]
+ for (const [name, run] of Object.entries(WORKLOADS)) {
+ if (only && !name.includes(only)) {
+ continue
+ }
+ for (const n of SIZES) {
+ await bench(name, n, run)
+ }
+ }
+}
+
+void main().then(
+ () => process.exit(0),
+ (error) => {
+ console.error(error)
+ process.exit(1)
+ }
+)
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 2e9c7b852ea8..4c45c73a6630 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -6667,6 +6667,28 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "ingestion-framework"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "bytes",
+ "chrono",
+ "criterion",
+ "futures",
+ "indexmap 2.13.0",
+ "metrics",
+ "metrics-exporter-prometheus",
+ "rand 0.8.5",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+]
+
[[package]]
name = "ingestion-worker-proto"
version = "0.1.0"
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index a06d02b8d485..8576a51fb387 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -60,6 +60,7 @@ members = [
"kafka-assigner",
"kafka-assigner-proto",
"ingestion-consumer",
+ "ingestion-framework",
"ingestion-control-plane",
"ingestion-worker-proto",
"kafka-deduplicator",
diff --git a/rust/ingestion-framework/Cargo.toml b/rust/ingestion-framework/Cargo.toml
new file mode 100644
index 000000000000..ffd01a213444
--- /dev/null
+++ b/rust/ingestion-framework/Cargo.toml
@@ -0,0 +1,33 @@
+[package]
+name = "ingestion-framework"
+version = "0.1.0"
+edition = "2021"
+description = "Rust port of the Node ingestion pipeline framework (nodejs/src/ingestion/framework)"
+
+[lib]
+path = "src/lib.rs"
+
+[dependencies]
+anyhow = { workspace = true }
+async-trait = { workspace = true }
+bytes = { workspace = true }
+chrono = { workspace = true }
+futures = { workspace = true }
+indexmap = { version = "2", features = ["serde"] }
+metrics = { workspace = true }
+rand = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+
+[dev-dependencies]
+criterion = { version = "0.5", features = ["async_tokio"] }
+metrics-exporter-prometheus = { workspace = true }
+tokio = { workspace = true, features = ["test-util"] }
+tracing-subscriber = { workspace = true }
+
+[[bench]]
+name = "framework"
+harness = false
diff --git a/rust/ingestion-framework/PARITY.md b/rust/ingestion-framework/PARITY.md
new file mode 100644
index 000000000000..e40b1ce46e5c
--- /dev/null
+++ b/rust/ingestion-framework/PARITY.md
@@ -0,0 +1,136 @@
+# Parity with the Node ingestion framework
+
+This crate is a port of `nodejs/src/ingestion/framework`.
+The two implementations are kept in step: a change to one lands with the same change to the other, and the mirrored tests below change together.
+
+## File map
+
+| Node (`nodejs/src/ingestion/framework/`) | Rust (`rust/ingestion-framework/src/`) |
+| --------------------------------------------- | ------------------------------------------------------------------------------------------ |
+| `results.ts` | `results.rs` |
+| `pipeline.interface.ts` | `context.rs` (types), `pipeline.rs` (`Pipeline` trait) |
+| `chunk-pipeline.interface.ts` | `chunk_pipeline.rs` (`ChunkPipeline` trait), `context.rs` (`ChunkResults`) |
+| `steps.ts` | `steps.rs` |
+| `start-pipeline.ts`, `step-pipeline.ts` | `pipeline.rs` |
+| `branching-pipeline.ts` | `branching.rs` |
+| `timeout-barrier-pipeline.ts` | `timeout_barrier.rs` |
+| `batch-budget.ts` | `budget.rs` |
+| `base-chunk-pipeline.ts` | `chunk_pipeline.rs` (`BaseChunkPipeline`, `apply_chunk_step_to_results`) |
+| `buffering-chunk-pipeline.ts` | `chunk_pipeline.rs` (`BufferingChunkPipeline`) |
+| `sequential-chunk-pipeline.ts` | `chunk_pipeline.rs` (`SequentialChunkPipeline`) |
+| `gathering-chunk-pipeline.ts` | `chunk_pipeline.rs` (`GatheringChunkPipeline`) |
+| `chunk-pipeline-unwrapper.ts` | `chunk_pipeline.rs` (`ChunkPipelineUnwrapper`) |
+| `concurrent-chunk-pipeline.ts` | `concurrent.rs` |
+| `concurrently-grouping-chunk-pipeline.ts` | `grouping.rs` |
+| `filter-map-chunk-pipeline.ts` | `filter_map.rs` |
+| `fan-out-fan-in-chunk-pipeline.ts` | `fan_out_fan_in.rs` |
+| `interleaving-chunk-pipeline.ts` | `interleaving.rs` |
+| `resettable-signal.ts` | `sync.rs` (`ResettableSignal`); `sync.rs` (`FifoLimiter`) replaces `p-limit` |
+| `result-handling-pipeline.ts` | `result_handling.rs` |
+| `result-handling-helpers.ts` | `result_handling.rs` (`produce_message_to_dlq`, `redirect_message_to_output`, headers) |
+| `side-effect-handling-pipeline.ts` | `side_effect_handling.rs` |
+| `ingestion-warning-handling-chunk-pipeline.ts`| `warning_handling.rs` |
+| `per-element-step.ts` | `per_element.rs` (`per_element`, also exported from `builders.rs`) |
+| `retry.ts` (+ `retryIfRetriable`) | `retry.rs` |
+| `batching-pipeline.ts` | `batching.rs` |
+| `metrics.ts` | `metrics.rs` (same metric names and labels) |
+| `helpers.ts` | `helpers.rs` |
+| `builders/pipeline-builders.ts` | `builders.rs` (`new_pipeline`, `PipelineBuilder`, `BranchingPipelineBuilder`) |
+| `builders/chunk-pipeline-builders.ts` | `builders.rs` (`new_chunk_pipeline`, `ChunkPipelineBuilder`, group/fan-out/result builders)|
+| `builders/helpers.ts` | `builders.rs` (`new_batching_pipeline`) |
+| `~/common/outputs` (`IngestionOutputs`) | `outputs.rs` (`IngestionOutputs` trait, `KafkaMessage`, `Headers`, `MemoryOutputs`) |
+| `~/common/utils/promise-scheduler` | `side_effects.rs` (`PromiseScheduler`, `SideEffectScheduler`) |
+| `~/ingestion/common/ingestion-warnings.ts` (`serializeIngestionWarning`, `emitIngestionWarning`) | `warning_handling.rs` (`serialize_ingestion_warning`, `OutputsWarningEmitter`), `js_json.rs` (`JSON.stringify` output) |
+| `~/ingestion/common/ingestion-warning-types.ts` (`INGESTION_WARNING_TYPES`) | `warning_handling.rs` (`warning_classification`), read from `rust/common/ingestion_warnings/warning_types.generated.json`, which the Node registry exports |
+| `span-attributes.ts`, `tophog/`, `extensions/`| not ported (see below) |
+
+## Mirrored tests
+
+Each living-docs chapter `docs/NN-*.test.ts` has a Rust twin `tests/docs_NN_*.rs`, with one `#[tokio::test]` per Node `it(...)`.
+The Rust test name is the snake_case form of the Node test name.
+Jest fake timers map to `#[tokio::test(start_paused = true)]`: tokio advances the paused clock by itself when every task waits, and `tokio::time::advance` replaces a synchronous `jest.advanceTimersByTime`.
+
+`per-element-step.test.ts` has the twin `tests/per_element_step.rs`.
+
+`tests/unit_semantics.rs` pins behavior the chapters leave implicit: poison-then-drain for the concurrent and grouping stages, feed-while-parked, FIFO admission under `max_concurrency`, filter-map passthrough order, DLQ header layout, DLQ rethrow, warnings on timeouts, fan-out DLQ aggregation, and concurrent batch feeds on a multi-thread runtime.
+
+`tests/review_regressions.rs` pins divergences that a review found and that are now fixed: the ingestion warning bytes, the fan-out `dlq_reason`, produce order on a multi-thread runtime, the redirect produce before `next()` returns, a dropped batching `next()`, scheduler cleanup, `tries: 0`, top-level retriability, a fan-in panic, `max_concurrency: Some(0)`, the cost of a group start, overlapping and cancelled scheduler waits, the parents a fan-in panic leaves ready, a group processor that panics before it returns its future, a dropped limiter `acquire()`, and retry options that give an out-of-range sleep.
+
+`consumeAll(pipeline, ms)` and `jest.advanceTimersByTimeAsync(ms)` before an `await` map to `consume_all_within` and `within_ms` in `tests/common`, which fail the test when the work needs more virtual time than Node gives it.
+
+### Skipped cases
+
+| Node test | Why |
+| ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `docs/16-tophog-metrics.test.ts` (all four cases) | TopHog (`framework/tophog/`) is a separate metrics aggregation subsystem, not part of the pipeline composition this crate ports. Port it as its own module if a Rust pipeline needs it. |
+
+Everything else is mirrored. Where a Node assertion relies on a JS runtime detail, the Rust test checks the same observable behavior another way:
+
+- `13-conventions` "pipelines are created via factory functions" uses `Arc::ptr_eq` for `not.toBe`.
+- `07-result-handling` "REDIRECT with awaitAck=true" checks that `next()` does not finish within 50 ms while the ack is held, instead of `setImmediate`.
+- `08-side-effects` "steps can add side effects" checks that the side effect ran before the step result is awaited, because `SideEffect::spawn` runs the future up to its first pending point, as a Node promise executor runs.
+
+## Known differences
+
+These are deliberate. Each one follows from a language difference. Items 1 to 15 do not change what a pipeline emits; items 16 to 19 change only timing or a failure path.
+
+1. **Step names are explicit.** Node reads `fn.name`. Rust closures have no name, so `step("name", f)` and `chunk_step("name", f)` take one. Use the Node function name.
+2. **A step error is `Err`, not a throw.** `PipelineError::retriable` / `non_retriable` stand in for `error.isRetriable`. An error with neither flag behaves like a Node error with no `isRetriable`: retried, then propagated. `is_retriable` reads the outermost error only, as Node reads the property of the thrown object: `PipelineError::non_retriable(..).context(..)` has no flag, so it retries. A `SharedError` reads through, because it is the same error object shared by more than one caller.
+3. **Side effects start on creation.** A Node promise runs its executor (or an async function its body up to the first `await`) when it is created; a Rust future runs only when polled. `SideEffect::spawn` polls the future once on the caller's thread and moves the unfinished rest to a tokio task, so DLQ, redirect and warning produces are issued in item order inside `next()`, as in Node. A panic in that first poll becomes a failed side effect, as a throw before the first `await` rejects a promise. A `SideEffect` cannot be cloned, so a step's side effects and warnings move from the result into the context (Node keeps a copy on both).
+4. **`ResultWithContext.result` is an `Outcome`.** Side effects and warnings live only on the context once a step finishes.
+5. **The user context is a field.** Node spreads `C` into the context object; Rust stores it in `PipelineContext.data` with `Deref`. `message_id` (Node's `BatchingContext`) is a base context field, so batching does not change the context type. `traceContext` is not ported.
+6. **Structural context access is a trait.** `message_aware`/`handle_results` require `HasMessage`; `team_aware`/`handle_ingestion_warnings` require `HasTeamId`.
+7. **`filter_map` output context.** Node types it as the union `CSubOutput | CIntermediate`. Rust requires `CSubOutput: From<CIntermediate>` and converts non-OK passthroughs with it.
+8. **Batch context is not spread into values.** Node's sub-pipeline values are `TInput & CBatch`. Rust's are `WithBatch<T, CB>` (value plus `Arc<CB>`, with `Deref` to the value). The caller's feed context returns as `BatchResult.feed_context` rather than merged into `batch_context`.
+9. **Retries clone the input.** `pipe_with_retry` needs `T: Clone + Sync`, because each attempt takes the value by value. Every attempt but the last possible one gets a clone, so `tries: 1` clones nothing; the first of several attempts cannot take the original, because a retry needs it again. Node passes the same reference.
+10. **Compile-time checks that become run-time.** Node checks branch exhaustiveness, redirect output names and warning types in the type system. Rust uses string names, so an unknown branch is the run-time DLQ that Node also has, an unknown output fails in the `IngestionOutputs` implementation, and an unregistered warning type fails its side effect, as Node's registry lookup throws at run time.
+11. **Ingestion warnings.** The Node `IngestionWarningLimiter` is an app concern: `IngestionWarningEmitter` is the seam, and `OutputsWarningEmitter` emits with no limiter. `serialize_ingestion_warning` writes the Node bytes with `js_json` (key order, `JSON.stringify` numbers and escapes, array-index keys first), with no dependence on serde_json's `preserve_order` feature. `category` and `severity` come from the registry JSON that the Node registry exports. `PipelineWarning.details` is an `IndexMap`, so its keys keep insertion order; build ordered details with `with_detail`, because a `json!` object sorts its keys unless the build enables `preserve_order`. A nested object inside a detail value is a `serde_json::Map` and is written in that map's order, which is sorted in a build without `preserve_order`; Node keeps insertion order there too. Keep details flat where the bytes must match.
+12. **Fan-out DLQ error.** Node builds `AggregateError(errors, reason)`; Rust builds `errors::AggregateError` with the same message and every sub error. The `dlq_reason` header is identical.
+13. **Concurrency.** Concurrent items and groups run as tokio tasks, so on a multi-thread runtime they run in parallel; Node interleaves them on one thread. Ordering guarantees (FIFO emission, per-key order, FIFO admission under a cap) are the same. When a group completes, the queues still waiting all belong to running groups, so their relative order no longer matters and the remove is O(1). The interleaving stage keeps the pending sub-pipeline pull as a stored boxed future so it survives across `next()` calls, which is how Node keeps its memoized promise. That future is polled only while a `next()` is awaited. Item and group work runs in its own tasks and never pauses, but sub-pipeline stages that the pull itself drives (for example a `sequentially` inside `filter_map`) advance only while the caller waits on `next()`. A Node promise advances in the background. This changes timing, not results or order; every consumer loops on `next()` anyway. A spawned task per `next()` was measured first and cost a cross-thread wake per emitted chunk.
+14. **Clock.** `BatchBudget` reads `tokio::time::Instant`, so it follows a paused test clock as Node's follows `Date.now()` under fake timers.
+15. **Not ported:** `span-attributes.ts` and OTel span parenting (Rust uses `tracing`), `tophog/`, and `extensions/`.
+16. **Cancel safety.** Node has no cancellation. `BatchingPipeline::next()` is cancel-safe: it keeps its sub-pipeline pull, the results it has not routed, and a running afterBatch hook under its pump lock, and the next call resumes them, so a dropped call (for example from an HTTP handler whose client went away) loses no batch and runs no hook twice. A `ChunkPipeline::next()` is not cancel-safe: the interleaving stages keep their pull, but a stage such as `pipe_chunk`, result handling or side-effect handling holds the chunk in the dropped future. Do not race a chunk pipeline's `next()` against a timeout. A dropped batching `next()` also leaves the hook paused until the next call polls it, where Node's would run on.
+17. **`PromiseScheduler` keeps the first failure.** Node removes a promise when it settles, so a rejection that settles before `waitForAll()` is not seen there. The Rust scheduler keeps it, as the `FailureLatchingPromiseScheduler` of the Node metrics consumer does, because a consumer that misses it stores the offsets of a batch whose side effect failed. A settled side effect leaves the set at the next `schedule()` that finds the set grown to twice its size after the last cleanup, at `pending_count()`, or when a `wait_for_all()` sees it settle. As in Node, a side effect stays in the set until it settles, so an overlapping `wait_for_all()` also waits for it, and a cancelled one leaves it, and its failure, for the next call. `wait_for_all()` waits for every side effect pending at the call and then returns the first failure; Node's `Promise.all` rejects at the first failure without waiting for the rest.
+18. **User callbacks that panic.** A panic in a grouping function, fan-out function or fan-in function becomes an error from `next()`, as a Node throw rejects `next()` and poisons the stage. No lock is held while user code runs. `max_concurrency: Some(0)` panics when the stage is built, as `pLimit(0)` throws.
+19. **`per_element` rejects late.** Node's `Promise.all` rejects at the first failed step, and the other steps run on. Rust waits for the other steps and then returns the first error to occur.
+
+## Benchmarks
+
+`benches/framework.rs` (criterion) and `nodejs/src/ingestion/framework/benchmarks/framework-bench.ts` run the same six workloads at sizes 1, 100 and 10 000, with trivial steps, so the numbers are framework overhead per element. The Rust benchmark installs a Prometheus recorder, so both sides pay for their metrics.
+
+```bash
+cd rust && cargo bench -p ingestion-framework --bench framework
+```
+
+```bash
+cd nodejs && DATABASE_URL=postgres://localhost/bench pnpm exec tsx src/ingestion/framework/benchmarks/framework-bench.ts
+```
+
+### Measured overhead
+
+One 4-vCPU cloud container, the two runs one after the other. Mean ns per element; lower is better. Rust numbers are on a current-thread runtime, which matches the single Node event loop.
+
+| Workload | Size | Node (ns/elem) | Rust (ns/elem) | Node ÷ Rust |
+| -------------------------------- | ------ | -------------- | -------------- | ----------- |
+| `step_pipeline_5_steps` | 1 | 30 383 | 2 379 | 12.8× |
+| `step_pipeline_5_steps` | 100 | 17 827 | 2 349 | 7.6× |
+| `step_pipeline_5_steps` | 10 000 | 19 693 | 2 307 | 8.5× |
+| `pipe_chunk_5_steps` | 1 | 25 940 | 3 483 | 7.4× |
+| `pipe_chunk_5_steps` | 100 | 722 | 431 | 1.7× |
+| `pipe_chunk_5_steps` | 10 000 | 740 | 541 | 1.4× |
+| `concurrently_max50` | 1 | 24 758 | 3 613 | 6.9× |
+| `concurrently_max50` | 100 | 8 488 | 1 873 | 4.5× |
+| `concurrently_max50` | 10 000 | 10 007 | 2 187 | 4.6× |
+| `per_element_max50` | 1 | 20 834 | 2 399 | 8.7× |
+| `per_element_max50` | 100 | 1 912 | 1 118 | 1.7× |
+| `per_element_max50` | 10 000 | 4 108 | 2 248 | 1.8× |
+| `concurrently_per_group_16` | 1 | 14 492 | 5 165 | 2.8× |
+| `concurrently_per_group_16` | 100 | 7 470 | 1 302 | 5.7× |
+| `concurrently_per_group_16` | 10 000 | 8 586 | 1 091 | 7.9× |
+| `batching_full` | 1 | 53 331 | 8 726 | 6.1× |
+| `batching_full` | 100 | 7 091 | 1 454 | 4.9× |
+| `batching_full` | 10 000 | 6 868 | 1 400 | 4.9× |
+
+On a 4-worker multi-thread runtime, the 10 000-element workloads run at 1 885 (`concurrently_max50`), 947 (`concurrently_per_group_16`) and 1 798 (`batching_full`) ns per element. Single-element batches cost about 50 µs there, which is worker wake-up latency, so a current-thread runtime suits a consumer that feeds tiny batches.
+
+The Node per-step cost is dominated by the per-step instrumentation (`instrumentFn`, the prom-client step histogram); the Rust port records the same histogram through the `metrics` crate.
diff --git a/rust/ingestion-framework/benches/framework.rs b/rust/ingestion-framework/benches/framework.rs
new file mode 100644
index 000000000000..7e82cdaacb79
--- /dev/null
+++ b/rust/ingestion-framework/benches/framework.rs
@@ -0,0 +1,257 @@
+//! Workloads and sizes match `framework-bench.ts`. A current-thread runtime matches the single Node event loop.
+
+use std::sync::Arc;
+
+use criterion::{criterion_group, BenchmarkId, Criterion, Throughput};
+use ingestion_framework::{
+ before_batch_step, chunk_step, create_ok_context, new_batching_pipeline, new_chunk_pipeline,
+ new_pipeline, ok, passthrough_after_batch_step, per_element, step, BatchingPipelineOptions,
+ ChunkProcessingStep, DynChunkPipeline, OkResultWithContext, Outcome, ProcessingStep, WithBatch,
+};
+
+const SIZES: [usize; 3] = [1, 100, 10_000];
+
+fn inc(name: &str) -> ProcessingStep<i64, i64> {
+ ProcessingStep::sync(name, |v: i64| Ok(ok(v + 1)))
+}
+
+fn inc_chunk(name: &str) -> ChunkProcessingStep<i64, i64> {
+ ChunkProcessingStep::sync(name, |v: Vec<i64>| {
+ Ok(v.into_iter().map(|x| ok(x + 1)).collect())
+ })
+}
+
+fn inputs(n: usize) -> Vec<OkResultWithContext<i64, ()>> {
+ (0..n as i64).map(|v| create_ok_context(v, ())).collect()
+}
+
+fn current_thread() -> tokio::runtime::Runtime {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+}
+
+fn multi_thread() -> tokio::runtime::Runtime {
+ tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(4)
+ .enable_all()
+ .build()
+ .unwrap()
+}
+
+async fn drain(pipeline: &DynChunkPipeline<i64, i64, (), ()>) -> usize {
+ let mut n = 0;
+ while let Some(chunk) = pipeline.next().await.unwrap() {
+ n += chunk.len();
+ }
+ n
+}
+
+fn step_pipeline_5_steps(c: &mut Criterion) {
+ let rt = current_thread();
+ let pipeline = new_pipeline::<i64, ()>()
+ .pipe(inc("s1"))
+ .pipe(inc("s2"))
+ .pipe(inc("s3"))
+ .pipe(inc("s4"))
+ .pipe(inc("s5"))
+ .build();
+ let mut group = c.benchmark_group("step_pipeline_5_steps");
+ for n in SIZES {
+ group.throughput(Throughput::Elements(n as u64));
+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {
+ b.to_async(&rt).iter(|| {
+ let pipeline = pipeline.clone();
+ async move {
+ for input in inputs(n) {
+ pipeline.process(input).await.unwrap();
+ }
+ }
+ })
+ });
+ }
+ group.finish();
+}
+
+fn pipe_chunk_chain(c: &mut Criterion) {
+ let rt = current_thread();
+ let mut group = c.benchmark_group("pipe_chunk_5_steps");
+ for n in SIZES {
+ group.throughput(Throughput::Elements(n as u64));
+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {
+ b.to_async(&rt).iter(|| async move {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(inc_chunk("c1"))
+ .pipe_chunk(inc_chunk("c2"))
+ .pipe_chunk(inc_chunk("c3"))
+ .pipe_chunk(inc_chunk("c4"))
+ .pipe_chunk(inc_chunk("c5"))
+ .build();
+ pipeline.feed(inputs(n));
+ assert_eq!(drain(&pipeline).await, n);
+ })
+ });
+ }
+ group.finish();
+}
+
+fn concurrently(c: &mut Criterion) {
+ for (label, rt) in [
+ ("concurrently_max50", current_thread()),
+ ("concurrently_max50_multi_thread", multi_thread()),
+ ] {
+ let mut group = c.benchmark_group(label);
+ for n in SIZES {
+ group.throughput(Throughput::Elements(n as u64));
+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {
+ b.to_async(&rt).iter(|| async move {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently(|b| b.pipe(inc("s1")).pipe(inc("s2")), Some(50))
+ .build();
+ pipeline.feed(inputs(n));
+ assert_eq!(drain(&pipeline).await, n);
+ })
+ });
+ }
+ group.finish();
+ }
+}
+
+fn per_element_max50(c: &mut Criterion) {
+ let rt = current_thread();
+ let mut group = c.benchmark_group("per_element_max50");
+ for n in SIZES {
+ group.throughput(Throughput::Elements(n as u64));
+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {
+ b.to_async(&rt).iter(|| async move {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(per_element(inc("s1"), Some(50)))
+ .pipe_chunk(per_element(inc("s2"), Some(50)))
+ .build();
+ pipeline.feed(inputs(n));
+ assert_eq!(drain(&pipeline).await, n);
+ })
+ });
+ }
+ group.finish();
+}
+
+fn concurrently_per_group(c: &mut Criterion) {
+ for (label, rt) in [
+ ("concurrently_per_group_16", current_thread()),
+ ("concurrently_per_group_16_multi_thread", multi_thread()),
+ ] {
+ let mut group = c.benchmark_group(label);
+ for n in SIZES {
+ group.throughput(Throughput::Elements(n as u64));
+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {
+ b.to_async(&rt).iter(|| async move {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently_per_group(
+ |v: &i64| v % 16,
+ |g| g.sequentially(|b| b.pipe(inc("s1")).pipe(inc("s2"))),
+ None,
+ )
+ .build();
+ pipeline.feed(inputs(n));
+ assert_eq!(drain(&pipeline).await, n);
+ })
+ });
+ }
+ group.finish();
+ }
+}
+
+type V = WithBatch<i64, u64>;
+
+fn batching(c: &mut Criterion) {
+ for (label, rt) in [
+ ("batching_full", current_thread()),
+ ("batching_full_multi_thread", multi_thread()),
+ ] {
+ let mut group = c.benchmark_group(label);
+ for n in SIZES {
+ group.throughput(Throughput::Elements(n as u64));
+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {
+ b.to_async(&rt).iter(|| async move {
+ let pipeline = Arc::new(new_batching_pipeline::<i64, V, (), (), u64, ()>(
+ |b| b.pipe(before_batch_step("before", |id| id)),
+ |b| {
+ b.pipe_chunk(chunk_step("prepare", |v: Vec<V>| async move {
+ Ok(v.into_iter()
+ .map(|mut x| {
+ x.value += 1;
+ ok(x)
+ })
+ .collect())
+ }))
+ .concurrently_per_group(
+ |v: &V| v.value % 16,
+ |g| {
+ g.sequentially(|s| {
+ s.pipe(step("work", |mut v: V| async move {
+ v.value += 1;
+ Ok(ok(v))
+ }))
+ })
+ },
+ Some(50),
+ )
+ .gather()
+ },
+ |b| b.pipe(passthrough_after_batch_step("after")),
+ BatchingPipelineOptions::default(),
+ ));
+ assert!(pipeline.feed(inputs(n), (), None).await.unwrap().is_ok());
+ let done = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(done.elements.len(), n);
+ // Each input `i` passes two `+ 1` steps, as in the Node workload.
+ let sum: i64 = done
+ .elements
+ .iter()
+ .map(|e| match &e.result {
+ Outcome::Ok(v) => v.value,
+ Outcome::NonOk(_) => panic!("batching_full produced a non-ok result"),
+ })
+ .sum();
+ let n = n as i64;
+ assert_eq!(sum, n * (n - 1) / 2 + 2 * n);
+ })
+ });
+ }
+ group.finish();
+ }
+}
+
+criterion_group!(
+ benches,
+ step_pipeline_5_steps,
+ pipe_chunk_chain,
+ concurrently,
+ per_element_max50,
+ concurrently_per_group,
+ batching
+);
+/// Without a recorder every metric call is a no-op, while Node updates real `prom-client` metrics.
+fn install_metrics_recorder() {
+ const BUCKETS: &[f64] = &[
+ 1.0, 5.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0,
+ ];
+ let handle = metrics_exporter_prometheus::PrometheusBuilder::new()
+ .set_buckets(BUCKETS)
+ .expect("buckets are not empty")
+ .install_recorder()
+ .expect("no other recorder is installed");
+ // Histogram samples stay buffered until upkeep drains them, as the production exporter does.
+ std::thread::spawn(move || loop {
+ std::thread::sleep(std::time::Duration::from_secs(1));
+ handle.run_upkeep();
+ });
+}
+
+fn main() {
+ install_metrics_recorder();
+ benches();
+ Criterion::default().configure_from_args().final_summary();
+}
diff --git a/rust/ingestion-framework/src/batching.rs b/rust/ingestion-framework/src/batching.rs
new file mode 100644
index 000000000000..c2219ccbb961
--- /dev/null
+++ b/rust/ingestion-framework/src/batching.rs
@@ -0,0 +1,458 @@
+//! Node spreads the batch context into each value. Here a value is `WithBatch<T, CB>`, and the feed context is a `BatchResult` field.
+
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::ops::{Deref, DerefMut};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+
+use futures::future::BoxFuture;
+use futures::FutureExt;
+
+use crate::budget::BatchBudget;
+use crate::chunk_pipeline::DynChunkPipeline;
+use crate::context::{create_ok_context, ChunkResults, OkResultWithContext, ResultWithContext};
+use crate::metrics::{BATCH_BUDGET_EXHAUSTED_COUNTER, BATCH_BUDGET_OVERRUN_HISTOGRAM};
+use crate::pipeline::DynPipeline;
+use crate::results::Outcome;
+use crate::side_effects::SideEffect;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FeedRejectionKind {
+ AtCapacity,
+ BeforeBatchFailed,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum FeedResult {
+ Ok,
+ Rejected {
+ kind: FeedRejectionKind,
+ reason: String,
+ },
+}
+
+impl FeedResult {
+ pub fn is_ok(&self) -> bool {
+ matches!(self, FeedResult::Ok)
+ }
+}
+
+#[derive(Debug)]
+pub struct WithBatch<T, CB> {
+ pub value: T,
+ pub batch: Arc<CB>,
+}
+
+impl<T, CB> Deref for WithBatch<T, CB> {
+ type Target = T;
+ fn deref(&self) -> &T {
+ &self.value
+ }
+}
+
+impl<T, CB> DerefMut for WithBatch<T, CB> {
+ fn deref_mut(&mut self) -> &mut T {
+ &mut self.value
+ }
+}
+
+impl<T: Clone, CB> Clone for WithBatch<T, CB> {
+ fn clone(&self) -> Self {
+ Self {
+ value: self.value.clone(),
+ batch: self.batch.clone(),
+ }
+ }
+}
+
+pub struct BeforeBatchInput<T, CI> {
+ pub elements: Vec<OkResultWithContext<T, CI>>,
+ pub batch_id: u64,
+}
+
+/// beforeBatch must return exactly as many elements as it received, or `feed()` fails.
+pub struct BeforeBatchOutput<T, CI, CB> {
+ pub elements: Vec<OkResultWithContext<T, CI>>,
+ pub batch_context: CB,
+ pub batch_id: u64,
+}
+
+pub struct AfterBatchInput<O, CO, CB> {
+ pub elements: ChunkResults<O, CO>,
+ pub batch_context: Arc<CB>,
+ pub batch_id: u64,
+}
+
+pub type AfterBatchOutput<O, CO, CB> = AfterBatchInput<O, CO, CB>;
+
+pub struct BatchResult<O, CO, CB, CF> {
+ pub elements: ChunkResults<O, CO>,
+ pub batch_context: Arc<CB>,
+ pub feed_context: CF,
+ pub batch_id: u64,
+ pub side_effects: Vec<SideEffect>,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct BatchingPipelineOptions {
+ pub concurrent_batches: usize,
+}
+
+impl Default for BatchingPipelineOptions {
+ fn default() -> Self {
+ Self {
+ concurrent_batches: 1,
+ }
+ }
+}
+
+// The overrun includes afterBatch, because the checkpoints only stop work from starting.
+fn record_budget_overrun(budget: &BatchBudget) {
+ if !budget.exhausted() {
+ return;
+ }
+ metrics::counter!(BATCH_BUDGET_EXHAUSTED_COUNTER).increment(1);
+ if let Some(at) = budget.soft_at() {
+ let overrun = tokio::time::Instant::now().saturating_duration_since(at);
+ metrics::histogram!(BATCH_BUDGET_OVERRUN_HISTOGRAM).record(overrun.as_secs_f64());
+ }
+}
+
+struct TrackedBatch<O, CO, CB, CF> {
+ batch_context: Arc<CB>,
+ feed_context: Option<CF>,
+ budget: BatchBudget,
+ message_ids: Vec<u64>,
+ inflight: HashSet<u64>,
+ results: HashMap<u64, ResultWithContext<O, CO>>,
+ before_side_effects: Vec<SideEffect>,
+}
+
+struct BatchingState<O, CO, CB, CF> {
+ next_batch_id: u64,
+ next_message_id: u64,
+ batches: HashMap<u64, TrackedBatch<O, CO, CB, CF>>,
+ message_to_batch: HashMap<u64, u64>,
+ completed: VecDeque<BatchResult<O, CO, CB, CF>>,
+}
+
+pub struct BatchingPipeline<T, O, CI, CO, CB, CF> {
+ sub: DynChunkPipeline<WithBatch<T, CB>, O, CI, CO>,
+ before: DynPipeline<BeforeBatchInput<T, CI>, BeforeBatchOutput<T, CI, CB>, ()>,
+ after: DynPipeline<AfterBatchInput<O, CO, CB>, AfterBatchOutput<O, CO, CB>, ()>,
+ options: BatchingPipelineOptions,
+ state: Mutex<BatchingState<O, CO, CB, CF>>,
+ // Bumped where a batch is fed, so the pump can tell a feed that raced its pull from lost messages.
+ feed_epoch: AtomicU64,
+ // The stages assume one caller. tokio's Mutex is FIFO-fair, like `pLimit(1)`.
+ feed_lock: tokio::sync::Mutex<()>,
+ // Pump progress lives here, not in the `next()` future, so a dropped `next()` loses nothing.
+ pump: tokio::sync::Mutex<PumpState<O, CO, CB, CF>>,
+}
+
+type SubPull<O, CO> = BoxFuture<'static, anyhow::Result<Option<ChunkResults<O, CO>>>>;
+type AfterRun<O, CO, CB> =
+ BoxFuture<'static, anyhow::Result<ResultWithContext<AfterBatchOutput<O, CO, CB>, ()>>>;
+
+struct PendingPull<O, CO> {
+ epoch: u64,
+ future: SubPull<O, CO>,
+}
+
+struct RunningAfter<O, CO, CB, CF> {
+ batch_id: u64,
+ batch: TrackedBatch<O, CO, CB, CF>,
+ future: AfterRun<O, CO, CB>,
+}
+
+struct PumpState<O, CO, CB, CF> {
+ pull: Option<PendingPull<O, CO>>,
+ unrouted: VecDeque<ResultWithContext<O, CO>>,
+ after: Option<RunningAfter<O, CO, CB, CF>>,
+}
+
+impl<T, O, CI, CO, CB, CF> BatchingPipeline<T, O, CI, CO, CB, CF>
+where
+ T: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+ CB: Send + Sync + 'static,
+ CF: Send + 'static,
+{
+ pub fn new(
+ sub: DynChunkPipeline<WithBatch<T, CB>, O, CI, CO>,
+ before: DynPipeline<BeforeBatchInput<T, CI>, BeforeBatchOutput<T, CI, CB>, ()>,
+ after: DynPipeline<AfterBatchInput<O, CO, CB>, AfterBatchOutput<O, CO, CB>, ()>,
+ options: BatchingPipelineOptions,
+ ) -> Self {
+ Self {
+ sub,
+ before,
+ after,
+ options,
+ state: Mutex::new(BatchingState {
+ next_batch_id: 0,
+ next_message_id: 0,
+ batches: HashMap::new(),
+ message_to_batch: HashMap::new(),
+ completed: VecDeque::new(),
+ }),
+ feed_epoch: AtomicU64::new(0),
+ feed_lock: tokio::sync::Mutex::new(()),
+ pump: tokio::sync::Mutex::new(PumpState {
+ pull: None,
+ unrouted: VecDeque::new(),
+ after: None,
+ }),
+ }
+ }
+
+ pub async fn feed(
+ &self,
+ elements: Vec<OkResultWithContext<T, CI>>,
+ feed_context: CF,
+ budget: Option<BatchBudget>,
+ ) -> anyhow::Result<FeedResult> {
+ let _guard = self.feed_lock.lock().await;
+ let budget = budget.unwrap_or_else(BatchBudget::unlimited);
+
+ // An empty feed could never complete, so it registers nothing and runs no hooks.
+ if elements.is_empty() {
+ return Ok(FeedResult::Ok);
+ }
+
+ let batch_id = {
+ let mut state = self.state.lock().unwrap();
+ if state.batches.len() >= self.options.concurrent_batches {
+ return Ok(FeedResult::Rejected {
+ kind: FeedRejectionKind::AtCapacity,
+ reason: format!(
+ "at concurrent batch capacity ({})",
+ self.options.concurrent_batches
+ ),
+ });
+ }
+ let id = state.next_batch_id;
+ state.next_batch_id += 1;
+ id
+ };
+
+ let fed_count = elements.len();
+ let before = self
+ .before
+ .process(create_ok_context(
+ BeforeBatchInput { elements, batch_id },
+ (),
+ ))
+ .await?;
+ let before_side_effects;
+ let output = match before.result {
+ Outcome::Ok(output) => {
+ before_side_effects = before.context.side_effects;
+ output
+ }
+ Outcome::NonOk(_) => {
+ return Ok(FeedResult::Rejected {
+ kind: FeedRejectionKind::BeforeBatchFailed,
+ reason: format!("beforeBatch hook returned non-ok result for batch {batch_id}"),
+ })
+ }
+ };
+
+ // A changed count leaves a batch that never completes, so fail before registering it.
+ if output.elements.len() != fed_count {
+ anyhow::bail!(
+ "batching_pipeline beforeBatch changed element count ({} -> {}) for batch {}",
+ fed_count,
+ output.elements.len(),
+ batch_id
+ );
+ }
+
+ let batch_context = Arc::new(output.batch_context);
+ let mut state = self.state.lock().unwrap();
+ let mut message_ids = Vec::with_capacity(fed_count);
+ let mut inflight = HashSet::with_capacity(fed_count);
+ let tagged: Vec<_> = output
+ .elements
+ .into_iter()
+ .map(|element| {
+ let message_id = state.next_message_id;
+ state.next_message_id += 1;
+ message_ids.push(message_id);
+ inflight.insert(message_id);
+ state.message_to_batch.insert(message_id, batch_id);
+ let mut context = element.context;
+ context.message_id = Some(message_id);
+ context.budget = Some(budget.clone());
+ OkResultWithContext {
+ value: WithBatch {
+ value: element.value,
+ batch: batch_context.clone(),
+ },
+ context,
+ }
+ })
+ .collect();
+ state.batches.insert(
+ batch_id,
+ TrackedBatch {
+ batch_context,
+ feed_context: Some(feed_context),
+ budget,
+ message_ids,
+ inflight,
+ results: HashMap::new(),
+ before_side_effects,
+ },
+ );
+ self.sub.feed(tagged);
+ // Bump after buffering, so a pump that reads the new epoch also finds the elements.
+ self.feed_epoch.fetch_add(1, Ordering::SeqCst);
+ Ok(FeedResult::Ok)
+ }
+
+ /// Cancel-safe: the next call resumes a dropped call's pull, results and afterBatch run.
+ pub async fn next(&self) -> anyhow::Result<Option<BatchResult<O, CO, CB, CF>>> {
+ let mut pump = self.pump.lock().await;
+ self.pump(&mut pump).await
+ }
+
+ async fn pump(
+ &self,
+ pump: &mut PumpState<O, CO, CB, CF>,
+ ) -> anyhow::Result<Option<BatchResult<O, CO, CB, CF>>> {
+ loop {
+ // Finish an afterBatch run first. A dropped call may have left one.
+ if let Some(running) = pump.after.as_mut() {
+ let after = running.future.as_mut().await;
+ let running = pump.after.take().expect("present");
+ self.complete_batch(running, after)?;
+ }
+
+ while let Some(result) = pump.unrouted.pop_front() {
+ if let Some(running) = self.route(result)? {
+ pump.after = Some(running);
+ break;
+ }
+ }
+ if pump.after.is_some() {
+ continue;
+ }
+
+ // Node returns a completed batch only after it routed the whole chunk.
+ if let Some(done) = self.state.lock().unwrap().completed.pop_front() {
+ return Ok(Some(done));
+ }
+
+ if pump.pull.is_none() {
+ let sub = self.sub.clone();
+ pump.pull = Some(PendingPull {
+ epoch: self.feed_epoch.load(Ordering::SeqCst),
+ future: async move { sub.next().await }.boxed(),
+ });
+ }
+ let pulled = pump.pull.as_mut().expect("present").future.as_mut().await;
+ let epoch_at_pull = pump.pull.take().expect("present").epoch;
+ let Some(results) = pulled? else {
+ let (batches, messages) = {
+ let state = self.state.lock().unwrap();
+ (state.batches.len(), state.message_to_batch.len())
+ };
+ if batches > 0 {
+ // A feed that landed during an empty pull is picked up by a retry.
+ if self.feed_epoch.load(Ordering::SeqCst) != epoch_at_pull {
+ continue;
+ }
+ anyhow::bail!(
+ "batching_pipeline sub-pipeline returned null with {batches} in-flight batches and {messages} in-flight messages"
+ );
+ }
+ return Ok(None);
+ };
+ pump.unrouted.extend(results);
+ }
+ }
+
+ fn route(
+ &self,
+ result: ResultWithContext<O, CO>,
+ ) -> anyhow::Result<Option<RunningAfter<O, CO, CB, CF>>> {
+ let mut state = self.state.lock().unwrap();
+ let message_id = result.context.message_id.ok_or_else(|| {
+ anyhow::anyhow!("batching_pipeline received result without messageId")
+ })?;
+ let Some(batch_id) = state.message_to_batch.get(&message_id).copied() else {
+ anyhow::bail!("batching_pipeline received result with unknown messageId {message_id}");
+ };
+ let Some(batch) = state.batches.get_mut(&batch_id) else {
+ anyhow::bail!(
+ "batching_pipeline has batchId mapping but batch {batch_id} is missing for messageId {message_id}"
+ );
+ };
+ batch.inflight.remove(&message_id);
+ batch.results.insert(message_id, result);
+ if !batch.inflight.is_empty() {
+ return Ok(None);
+ }
+ let mut batch = state.batches.remove(&batch_id).expect("present");
+ for id in &batch.message_ids {
+ state.message_to_batch.remove(id);
+ }
+ drop(state);
+ let ordered: Vec<_> = batch
+ .message_ids
+ .iter()
+ .map(|id| {
+ batch
+ .results
+ .remove(id)
+ .expect("every message has a result")
+ })
+ .collect();
+ let after = self.after.clone();
+ let input = create_ok_context(
+ AfterBatchInput {
+ elements: ordered,
+ batch_context: batch.batch_context.clone(),
+ batch_id,
+ },
+ (),
+ );
+ Ok(Some(RunningAfter {
+ batch_id,
+ batch,
+ future: async move { after.process(input).await }.boxed(),
+ }))
+ }
+
+ fn complete_batch(
+ &self,
+ running: RunningAfter<O, CO, CB, CF>,
+ after: anyhow::Result<ResultWithContext<AfterBatchOutput<O, CO, CB>, ()>>,
+ ) -> anyhow::Result<()> {
+ let RunningAfter {
+ batch_id,
+ mut batch,
+ ..
+ } = running;
+ let after = after?;
+ record_budget_overrun(&batch.budget);
+ let Outcome::Ok(after_output) = after.result else {
+ anyhow::bail!(
+ "batching_pipeline afterBatch hook returned non-ok result for batch {batch_id}"
+ );
+ };
+ let mut side_effects = std::mem::take(&mut batch.before_side_effects);
+ side_effects.extend(after.context.side_effects);
+ self.state.lock().unwrap().completed.push_back(BatchResult {
+ elements: after_output.elements,
+ batch_context: batch.batch_context,
+ feed_context: batch.feed_context.take().expect("set at feed"),
+ batch_id,
+ side_effects,
+ });
+ Ok(())
+ }
+}
diff --git a/rust/ingestion-framework/src/branching.rs b/rust/ingestion-framework/src/branching.rs
new file mode 100644
index 000000000000..02613c0679ce
--- /dev/null
+++ b/rust/ingestion-framework/src/branching.rs
@@ -0,0 +1,66 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+use crate::context::{OkResultWithContext, ResultWithContext};
+use crate::pipeline::{DynPipeline, Pipeline};
+use crate::results::{NonOk, Outcome};
+
+pub type BranchDecisionFn<T> = Arc<dyn Fn(&T) -> String + Send + Sync>;
+
+pub struct BranchingPipeline<I, M, O, C> {
+ decision: BranchDecisionFn<M>,
+ branches: HashMap<String, DynPipeline<M, O, C>>,
+ previous: DynPipeline<I, M, C>,
+}
+
+impl<I, M, O, C> BranchingPipeline<I, M, O, C> {
+ pub fn new(
+ decision: BranchDecisionFn<M>,
+ branches: HashMap<String, DynPipeline<M, O, C>>,
+ previous: DynPipeline<I, M, C>,
+ ) -> Self {
+ Self {
+ decision,
+ branches,
+ previous,
+ }
+ }
+}
+
+#[async_trait]
+impl<I, M, O, C> Pipeline<I, O, C> for BranchingPipeline<I, M, O, C>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ C: Send + 'static,
+{
+ async fn process(
+ &self,
+ input: OkResultWithContext<I, C>,
+ ) -> anyhow::Result<ResultWithContext<O, C>> {
+ let previous = self.previous.process(input).await?;
+ let ok = match previous.into_ok() {
+ Ok(ok) => ok,
+ Err((non_ok, context)) => {
+ return Ok(ResultWithContext::<M, C>::non_ok(non_ok, context))
+ }
+ };
+
+ let branch_name = (self.decision)(&ok.value);
+ match self.branches.get(&branch_name) {
+ Some(branch) => branch.process(ok).await,
+ None => Ok(ResultWithContext {
+ result: Outcome::NonOk(NonOk::Dlq {
+ reason: format!("Unknown branch: {branch_name}"),
+ error: Some(Arc::new(anyhow::anyhow!(
+ "Branch '{branch_name}' not found in branching pipeline"
+ ))),
+ }),
+ context: ok.context,
+ }),
+ }
+ }
+}
diff --git a/rust/ingestion-framework/src/budget.rs b/rust/ingestion-framework/src/budget.rs
new file mode 100644
index 000000000000..890fafce6f5b
--- /dev/null
+++ b/rust/ingestion-framework/src/budget.rs
@@ -0,0 +1,77 @@
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use tokio::time::Instant;
+
+/// Uses `tokio::time::Instant`, so a paused test clock controls it, as Jest fake timers do in Node.
+#[derive(Clone, Debug)]
+pub struct BatchBudget {
+ inner: Arc<BudgetInner>,
+}
+
+#[derive(Debug)]
+struct BudgetInner {
+ soft_at: Option<Instant>,
+ expired: AtomicBool,
+}
+
+impl BatchBudget {
+ pub fn soft_deadline(soft_at: Instant) -> Self {
+ Self {
+ inner: Arc::new(BudgetInner {
+ soft_at: Some(soft_at),
+ expired: AtomicBool::new(false),
+ }),
+ }
+ }
+
+ pub fn from_now(allowance: Duration) -> Self {
+ Self::soft_deadline(Instant::now() + allowance)
+ }
+
+ pub fn unlimited() -> Self {
+ Self {
+ inner: Arc::new(BudgetInner {
+ soft_at: None,
+ expired: AtomicBool::new(false),
+ }),
+ }
+ }
+
+ pub fn is_unlimited(&self) -> bool {
+ self.inner.soft_at.is_none()
+ }
+
+ pub fn soft_at(&self) -> Option<Instant> {
+ self.inner.soft_at
+ }
+
+ /// Reads the clock, not a timer flag, so a step that holds the thread past the deadline sees it spent.
+ pub fn exhausted(&self) -> bool {
+ if self.inner.expired.load(Ordering::Relaxed) {
+ return true;
+ }
+ match self.inner.soft_at {
+ None => false,
+ Some(at) => {
+ let expired = Instant::now() >= at;
+ if expired {
+ self.inner.expired.store(true, Ordering::Relaxed);
+ }
+ expired
+ }
+ }
+ }
+}
+
+const MAX_TIMER_DELAY_MS: u64 = (1 << 31) - 1;
+
+/// Zero, or an allowance above the Node timer limit, means no deadline.
+pub fn budget_deadline(armed_at: Instant, budget_ms: u64) -> Option<Instant> {
+ if budget_ms == 0 || budget_ms > MAX_TIMER_DELAY_MS {
+ None
+ } else {
+ Some(armed_at + Duration::from_millis(budget_ms))
+ }
+}
diff --git a/rust/ingestion-framework/src/builders.rs b/rust/ingestion-framework/src/builders.rs
new file mode 100644
index 000000000000..2afb2703b3ad
--- /dev/null
+++ b/rust/ingestion-framework/src/builders.rs
@@ -0,0 +1,588 @@
+//! Branch and redirect names are strings, so Node's compile-time coverage checks run at run time, and an unknown branch is a DLQ.
+
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::sync::Arc;
+
+use futures::FutureExt;
+
+pub use crate::per_element::per_element;
+
+use crate::batching::{
+ AfterBatchInput, AfterBatchOutput, BatchingPipeline, BatchingPipelineOptions, BeforeBatchInput,
+ BeforeBatchOutput, WithBatch,
+};
+use crate::branching::BranchingPipeline;
+use crate::chunk_pipeline::{
+ apply_chunk_step_to_results, BaseChunkPipeline, BufferingChunkPipeline, DynChunkPipeline,
+ GatheringChunkPipeline, SequentialChunkPipeline,
+};
+use crate::concurrent::ConcurrentChunkProcessingPipeline;
+use crate::context::ChunkResults;
+use crate::fan_out_fan_in::{
+ FanInFunction, FanOutFanInChunkPipeline, FanOutFunction, FanOutSubContext,
+};
+use crate::filter_map::{FilterMapChunkPipeline, FilterMapMappingFunction};
+use crate::grouping::{
+ process_group_chunk_sequentially, ConcurrentlyGroupingChunkPipeline, GroupChunkProcessor,
+};
+use crate::outputs::{HasMessage, HasTeamId};
+use crate::pipeline::{DynPipeline, StartPipeline, StepPipeline};
+use crate::result_handling::{PipelineConfig, ResultHandlingPipeline};
+use crate::retry::{with_chunk_retry, with_step_retry, RetryOptions};
+use crate::side_effect_handling::{
+ SideEffectHandlingConfig, SideEffectHandlingPipeline, SideEffectHandlingProcessor,
+};
+use crate::side_effects::SideEffectScheduler;
+use crate::steps::{ChunkProcessingStep, ProcessingStep};
+use crate::timeout_barrier::TimeoutBarrierPipeline;
+use crate::warning_handling::{IngestionWarningEmitter, IngestionWarningHandlingChunkPipeline};
+
+pub struct StartPipelineBuilder<T, C> {
+ _marker: std::marker::PhantomData<fn(T, C)>,
+}
+
+impl<T, C> Default for StartPipelineBuilder<T, C> {
+ fn default() -> Self {
+ Self {
+ _marker: std::marker::PhantomData,
+ }
+ }
+}
+
+pub fn new_pipeline<T, C>() -> StartPipelineBuilder<T, C> {
+ StartPipelineBuilder::default()
+}
+
+impl<T: Send + 'static, C: Send + 'static> StartPipelineBuilder<T, C> {
+ fn start(&self) -> DynPipeline<T, T, C> {
+ Arc::new(StartPipeline)
+ }
+
+ pub fn pipe<U: Send + 'static>(self, step: ProcessingStep<T, U>) -> PipelineBuilder<T, U, C> {
+ PipelineBuilder::new(Arc::new(StepPipeline::new(step, self.start())))
+ }
+
+ /// Retrying passes the value again, so it must be `Clone`.
+ pub fn pipe_with_retry<U: Send + 'static>(
+ self,
+ step: ProcessingStep<T, U>,
+ retry: RetryOptions,
+ ) -> PipelineBuilder<T, U, C>
+ where
+ T: Clone + Sync,
+ {
+ self.pipe(with_step_retry(step, retry))
+ }
+
+ pub fn branching<U: Send + 'static>(
+ self,
+ decision: impl Fn(&T) -> String + Send + Sync + 'static,
+ callback: impl FnOnce(
+ BranchingPipelineBuilder<T, T, U, C>,
+ ) -> BranchingPipelineBuilder<T, T, U, C>,
+ ) -> PipelineBuilder<T, U, C> {
+ let builder = BranchingPipelineBuilder {
+ decision: Arc::new(decision),
+ previous: self.start(),
+ branches: HashMap::new(),
+ };
+ PipelineBuilder::new(callback(builder).build())
+ }
+}
+
+pub struct PipelineBuilder<I, O, C> {
+ pipeline: DynPipeline<I, O, C>,
+}
+
+impl<I: Send + 'static, O: Send + 'static, C: Send + 'static> PipelineBuilder<I, O, C> {
+ pub fn new(pipeline: DynPipeline<I, O, C>) -> Self {
+ Self { pipeline }
+ }
+
+ pub fn pipe<U: Send + 'static>(self, step: ProcessingStep<O, U>) -> PipelineBuilder<I, U, C> {
+ PipelineBuilder::new(Arc::new(StepPipeline::new(step, self.pipeline)))
+ }
+
+ pub fn pipe_with_retry<U: Send + 'static>(
+ self,
+ step: ProcessingStep<O, U>,
+ retry: RetryOptions,
+ ) -> PipelineBuilder<I, U, C>
+ where
+ O: Clone + Sync,
+ {
+ self.pipe(with_step_retry(step, retry))
+ }
+
+ pub fn branching<U: Send + 'static>(
+ self,
+ decision: impl Fn(&O) -> String + Send + Sync + 'static,
+ callback: impl FnOnce(
+ BranchingPipelineBuilder<I, O, U, C>,
+ ) -> BranchingPipelineBuilder<I, O, U, C>,
+ ) -> PipelineBuilder<I, U, C> {
+ let builder = BranchingPipelineBuilder {
+ decision: Arc::new(decision),
+ previous: self.pipeline,
+ branches: HashMap::new(),
+ };
+ PipelineBuilder::new(callback(builder).build())
+ }
+
+ pub fn timeout_barrier(self) -> Self {
+ PipelineBuilder::new(Arc::new(TimeoutBarrierPipeline::new(self.pipeline)))
+ }
+
+ pub fn handle_side_effects(
+ self,
+ scheduler: Arc<dyn SideEffectScheduler>,
+ config: SideEffectHandlingConfig,
+ ) -> Self {
+ PipelineBuilder::new(Arc::new(SideEffectHandlingProcessor::new(
+ self.pipeline,
+ scheduler,
+ config,
+ )))
+ }
+
+ pub fn build(self) -> DynPipeline<I, O, C> {
+ self.pipeline
+ }
+}
+
+pub struct BranchingPipelineBuilder<I, M, O, C> {
+ decision: crate::branching::BranchDecisionFn<M>,
+ previous: DynPipeline<I, M, C>,
+ branches: HashMap<String, DynPipeline<M, O, C>>,
+}
+
+impl<I: Send + 'static, M: Send + 'static, O: Send + 'static, C: Send + 'static>
+ BranchingPipelineBuilder<I, M, O, C>
+{
+ pub fn branch(
+ mut self,
+ name: &str,
+ callback: impl FnOnce(StartPipelineBuilder<M, C>) -> PipelineBuilder<M, O, C>,
+ ) -> Self {
+ let pipeline = callback(StartPipelineBuilder::default()).build();
+ self.branches.insert(name.to_string(), pipeline);
+ self
+ }
+
+ pub fn build(self) -> DynPipeline<I, O, C> {
+ Arc::new(BranchingPipeline::new(
+ self.decision,
+ self.branches,
+ self.previous,
+ ))
+ }
+}
+
+pub fn new_chunk_pipeline<T: Send + 'static, C: Send + 'static>() -> ChunkPipelineBuilder<T, T, C, C>
+{
+ ChunkPipelineBuilder::new(Arc::new(BufferingChunkPipeline::<T, C>::new()))
+}
+
+pub struct ChunkPipelineBuilder<I, O, CI, CO> {
+ pipeline: DynChunkPipeline<I, O, CI, CO>,
+}
+
+impl<I, O, CI, CO> ChunkPipelineBuilder<I, O, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ pub fn new(pipeline: DynChunkPipeline<I, O, CI, CO>) -> Self {
+ Self { pipeline }
+ }
+
+ pub fn pipe_chunk<U: Send + 'static>(
+ self,
+ step: ChunkProcessingStep<O, U>,
+ ) -> ChunkPipelineBuilder<I, U, CI, CO> {
+ ChunkPipelineBuilder::new(Arc::new(BaseChunkPipeline::new(step, self.pipeline)))
+ }
+
+ pub fn pipe_chunk_with_retry<U: Send + 'static>(
+ self,
+ step: ChunkProcessingStep<O, U>,
+ retry: RetryOptions,
+ ) -> ChunkPipelineBuilder<I, U, CI, CO>
+ where
+ O: Clone + Sync,
+ {
+ self.pipe_chunk(with_chunk_retry(step, retry))
+ }
+
+ /// Panics on `Some(0)`, as Node's `pLimit(0)` throws.
+ pub fn concurrently<U: Send + 'static>(
+ self,
+ callback: impl FnOnce(StartPipelineBuilder<O, CO>) -> PipelineBuilder<O, U, CO>,
+ max_concurrency: Option<usize>,
+ ) -> ChunkPipelineBuilder<I, U, CI, CO> {
+ let processor = callback(StartPipelineBuilder::default()).build();
+ ChunkPipelineBuilder::new(Arc::new(ConcurrentChunkProcessingPipeline::new(
+ processor,
+ self.pipeline,
+ max_concurrency,
+ )))
+ }
+
+ pub fn sequentially<U: Send + 'static>(
+ self,
+ callback: impl FnOnce(StartPipelineBuilder<O, CO>) -> PipelineBuilder<O, U, CO>,
+ ) -> ChunkPipelineBuilder<I, U, CI, CO> {
+ let processor = callback(StartPipelineBuilder::default()).build();
+ ChunkPipelineBuilder::new(Arc::new(SequentialChunkPipeline::new(
+ processor,
+ self.pipeline,
+ )))
+ }
+
+ pub fn gather(self) -> Self {
+ ChunkPipelineBuilder::new(Arc::new(GatheringChunkPipeline::new(self.pipeline)))
+ }
+
+ pub fn filter_map<MM, U, CMM, CSO>(
+ self,
+ mapping: impl Fn(
+ crate::context::OkResultWithContext<O, CO>,
+ ) -> crate::context::OkResultWithContext<MM, CMM>
+ + Send
+ + Sync
+ + 'static,
+ sub_callback: impl FnOnce(
+ ChunkPipelineBuilder<MM, MM, CMM, CMM>,
+ ) -> ChunkPipelineBuilder<MM, U, CMM, CSO>,
+ ) -> ChunkPipelineBuilder<I, U, CI, CSO>
+ where
+ MM: Send + 'static,
+ U: Send + 'static,
+ CMM: Send + 'static,
+ CSO: From<CO> + Send + 'static,
+ {
+ let sub = sub_callback(new_chunk_pipeline::<MM, CMM>()).build();
+ let mapping: FilterMapMappingFunction<O, MM, CO, CMM> = Arc::new(mapping);
+ ChunkPipelineBuilder::new(Arc::new(FilterMapChunkPipeline::new(
+ self.pipeline,
+ mapping,
+ sub,
+ )))
+ }
+
+ pub fn fan_out<S: Send + 'static>(
+ self,
+ name: &str,
+ fan_out: impl Fn(&O) -> Vec<S> + Send + Sync + 'static,
+ ) -> FanOutBuilder<I, O, S, CI, CO> {
+ FanOutBuilder {
+ previous: self.pipeline,
+ name: name.to_string(),
+ fan_out: Arc::new(fan_out),
+ }
+ }
+
+ /// Panics on `Some(0)`, as Node's `pLimit(0)` throws.
+ pub fn concurrently_per_group<K, U>(
+ self,
+ grouping: impl Fn(&O) -> K + Send + Sync + 'static,
+ callback: impl FnOnce(GroupProcessingBuilder<O, O, CO>) -> GroupProcessingBuilder<O, U, CO>,
+ max_concurrency: Option<usize>,
+ ) -> ChunkPipelineBuilder<I, U, CI, CO>
+ where
+ K: Hash + Eq + Clone + Send + Sync + 'static,
+ U: Send + 'static,
+ {
+ let group = callback(GroupProcessingBuilder::identity());
+ ChunkPipelineBuilder::new(Arc::new(ConcurrentlyGroupingChunkPipeline::new(
+ Arc::new(grouping),
+ group.processor,
+ self.pipeline,
+ max_concurrency,
+ )))
+ }
+
+ pub fn handle_side_effects(
+ self,
+ scheduler: Arc<dyn SideEffectScheduler>,
+ config: SideEffectHandlingConfig,
+ ) -> Self {
+ ChunkPipelineBuilder::new(Arc::new(SideEffectHandlingPipeline::new(
+ self.pipeline,
+ scheduler,
+ config,
+ )))
+ }
+
+ pub fn message_aware<U, CU>(
+ self,
+ callback: impl FnOnce(Self) -> ChunkPipelineBuilder<I, U, CI, CU>,
+ ) -> MessageAwareChunkPipelineBuilder<I, U, CI, CU>
+ where
+ U: Send + 'static,
+ CU: HasMessage + Send + 'static,
+ {
+ MessageAwareChunkPipelineBuilder {
+ pipeline: callback(self).build(),
+ }
+ }
+
+ pub fn team_aware<U, CU>(
+ self,
+ callback: impl FnOnce(Self) -> ChunkPipelineBuilder<I, U, CI, CU>,
+ ) -> TeamAwareChunkPipelineBuilder<I, U, CI, CU>
+ where
+ U: Send + 'static,
+ CU: HasTeamId + Send + 'static,
+ {
+ TeamAwareChunkPipelineBuilder {
+ pipeline: callback(self).build(),
+ }
+ }
+
+ pub fn build(self) -> DynChunkPipeline<I, O, CI, CO> {
+ self.pipeline
+ }
+}
+
+pub struct GroupProcessingBuilder<T, U, C> {
+ processor: GroupChunkProcessor<T, U, C>,
+}
+
+impl<T: Send + 'static, C: Send + 'static> GroupProcessingBuilder<T, T, C> {
+ fn identity() -> Self {
+ Self {
+ processor: Arc::new(|items| futures::future::ready(Ok(items)).boxed()),
+ }
+ }
+}
+
+impl<T: Send + 'static, U: Send + 'static, C: Send + 'static> GroupProcessingBuilder<T, U, C> {
+ fn stage<V: Send + 'static>(
+ self,
+ next: GroupChunkProcessor<U, V, C>,
+ ) -> GroupProcessingBuilder<T, V, C> {
+ let first = self.processor;
+ GroupProcessingBuilder {
+ processor: Arc::new(move |items: ChunkResults<T, C>| {
+ let first = first.clone();
+ let next = next.clone();
+ async move { next(first(items).await?).await }.boxed()
+ }),
+ }
+ }
+
+ pub fn pipe_chunk<V: Send + 'static>(
+ self,
+ step: ChunkProcessingStep<U, V>,
+ ) -> GroupProcessingBuilder<T, V, C> {
+ self.stage(Arc::new(move |items| {
+ let step = step.clone();
+ async move { apply_chunk_step_to_results(&step, items).await }.boxed()
+ }))
+ }
+
+ pub fn sequentially<V: Send + 'static>(
+ self,
+ callback: impl FnOnce(StartPipelineBuilder<U, C>) -> PipelineBuilder<U, V, C>,
+ ) -> GroupProcessingBuilder<T, V, C> {
+ let processor = callback(StartPipelineBuilder::default()).build();
+ self.stage(process_group_chunk_sequentially(processor))
+ }
+}
+
+pub struct FanOutBuilder<I, O, S, CI, CO> {
+ previous: DynChunkPipeline<I, O, CI, CO>,
+ name: String,
+ fan_out: FanOutFunction<O, S>,
+}
+
+impl<I, O, S, CI, CO> FanOutBuilder<I, O, S, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ S: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ pub fn via<SO: Send + 'static>(
+ self,
+ callback: impl FnOnce(
+ ChunkPipelineBuilder<S, S, FanOutSubContext, FanOutSubContext>,
+ )
+ -> ChunkPipelineBuilder<S, SO, FanOutSubContext, FanOutSubContext>,
+ ) -> FanInBuilder<I, O, S, SO, CI, CO> {
+ FanInBuilder {
+ previous: self.previous,
+ fan_out_name: self.name,
+ fan_out: self.fan_out,
+ sub: callback(new_chunk_pipeline()).build(),
+ }
+ }
+}
+
+pub struct FanInBuilder<I, O, S, SO, CI, CO> {
+ previous: DynChunkPipeline<I, O, CI, CO>,
+ fan_out_name: String,
+ fan_out: FanOutFunction<O, S>,
+ sub: DynChunkPipeline<S, SO, FanOutSubContext, FanOutSubContext>,
+}
+
+impl<I, O, S, SO, CI, CO> FanInBuilder<I, O, S, SO, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ S: Send + 'static,
+ SO: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ pub fn fan_in<U: Send + 'static>(
+ self,
+ name: &str,
+ fan_in: impl Fn(O, Vec<SO>) -> U + Send + Sync + 'static,
+ ) -> ChunkPipelineBuilder<I, U, CI, CO> {
+ let fan_in: FanInFunction<O, SO, U> = Arc::new(fan_in);
+ ChunkPipelineBuilder::new(Arc::new(FanOutFanInChunkPipeline::new(
+ self.previous,
+ &self.fan_out_name,
+ self.fan_out,
+ self.sub,
+ name,
+ fan_in,
+ )))
+ }
+}
+
+pub struct MessageAwareChunkPipelineBuilder<I, O, CI, CO> {
+ pipeline: DynChunkPipeline<I, O, CI, CO>,
+}
+
+impl<I, O, CI, CO> MessageAwareChunkPipelineBuilder<I, O, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: HasMessage + Send + 'static,
+{
+ pub fn handle_results(
+ self,
+ config: PipelineConfig,
+ ) -> ResultHandledChunkPipelineBuilder<I, O, CI, CO> {
+ ResultHandledChunkPipelineBuilder {
+ pipeline: Arc::new(ResultHandlingPipeline::new(self.pipeline, config)),
+ }
+ }
+}
+
+/// The only next step is `handle_side_effects`, so side effects are always handled before build.
+pub struct ResultHandledChunkPipelineBuilder<I, O, CI, CO> {
+ pipeline: DynChunkPipeline<I, O, CI, CO>,
+}
+
+impl<I, O, CI, CO> ResultHandledChunkPipelineBuilder<I, O, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ pub fn handle_side_effects(
+ self,
+ scheduler: Arc<dyn SideEffectScheduler>,
+ config: SideEffectHandlingConfig,
+ ) -> ChunkPipelineBuilder<I, O, CI, CO> {
+ ChunkPipelineBuilder::new(Arc::new(SideEffectHandlingPipeline::new(
+ self.pipeline,
+ scheduler,
+ config,
+ )))
+ }
+}
+
+pub struct TeamAwareChunkPipelineBuilder<I, O, CI, CO> {
+ pipeline: DynChunkPipeline<I, O, CI, CO>,
+}
+
+impl<I, O, CI, CO> TeamAwareChunkPipelineBuilder<I, O, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: HasTeamId + Send + 'static,
+{
+ pub fn handle_ingestion_warnings(
+ self,
+ emitter: Arc<dyn IngestionWarningEmitter>,
+ ) -> ChunkPipelineBuilder<I, O, CI, CO> {
+ ChunkPipelineBuilder::new(Arc::new(IngestionWarningHandlingChunkPipeline::new(
+ emitter,
+ self.pipeline,
+ )))
+ }
+
+ pub fn into_builder(self) -> ChunkPipelineBuilder<I, O, CI, CO> {
+ ChunkPipelineBuilder::new(self.pipeline)
+ }
+}
+
+#[allow(clippy::type_complexity)]
+pub fn new_batching_pipeline<T, O, CI, CO, CB, CF>(
+ before: impl FnOnce(
+ StartPipelineBuilder<BeforeBatchInput<T, CI>, ()>,
+ )
+ -> PipelineBuilder<BeforeBatchInput<T, CI>, BeforeBatchOutput<T, CI, CB>, ()>,
+ batch: impl FnOnce(
+ ChunkPipelineBuilder<WithBatch<T, CB>, WithBatch<T, CB>, CI, CI>,
+ ) -> ChunkPipelineBuilder<WithBatch<T, CB>, O, CI, CO>,
+ after: impl FnOnce(
+ StartPipelineBuilder<AfterBatchInput<O, CO, CB>, ()>,
+ )
+ -> PipelineBuilder<AfterBatchInput<O, CO, CB>, AfterBatchOutput<O, CO, CB>, ()>,
+ options: BatchingPipelineOptions,
+) -> BatchingPipeline<T, O, CI, CO, CB, CF>
+where
+ T: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+ CB: Send + Sync + 'static,
+ CF: Send + 'static,
+{
+ let sub = batch(new_chunk_pipeline()).build();
+ let before = before(StartPipelineBuilder::default()).build();
+ let after = after(StartPipelineBuilder::default()).build();
+ BatchingPipeline::new(sub, before, after, options)
+}
+
+pub fn before_batch_step<T, CI, CB>(
+ name: &str,
+ make_context: impl Fn(u64) -> CB + Send + Sync + 'static,
+) -> ProcessingStep<BeforeBatchInput<T, CI>, BeforeBatchOutput<T, CI, CB>>
+where
+ T: Send + 'static,
+ CI: Send + 'static,
+ CB: Send + 'static,
+{
+ ProcessingStep::sync(name, move |input: BeforeBatchInput<T, CI>| {
+ Ok(crate::results::ok(BeforeBatchOutput {
+ batch_context: make_context(input.batch_id),
+ elements: input.elements,
+ batch_id: input.batch_id,
+ }))
+ })
+}
+
+pub fn passthrough_after_batch_step<O, CO, CB>(
+ name: &str,
+) -> ProcessingStep<AfterBatchInput<O, CO, CB>, AfterBatchOutput<O, CO, CB>>
+where
+ O: Send + 'static,
+ CO: Send + 'static,
+ CB: Send + Sync + 'static,
+{
+ ProcessingStep::sync(name, |input| Ok(crate::results::ok(input)))
+}
diff --git a/rust/ingestion-framework/src/chunk_pipeline.rs b/rust/ingestion-framework/src/chunk_pipeline.rs
new file mode 100644
index 000000000000..c7ea524beefe
--- /dev/null
+++ b/rust/ingestion-framework/src/chunk_pipeline.rs
@@ -0,0 +1,321 @@
+use std::sync::{Arc, Mutex};
+use std::time::Instant;
+
+use async_trait::async_trait;
+
+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};
+use crate::metrics::{record_budget_checkpoint, record_step_duration, BudgetCheckpoint};
+use crate::pipeline::DynPipeline;
+use crate::results::{NonOk, Outcome};
+use crate::steps::ChunkProcessingStep;
+
+/// `&self`, because a stage takes a `feed()` while a `next()` is parked, as in Node. Never overlap two `next()` calls.
+#[async_trait]
+pub trait ChunkPipeline<I, O, CI, CO>: Send + Sync {
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>);
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>>;
+}
+
+pub type DynChunkPipeline<I, O, CI, CO> = Arc<dyn ChunkPipeline<I, O, CI, CO>>;
+
+pub struct BufferingChunkPipeline<T, C> {
+ buffer: Mutex<Vec<ResultWithContext<T, C>>>,
+}
+
+impl<T, C> Default for BufferingChunkPipeline<T, C> {
+ fn default() -> Self {
+ Self {
+ buffer: Mutex::new(Vec::new()),
+ }
+ }
+}
+
+impl<T, C> BufferingChunkPipeline<T, C> {
+ pub fn new() -> Self {
+ Self::default()
+ }
+}
+
+#[async_trait]
+impl<T: Send + 'static, C: Send + 'static> ChunkPipeline<T, T, C, C>
+ for BufferingChunkPipeline<T, C>
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<T, C>>) {
+ self.buffer
+ .lock()
+ .unwrap()
+ .extend(elements.into_iter().map(OkResultWithContext::into_result));
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<T, C>>> {
+ let results = std::mem::take(&mut *self.buffer.lock().unwrap());
+ Ok(if results.is_empty() {
+ None
+ } else {
+ Some(results)
+ })
+ }
+}
+
+/// The budget check runs per element, because one chunk can hold elements of batches with different budgets.
+pub async fn apply_chunk_step_to_results<T, U, C>(
+ step: &ChunkProcessingStep<T, U>,
+ items: ChunkResults<T, C>,
+) -> anyhow::Result<ChunkResults<U, C>>
+where
+ T: Send + 'static,
+ U: Send + 'static,
+{
+ let step_name = step.name();
+ enum Slot<U, C> {
+ Done(ResultWithContext<U, C>),
+ CutOff(crate::context::PipelineContext<C>),
+ Pending(crate::context::PipelineContext<C>),
+ }
+
+ let mut slots: Vec<Slot<U, C>> = Vec::with_capacity(items.len());
+ let mut values: Vec<T> = Vec::with_capacity(items.len());
+ for item in items {
+ match item.result {
+ Outcome::NonOk(non_ok) => slots.push(Slot::Done(ResultWithContext {
+ result: Outcome::NonOk(non_ok),
+ context: item.context,
+ })),
+ Outcome::Ok(value) => {
+ if item.context.budget.as_ref().is_some_and(|b| b.exhausted()) {
+ record_budget_checkpoint(BudgetCheckpoint::Chunk, step_name.clone());
+ slots.push(Slot::CutOff(item.context));
+ } else {
+ values.push(value);
+ slots.push(Slot::Pending(item.context));
+ }
+ }
+ }
+ }
+
+ let mut step_results = Vec::new();
+ if !values.is_empty() {
+ let value_count = values.len();
+ let started = Instant::now();
+ match step.call(values).await {
+ Ok(results) => {
+ record_step_duration(
+ step_name.clone(),
+ "chunk",
+ "chunk",
+ started.elapsed().as_secs_f64(),
+ );
+ step_results = results;
+ }
+ Err(error) => {
+ record_step_duration(
+ step_name.clone(),
+ "chunk",
+ "exception",
+ started.elapsed().as_secs_f64(),
+ );
+ // Node logs the origin of every OK item, also of the items the budget cut off.
+ let debug_contexts: Vec<_> = slots
+ .iter()
+ .filter_map(|s| match s {
+ Slot::Pending(c) | Slot::CutOff(c) => c.debug_context.clone(),
+ Slot::Done(_) => None,
+ })
+ .collect();
+ tracing::error!(
+ step = %step_name,
+ error = %error,
+ chunk_size = value_count,
+ debug_context = ?debug_contexts,
+ "Chunk step threw"
+ );
+ return Err(error);
+ }
+ }
+ if step_results.len() != value_count {
+ anyhow::bail!(
+ "Chunk pipeline step {} returned different number of results than input values: {} !== {}",
+ step_name,
+ step_results.len(),
+ value_count
+ );
+ }
+ }
+
+ let mut step_results = step_results.into_iter();
+ Ok(slots
+ .into_iter()
+ .map(|slot| match slot {
+ Slot::Done(done) => done,
+ Slot::CutOff(context) => ResultWithContext {
+ result: Outcome::NonOk(NonOk::Timeout {
+ reason: format!("budget exceeded before {step_name}"),
+ }),
+ context,
+ },
+ Slot::Pending(mut context) => {
+ let result = step_results.next().expect("length checked above");
+ let outcome = context.absorb_step(step_name, result);
+ ResultWithContext {
+ result: outcome,
+ context,
+ }
+ }
+ })
+ .collect())
+}
+
+pub struct BaseChunkPipeline<I, M, O, CI, CO> {
+ step: ChunkProcessingStep<M, O>,
+ previous: DynChunkPipeline<I, M, CI, CO>,
+}
+
+impl<I, M, O, CI, CO> BaseChunkPipeline<I, M, O, CI, CO> {
+ pub fn new(step: ChunkProcessingStep<M, O>, previous: DynChunkPipeline<I, M, CI, CO>) -> Self {
+ Self { step, previous }
+ }
+}
+
+#[async_trait]
+impl<I, M, O, CI, CO> ChunkPipeline<I, O, CI, CO> for BaseChunkPipeline<I, M, O, CI, CO>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.previous.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ match self.previous.next().await? {
+ None => Ok(None),
+ Some(results) => Ok(Some(
+ apply_chunk_step_to_results(&self.step, results).await?,
+ )),
+ }
+ }
+}
+
+pub struct SequentialChunkPipeline<I, M, O, CI, CO> {
+ processor: DynPipeline<M, O, CO>,
+ previous: DynChunkPipeline<I, M, CI, CO>,
+}
+
+impl<I, M, O, CI, CO> SequentialChunkPipeline<I, M, O, CI, CO> {
+ pub fn new(processor: DynPipeline<M, O, CO>, previous: DynChunkPipeline<I, M, CI, CO>) -> Self {
+ Self {
+ processor,
+ previous,
+ }
+ }
+}
+
+pub(crate) async fn process_sequentially<M, O, C>(
+ processor: &DynPipeline<M, O, C>,
+ items: ChunkResults<M, C>,
+) -> anyhow::Result<ChunkResults<O, C>>
+where
+ M: Send + 'static,
+ O: Send + 'static,
+ C: Send + 'static,
+{
+ let mut results = Vec::with_capacity(items.len());
+ for item in items {
+ match item.into_ok() {
+ Ok(ok) => results.push(processor.process(ok).await?),
+ Err((non_ok, context)) => {
+ results.push(ResultWithContext::<M, C>::non_ok(non_ok, context))
+ }
+ }
+ }
+ Ok(results)
+}
+
+#[async_trait]
+impl<I, M, O, CI, CO> ChunkPipeline<I, O, CI, CO> for SequentialChunkPipeline<I, M, O, CI, CO>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.previous.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ match self.previous.next().await? {
+ None => Ok(None),
+ Some(results) => Ok(Some(process_sequentially(&self.processor, results).await?)),
+ }
+ }
+}
+
+pub struct GatheringChunkPipeline<I, O, CI, CO> {
+ sub: DynChunkPipeline<I, O, CI, CO>,
+}
+
+impl<I, O, CI, CO> GatheringChunkPipeline<I, O, CI, CO> {
+ pub fn new(sub: DynChunkPipeline<I, O, CI, CO>) -> Self {
+ Self { sub }
+ }
+}
+
+#[async_trait]
+impl<I, O, CI, CO> ChunkPipeline<I, O, CI, CO> for GatheringChunkPipeline<I, O, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.sub.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ let mut all = Vec::new();
+ while let Some(chunk) = self.sub.next().await? {
+ all.extend(chunk);
+ }
+ Ok(if all.is_empty() { None } else { Some(all) })
+ }
+}
+
+pub struct ChunkPipelineUnwrapper<I, O, C> {
+ pipeline: DynChunkPipeline<I, O, C, C>,
+}
+
+impl<I: Send + 'static, O: Send + 'static, C: Send + 'static> ChunkPipelineUnwrapper<I, O, C> {
+ pub fn new(pipeline: DynChunkPipeline<I, O, C, C>) -> Self {
+ Self { pipeline }
+ }
+
+ pub fn feed(&self, elements: Vec<OkResultWithContext<I, C>>) {
+ self.pipeline.feed(elements)
+ }
+
+ pub async fn next(&self) -> anyhow::Result<Option<Vec<O>>> {
+ let Some(results) = self.pipeline.next().await? else {
+ return Ok(None);
+ };
+ let mut values = Vec::with_capacity(results.len());
+ let mut side_effects = 0;
+ for result in results {
+ side_effects += result.context.side_effects.len();
+ if let Outcome::Ok(value) = result.result {
+ values.push(value);
+ }
+ }
+ if side_effects > 0 {
+ tracing::warn!(
+ "ChunkPipelineUnwrapper found {side_effects} remaining side effects that were not handled"
+ );
+ }
+ Ok(Some(values))
+ }
+}
diff --git a/rust/ingestion-framework/src/concurrent.rs b/rust/ingestion-framework/src/concurrent.rs
new file mode 100644
index 000000000000..66873abc0089
--- /dev/null
+++ b/rust/ingestion-framework/src/concurrent.rs
@@ -0,0 +1,124 @@
+use std::collections::VecDeque;
+use std::sync::{Arc, Mutex};
+
+use async_trait::async_trait;
+use tokio::task::JoinHandle;
+
+use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};
+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};
+use crate::errors::join_error;
+use crate::interleaving::{InterleavingCallbacks, InterleavingChunkPipeline, PullOutcome};
+use crate::pipeline::DynPipeline;
+use crate::sync::FifoLimiter;
+
+enum Queued<O, C> {
+ Ready(ResultWithContext<O, C>),
+ Running(JoinHandle<anyhow::Result<ResultWithContext<O, C>>>),
+}
+
+struct ConcurrentCallbacks<I, M, O, CI, CO> {
+ processor: DynPipeline<M, O, CO>,
+ previous: DynChunkPipeline<I, M, CI, CO>,
+ limit: Option<Arc<FifoLimiter>>,
+ queue: Mutex<VecDeque<Queued<O, CO>>>,
+}
+
+#[async_trait]
+impl<I, M, O, CI, CO> InterleavingCallbacks<I, O, CI, CO> for ConcurrentCallbacks<I, M, O, CI, CO>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.previous.feed(elements);
+ }
+
+ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<O, CO>> {
+ let Some(previous) = self.previous.next().await? else {
+ return Ok(PullOutcome::Drained);
+ };
+ let mut queue = self.queue.lock().unwrap();
+ for item in previous {
+ match item.into_ok() {
+ Ok(ok) => {
+ let processor = self.processor.clone();
+ // Admission is reserved in pull order, so the head of the queue gets a slot first and emission order holds.
+ let permit = self.limit.as_ref().map(|limit| limit.acquire());
+ queue.push_back(Queued::Running(tokio::spawn(async move {
+ let _permit = match permit {
+ Some(permit) => Some(permit.await),
+ None => None,
+ };
+ processor.process(ok).await
+ })));
+ }
+ Err((non_ok, context)) => queue.push_back(Queued::Ready(
+ ResultWithContext::<M, CO>::non_ok(non_ok, context),
+ )),
+ }
+ }
+ Ok(PullOutcome::Drain)
+ }
+
+ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ let head = self.queue.lock().unwrap().pop_front();
+ match head {
+ None => Ok(None),
+ Some(Queued::Ready(result)) => Ok(Some(vec![result])),
+ Some(Queued::Running(handle)) => Ok(Some(vec![handle.await.map_err(join_error)??])),
+ }
+ }
+}
+
+pub struct ConcurrentChunkProcessingPipeline<I, M, O, CI, CO> {
+ inner: Inner<I, M, O, CI, CO>,
+}
+
+type Inner<I, M, O, CI, CO> =
+ InterleavingChunkPipeline<I, O, CI, CO, ConcurrentCallbacks<I, M, O, CI, CO>>;
+
+impl<I, M, O, CI, CO> ConcurrentChunkProcessingPipeline<I, M, O, CI, CO>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ pub fn new(
+ processor: DynPipeline<M, O, CO>,
+ previous: DynChunkPipeline<I, M, CI, CO>,
+ max_concurrency: Option<usize>,
+ ) -> Self {
+ Self {
+ inner: InterleavingChunkPipeline::new(Arc::new(ConcurrentCallbacks {
+ processor,
+ previous,
+ limit: max_concurrency.map(FifoLimiter::new),
+ queue: Mutex::new(VecDeque::new()),
+ })),
+ }
+ }
+}
+
+#[async_trait]
+impl<I, M, O, CI, CO> ChunkPipeline<I, O, CI, CO>
+ for ConcurrentChunkProcessingPipeline<I, M, O, CI, CO>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.inner.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ self.inner.next().await
+ }
+}
diff --git a/rust/ingestion-framework/src/context.rs b/rust/ingestion-framework/src/context.rs
new file mode 100644
index 000000000000..bb8c94568dc7
--- /dev/null
+++ b/rust/ingestion-framework/src/context.rs
@@ -0,0 +1,195 @@
+use std::ops::{Deref, DerefMut};
+use std::sync::Arc;
+
+use indexmap::IndexMap;
+use serde::{Deserialize, Serialize};
+
+use crate::budget::BatchBudget;
+use crate::results::{NonOk, Outcome, PipelineResult};
+use crate::side_effects::SideEffect;
+
+/// Keys keep insertion order, as in a JavaScript object, and re-inserting a key keeps its place.
+pub type WarningDetails = IndexMap<String, serde_json::Value>;
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct PipelineWarning {
+ #[serde(rename = "type")]
+ pub warning_type: String,
+ pub details: WarningDetails,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub pipeline_step: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub key: Option<String>,
+ #[serde(default)]
+ pub always_send: bool,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub source: Option<String>,
+}
+
+impl PipelineWarning {
+ /// A `json!` object has sorted keys without serde_json's `preserve_order`, so use `with_detail` where key order matters.
+ pub fn new(warning_type: impl Into<String>, details: serde_json::Value) -> Self {
+ Self {
+ warning_type: warning_type.into(),
+ details: match details {
+ serde_json::Value::Object(map) => map.into_iter().collect(),
+ _ => WarningDetails::new(),
+ },
+ pipeline_step: None,
+ key: None,
+ always_send: false,
+ source: None,
+ }
+ }
+
+ pub fn with_detail(
+ mut self,
+ key: impl Into<String>,
+ value: impl Into<serde_json::Value>,
+ ) -> Self {
+ self.details.insert(key.into(), value.into());
+ self
+ }
+}
+
+/// Node spreads `C` into the context. Here `C` lives in `data` behind `Deref`, and `message_id` lives here so batching keeps the type.
+pub struct PipelineContext<C> {
+ pub data: C,
+ pub last_step: Option<Arc<str>>,
+ pub debug_context: Option<serde_json::Value>,
+ pub budget: Option<BatchBudget>,
+ pub message_id: Option<u64>,
+ pub side_effects: Vec<SideEffect>,
+ pub warnings: Vec<PipelineWarning>,
+}
+
+impl<C> PipelineContext<C> {
+ pub fn new(data: C) -> Self {
+ Self {
+ data,
+ last_step: None,
+ debug_context: None,
+ budget: None,
+ message_id: None,
+ side_effects: Vec::new(),
+ warnings: Vec::new(),
+ }
+ }
+
+ pub fn last_step(&self) -> Option<&str> {
+ self.last_step.as_deref()
+ }
+
+ pub fn map_data<D>(self, f: impl FnOnce(C) -> D) -> PipelineContext<D> {
+ PipelineContext {
+ data: f(self.data),
+ last_step: self.last_step,
+ debug_context: self.debug_context,
+ budget: self.budget,
+ message_id: self.message_id,
+ side_effects: self.side_effects,
+ warnings: self.warnings,
+ }
+ }
+
+ pub(crate) fn absorb_step<T>(
+ &mut self,
+ step_name: &Arc<str>,
+ result: PipelineResult<T>,
+ ) -> Outcome<T> {
+ self.last_step = Some(step_name.clone());
+ self.side_effects.extend(result.side_effects);
+ self.warnings.extend(result.warnings);
+ result.outcome
+ }
+}
+
+impl<C: Default> Default for PipelineContext<C> {
+ fn default() -> Self {
+ Self::new(C::default())
+ }
+}
+
+impl<C> Deref for PipelineContext<C> {
+ type Target = C;
+ fn deref(&self) -> &C {
+ &self.data
+ }
+}
+
+impl<C> DerefMut for PipelineContext<C> {
+ fn deref_mut(&mut self) -> &mut C {
+ &mut self.data
+ }
+}
+
+impl<C: std::fmt::Debug> std::fmt::Debug for PipelineContext<C> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("PipelineContext")
+ .field("data", &self.data)
+ .field("last_step", &self.last_step)
+ .field("message_id", &self.message_id)
+ .field("side_effects", &self.side_effects.len())
+ .field("warnings", &self.warnings)
+ .finish()
+ }
+}
+
+#[derive(Debug)]
+pub struct ResultWithContext<T, C> {
+ pub result: Outcome<T>,
+ pub context: PipelineContext<C>,
+}
+
+impl<T, C> ResultWithContext<T, C> {
+ pub fn is_ok(&self) -> bool {
+ self.result.is_ok()
+ }
+
+ pub fn value(&self) -> Option<&T> {
+ self.result.value()
+ }
+
+ // Boxing the error side would allocate for each non-OK result on every step.
+ #[allow(clippy::result_large_err)]
+ pub fn into_ok(self) -> Result<OkResultWithContext<T, C>, (NonOk, PipelineContext<C>)> {
+ match self.result {
+ Outcome::Ok(value) => Ok(OkResultWithContext {
+ value,
+ context: self.context,
+ }),
+ Outcome::NonOk(non_ok) => Err((non_ok, self.context)),
+ }
+ }
+
+ pub(crate) fn non_ok<U>(non_ok: NonOk, context: PipelineContext<C>) -> ResultWithContext<U, C> {
+ ResultWithContext {
+ result: Outcome::NonOk(non_ok),
+ context,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub struct OkResultWithContext<T, C> {
+ pub value: T,
+ pub context: PipelineContext<C>,
+}
+
+impl<T, C> OkResultWithContext<T, C> {
+ pub fn into_result(self) -> ResultWithContext<T, C> {
+ ResultWithContext {
+ result: Outcome::Ok(self.value),
+ context: self.context,
+ }
+ }
+}
+
+pub fn create_ok_context<T, C>(value: T, data: C) -> OkResultWithContext<T, C> {
+ OkResultWithContext {
+ value,
+ context: PipelineContext::new(data),
+ }
+}
+
+pub type ChunkResults<T, C> = Vec<ResultWithContext<T, C>>;
diff --git a/rust/ingestion-framework/src/errors.rs b/rust/ingestion-framework/src/errors.rs
new file mode 100644
index 000000000000..ce31d7d21bb6
--- /dev/null
+++ b/rust/ingestion-framework/src/errors.rs
@@ -0,0 +1,121 @@
+//! `is_retriable` reads only the top-level error, as Node reads `isRetriable` on the thrown object and not on its `cause`.
+
+use std::fmt;
+use std::sync::Arc;
+
+#[derive(Debug, thiserror::Error)]
+#[error("{message}")]
+pub struct PipelineError {
+ pub message: String,
+ /// `Some(true)` retries, `Some(false)` goes to the DLQ, `None` retries and then crashes.
+ pub retriable: Option<bool>,
+}
+
+impl PipelineError {
+ pub fn retriable(message: impl Into<String>) -> anyhow::Error {
+ anyhow::Error::new(Self {
+ message: message.into(),
+ retriable: Some(true),
+ })
+ }
+
+ pub fn non_retriable(message: impl Into<String>) -> anyhow::Error {
+ anyhow::Error::new(Self {
+ message: message.into(),
+ retriable: Some(false),
+ })
+ }
+}
+
+/// A `context()` wrapper reads as `None`, as in Node. A `SharedError` is one error object that several callers share, so it reads through.
+pub fn is_retriable(error: &anyhow::Error) -> Option<bool> {
+ // `anyhow`'s `downcast_ref` also finds a context wrapper's inner error, so check the outermost.
+ let outermost: &(dyn std::error::Error + 'static) = error.chain().next()?;
+ if let Some(pipeline_error) = outermost.downcast_ref::<PipelineError>() {
+ return pipeline_error.retriable;
+ }
+ if let Some(shared) = outermost.downcast_ref::<SharedError>() {
+ return is_retriable(&shared.0);
+ }
+ None
+}
+
+/// A poisoned pipeline rejects every later `next()` with the same error, and `anyhow::Error` is not `Clone`.
+#[derive(Clone)]
+pub struct SharedError(pub Arc<anyhow::Error>);
+
+impl fmt::Debug for SharedError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt::Debug::fmt(&*self.0, f)
+ }
+}
+
+impl fmt::Display for SharedError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt::Display::fmt(&*self.0, f)
+ }
+}
+
+impl std::error::Error for SharedError {
+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+ self.0.source()
+ }
+}
+
+pub fn shared_error(error: Arc<anyhow::Error>) -> anyhow::Error {
+ anyhow::Error::new(SharedError(error))
+}
+
+pub fn join_error(error: tokio::task::JoinError) -> anyhow::Error {
+ anyhow::anyhow!("pipeline task failed: {error}")
+}
+
+/// As in JavaScript's `AggregateError`, the message is `message` only, and a `None` sub error is Node's `undefined`.
+pub struct AggregateError {
+ pub message: String,
+ pub errors: Vec<Option<Arc<anyhow::Error>>>,
+}
+
+impl fmt::Display for AggregateError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str(&self.message)
+ }
+}
+
+impl fmt::Debug for AggregateError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("AggregateError")
+ .field("message", &self.message)
+ .field("errors", &self.errors)
+ .finish()
+ }
+}
+
+impl std::error::Error for AggregateError {}
+
+pub fn panic_message(payload: &(dyn std::any::Any + Send)) -> String {
+ if let Some(message) = payload.downcast_ref::<&str>() {
+ (*message).to_string()
+ } else if let Some(message) = payload.downcast_ref::<String>() {
+ message.clone()
+ } else {
+ "unknown panic".to_string()
+ }
+}
+
+/// A panic becomes an error logged with the element's origin, as a Node callback throw rejects `next()`.
+pub fn call_user_fn<R>(
+ kind: &str,
+ name: &str,
+ debug_context: Option<&serde_json::Value>,
+ f: impl FnOnce() -> R,
+) -> anyhow::Result<R> {
+ match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
+ Ok(value) => Ok(value),
+ Err(payload) => {
+ let message = panic_message(payload.as_ref());
+ tracing::error!(error = %message, debug_context = ?debug_context, "{kind} {name} threw");
+ Err(anyhow::anyhow!("{kind} {name} threw: {message}"))
+ }
+ }
+}
diff --git a/rust/ingestion-framework/src/fan_out_fan_in.rs b/rust/ingestion-framework/src/fan_out_fan_in.rs
new file mode 100644
index 000000000000..a9f4c8d93048
--- /dev/null
+++ b/rust/ingestion-framework/src/fan_out_fan_in.rs
@@ -0,0 +1,372 @@
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+
+use async_trait::async_trait;
+
+use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};
+use crate::context::{ChunkResults, OkResultWithContext, PipelineContext, ResultWithContext};
+use crate::errors::{call_user_fn, AggregateError};
+use crate::interleaving::{InterleavingCallbacks, InterleavingChunkPipeline, PullOutcome};
+use crate::results::{NonOk, Outcome};
+
+/// Must be cheap, because heavy work belongs in the sub-pipeline.
+pub type FanOutFunction<T, S> = Arc<dyn Fn(&T) -> Vec<S> + Send + Sync>;
+
+pub type FanInFunction<T, SO, U> = Arc<dyn Fn(T, Vec<SO>) -> U + Send + Sync>;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct FanOutParentRef(u64);
+
+#[derive(Debug, Clone)]
+pub struct FanOutSubContext {
+ parent: FanOutParentRef,
+}
+
+impl FanOutSubContext {
+ pub fn parent(&self) -> FanOutParentRef {
+ self.parent
+ }
+}
+
+struct PendingParent<T, SO, C> {
+ original: T,
+ context: PipelineContext<C>,
+ outstanding: usize,
+ total: usize,
+ collected: Vec<SO>,
+ dlq_failures: Vec<(String, Option<Arc<anyhow::Error>>)>,
+ dlq_last_step: Option<Arc<str>>,
+ timed_out: Option<(String, Option<Arc<str>>)>,
+}
+
+enum Settled<T, SO, U, C> {
+ Done(ResultWithContext<U, C>),
+ FanIn {
+ original: T,
+ context: PipelineContext<C>,
+ collected: Vec<SO>,
+ },
+}
+
+struct FanState<T, SO, U, C> {
+ pending: HashMap<FanOutParentRef, PendingParent<T, SO, C>>,
+ ready: Vec<ResultWithContext<U, C>>,
+}
+
+struct FanCallbacks<I, T, S, SO, U, CI, C> {
+ previous: DynChunkPipeline<I, T, CI, C>,
+ fan_out: FanOutFunction<T, S>,
+ sub: DynChunkPipeline<S, SO, FanOutSubContext, FanOutSubContext>,
+ fan_in: FanInFunction<T, SO, U>,
+ fan_out_name: Arc<str>,
+ fan_in_name: Arc<str>,
+ next_ref: AtomicU64,
+ state: Mutex<FanState<T, SO, U, C>>,
+}
+
+impl<I, T, S, SO, U, CI, C> FanCallbacks<I, T, S, SO, U, CI, C> {
+ /// Call it with no lock held, so a panic in user code cannot poison the stage state.
+ fn complete_parent(
+ &self,
+ original: T,
+ mut context: PipelineContext<C>,
+ collected: Vec<SO>,
+ ) -> anyhow::Result<ResultWithContext<U, C>> {
+ context.last_step = Some(self.fan_in_name.clone());
+ let merged = call_user_fn(
+ "Fan-in",
+ &self.fan_in_name,
+ context.debug_context.as_ref(),
+ || (self.fan_in)(original, collected),
+ )?;
+ Ok(ResultWithContext {
+ result: Outcome::Ok(merged),
+ context,
+ })
+ }
+
+ fn finish(&self, settled: Settled<T, SO, U, C>) -> anyhow::Result<ResultWithContext<U, C>> {
+ match settled {
+ Settled::Done(result) => Ok(result),
+ Settled::FanIn {
+ original,
+ context,
+ collected,
+ } => self.complete_parent(original, context, collected),
+ }
+ }
+
+ fn settle(
+ &self,
+ state: &mut FanState<T, SO, U, C>,
+ sub: ResultWithContext<SO, FanOutSubContext>,
+ out: &mut Vec<Settled<T, SO, U, C>>,
+ ) -> anyhow::Result<()> {
+ let parent_ref = sub.context.data.parent;
+ let Some(parent) = state.pending.get_mut(&parent_ref) else {
+ anyhow::bail!(
+ "Fan-out/fan-in ({}/{}) received a sub-result without a known parent tag — the subpipeline must preserve context",
+ self.fan_out_name,
+ self.fan_in_name
+ );
+ };
+ let mut sub_context = sub.context;
+ parent
+ .context
+ .side_effects
+ .append(&mut sub_context.side_effects);
+ parent.context.warnings.append(&mut sub_context.warnings);
+ match sub.result {
+ Outcome::Ok(value) => parent.collected.push(value),
+ Outcome::NonOk(NonOk::Timeout { reason }) => {
+ if parent.timed_out.is_none() {
+ parent.timed_out = Some((reason, sub_context.last_step.clone()));
+ }
+ }
+ Outcome::NonOk(NonOk::Dlq { reason, error }) => {
+ parent.dlq_failures.push((reason, error));
+ parent.dlq_last_step = sub_context.last_step.clone();
+ }
+ Outcome::NonOk(NonOk::Drop { .. }) => {}
+ Outcome::NonOk(redirect @ NonOk::Redirect { .. }) => {
+ tracing::warn!(
+ fan_out_step = %self.fan_out_name,
+ fan_in_step = %self.fan_in_name,
+ reason = redirect.reason(),
+ "Fan-out subpipeline produced a redirect result; excluding it"
+ );
+ }
+ }
+ parent.outstanding -= 1;
+ if parent.outstanding > 0 {
+ return Ok(());
+ }
+ let mut parent = state.pending.remove(&parent_ref).expect("present");
+ let result = if let Some((reason, last_step)) = parent.timed_out.take() {
+ parent.context.last_step = last_step;
+ Settled::Done(ResultWithContext {
+ result: Outcome::NonOk(NonOk::Timeout {
+ reason: format!("fan-out sub-element timed out: {reason}"),
+ }),
+ context: parent.context,
+ })
+ } else if !parent.dlq_failures.is_empty() {
+ let mut reasons: Vec<&str> = Vec::new();
+ for (reason, _) in &parent.dlq_failures {
+ if !reasons.contains(&reason.as_str()) {
+ reasons.push(reason);
+ }
+ }
+ let reason = format!(
+ "{}/{} fan-out sub-elements dlq'd: {}",
+ parent.dlq_failures.len(),
+ parent.total,
+ reasons.join("; ")
+ );
+ parent.context.last_step = parent.dlq_last_step.take();
+ let error = AggregateError {
+ message: reason.clone(),
+ errors: parent.dlq_failures.into_iter().map(|(_, e)| e).collect(),
+ };
+ Settled::Done(ResultWithContext {
+ result: Outcome::NonOk(NonOk::Dlq {
+ error: Some(Arc::new(anyhow::Error::new(error))),
+ reason,
+ }),
+ context: parent.context,
+ })
+ } else {
+ Settled::FanIn {
+ original: parent.original,
+ context: parent.context,
+ collected: parent.collected,
+ }
+ };
+ out.push(result);
+ Ok(())
+ }
+}
+
+#[async_trait]
+impl<I, T, S, SO, U, CI, C> InterleavingCallbacks<I, U, CI, C>
+ for FanCallbacks<I, T, S, SO, U, CI, C>
+where
+ I: Send + 'static,
+ T: Send + 'static,
+ S: Send + 'static,
+ SO: Send + 'static,
+ U: Send + 'static,
+ CI: Send + 'static,
+ C: Send + 'static,
+{
+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.previous.feed(elements);
+ }
+
+ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<U, C>> {
+ let Some(previous) = self.previous.next().await? else {
+ return Ok(PullOutcome::Drained);
+ };
+ let mut settled = Vec::new();
+ let mut subs = Vec::new();
+ let mut parents = Vec::new();
+ // User callbacks run before the lock is taken, so a panic cannot poison it.
+ for element in previous {
+ let ok = match element.into_ok() {
+ Ok(ok) => ok,
+ Err((n, context)) => {
+ settled.push(ResultWithContext::<T, C>::non_ok(n, context));
+ continue;
+ }
+ };
+ let sub_values = call_user_fn(
+ "Fan-out",
+ &self.fan_out_name,
+ ok.context.debug_context.as_ref(),
+ || (self.fan_out)(&ok.value),
+ )?;
+ if sub_values.is_empty() {
+ settled.push(self.complete_parent(ok.value, ok.context, Vec::new())?);
+ continue;
+ }
+ let parent_ref = FanOutParentRef(self.next_ref.fetch_add(1, Ordering::Relaxed));
+ let total = sub_values.len();
+ for sub in sub_values {
+ let mut context = PipelineContext::new(FanOutSubContext { parent: parent_ref });
+ context.debug_context = ok.context.debug_context.clone();
+ context.budget = ok.context.budget.clone();
+ subs.push(OkResultWithContext {
+ value: sub,
+ context,
+ });
+ }
+ parents.push((
+ parent_ref,
+ PendingParent {
+ original: ok.value,
+ context: ok.context,
+ outstanding: total,
+ total,
+ collected: Vec::new(),
+ dlq_failures: Vec::new(),
+ dlq_last_step: None,
+ timed_out: None,
+ },
+ ));
+ }
+ if !parents.is_empty() {
+ self.state.lock().unwrap().pending.extend(parents);
+ }
+ let fed = !subs.is_empty();
+ if fed {
+ self.sub.feed(subs);
+ }
+ if !settled.is_empty() {
+ return Ok(PullOutcome::Emit(settled));
+ }
+ if !fed {
+ return Ok(PullOutcome::Emit(Vec::new()));
+ }
+ Ok(PullOutcome::Drain)
+ }
+
+ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<U, C>>> {
+ loop {
+ {
+ let mut state = self.state.lock().unwrap();
+ if !state.ready.is_empty() {
+ return Ok(Some(std::mem::take(&mut state.ready)));
+ }
+ }
+ let Some(sub_results) = self.sub.next().await? else {
+ let pending = self.state.lock().unwrap().pending.len();
+ if pending > 0 {
+ anyhow::bail!(
+ "Fan-out/fan-in ({}/{}) subpipeline drained with {} parent(s) still pending — sub-results were lost",
+ self.fan_out_name,
+ self.fan_in_name,
+ pending
+ );
+ }
+ return Ok(None);
+ };
+ let mut done = Vec::new();
+ {
+ let mut state = self.state.lock().unwrap();
+ for sub_result in sub_results {
+ self.settle(&mut state, sub_result, &mut done)?;
+ }
+ }
+ // A parent is ready before the next fan-in runs, so the poisoned drain returns it, as Node's readyResults does.
+ for settled in done {
+ let finished = self.finish(settled)?;
+ self.state.lock().unwrap().ready.push(finished);
+ }
+ }
+ }
+}
+
+pub struct FanOutFanInChunkPipeline<I, T, S, SO, U, CI, C> {
+ inner: Inner<I, T, S, SO, U, CI, C>,
+}
+
+type Inner<I, T, S, SO, U, CI, C> =
+ InterleavingChunkPipeline<I, U, CI, C, FanCallbacks<I, T, S, SO, U, CI, C>>;
+
+impl<I, T, S, SO, U, CI, C> FanOutFanInChunkPipeline<I, T, S, SO, U, CI, C>
+where
+ I: Send + 'static,
+ T: Send + 'static,
+ S: Send + 'static,
+ SO: Send + 'static,
+ U: Send + 'static,
+ CI: Send + 'static,
+ C: Send + 'static,
+{
+ pub fn new(
+ previous: DynChunkPipeline<I, T, CI, C>,
+ fan_out_name: &str,
+ fan_out: FanOutFunction<T, S>,
+ sub: DynChunkPipeline<S, SO, FanOutSubContext, FanOutSubContext>,
+ fan_in_name: &str,
+ fan_in: FanInFunction<T, SO, U>,
+ ) -> Self {
+ Self {
+ inner: InterleavingChunkPipeline::new(Arc::new(FanCallbacks {
+ previous,
+ fan_out,
+ sub,
+ fan_in,
+ fan_out_name: Arc::from(fan_out_name),
+ fan_in_name: Arc::from(fan_in_name),
+ next_ref: AtomicU64::new(0),
+ state: Mutex::new(FanState {
+ pending: HashMap::new(),
+ ready: Vec::new(),
+ }),
+ })),
+ }
+ }
+}
+
+#[async_trait]
+impl<I, T, S, SO, U, CI, C> ChunkPipeline<I, U, CI, C>
+ for FanOutFanInChunkPipeline<I, T, S, SO, U, CI, C>
+where
+ I: Send + 'static,
+ T: Send + 'static,
+ S: Send + 'static,
+ SO: Send + 'static,
+ U: Send + 'static,
+ CI: Send + 'static,
+ C: Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.inner.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<U, C>>> {
+ self.inner.next().await
+ }
+}
diff --git a/rust/ingestion-framework/src/filter_map.rs b/rust/ingestion-framework/src/filter_map.rs
new file mode 100644
index 000000000000..70eaf47ec071
--- /dev/null
+++ b/rust/ingestion-framework/src/filter_map.rs
@@ -0,0 +1,124 @@
+//! Rust has no union types, so the output context implements `From` the earlier one for passthroughs.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};
+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};
+use crate::interleaving::{InterleavingCallbacks, InterleavingChunkPipeline, PullOutcome};
+
+pub type FilterMapMappingFunction<T, U, C, CM> =
+ Arc<dyn Fn(OkResultWithContext<T, C>) -> OkResultWithContext<U, CM> + Send + Sync>;
+
+struct FilterMapCallbacks<I, M, MM, O, CI, CM, CMM, CO> {
+ previous: DynChunkPipeline<I, M, CI, CM>,
+ mapping: FilterMapMappingFunction<M, MM, CM, CMM>,
+ sub: DynChunkPipeline<MM, O, CMM, CO>,
+}
+
+#[async_trait]
+impl<I, M, MM, O, CI, CM, CMM, CO> InterleavingCallbacks<I, O, CI, CO>
+ for FilterMapCallbacks<I, M, MM, O, CI, CM, CMM, CO>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ MM: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CM: Send + 'static,
+ CMM: Send + 'static,
+ CO: From<CM> + Send + 'static,
+{
+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.previous.feed(elements);
+ }
+
+ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<O, CO>> {
+ let Some(previous) = self.previous.next().await? else {
+ return Ok(PullOutcome::Drained);
+ };
+ let mut ok = Vec::new();
+ let mut non_ok = Vec::new();
+ for element in previous {
+ match element.into_ok() {
+ Ok(element) => ok.push((self.mapping)(element)),
+ Err((n, context)) => non_ok.push(ResultWithContext::<M, CO>::non_ok(
+ n,
+ context.map_data(CO::from),
+ )),
+ }
+ }
+ let had_ok = !ok.is_empty();
+ if had_ok {
+ self.sub.feed(ok);
+ }
+ if !non_ok.is_empty() {
+ return Ok(PullOutcome::Emit(non_ok));
+ }
+ // A non-empty upstream chunk with nothing in it surfaces as an empty chunk, 1:1.
+ if !had_ok {
+ return Ok(PullOutcome::Emit(Vec::new()));
+ }
+ Ok(PullOutcome::Drain)
+ }
+
+ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ self.sub.next().await
+ }
+}
+
+pub struct FilterMapChunkPipeline<I, M, MM, O, CI, CM, CMM, CO> {
+ inner: Inner<I, M, MM, O, CI, CM, CMM, CO>,
+}
+
+type Inner<I, M, MM, O, CI, CM, CMM, CO> =
+ InterleavingChunkPipeline<I, O, CI, CO, FilterMapCallbacks<I, M, MM, O, CI, CM, CMM, CO>>;
+
+impl<I, M, MM, O, CI, CM, CMM, CO> FilterMapChunkPipeline<I, M, MM, O, CI, CM, CMM, CO>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ MM: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CM: Send + 'static,
+ CMM: Send + 'static,
+ CO: From<CM> + Send + 'static,
+{
+ pub fn new(
+ previous: DynChunkPipeline<I, M, CI, CM>,
+ mapping: FilterMapMappingFunction<M, MM, CM, CMM>,
+ sub: DynChunkPipeline<MM, O, CMM, CO>,
+ ) -> Self {
+ Self {
+ inner: InterleavingChunkPipeline::new(Arc::new(FilterMapCallbacks {
+ previous,
+ mapping,
+ sub,
+ })),
+ }
+ }
+}
+
+#[async_trait]
+impl<I, M, MM, O, CI, CM, CMM, CO> ChunkPipeline<I, O, CI, CO>
+ for FilterMapChunkPipeline<I, M, MM, O, CI, CM, CMM, CO>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ MM: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CM: Send + 'static,
+ CMM: Send + 'static,
+ CO: From<CM> + Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.inner.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ self.inner.next().await
+ }
+}
diff --git a/rust/ingestion-framework/src/grouping.rs b/rust/ingestion-framework/src/grouping.rs
new file mode 100644
index 000000000000..ef05f697da08
--- /dev/null
+++ b/rust/ingestion-framework/src/grouping.rs
@@ -0,0 +1,276 @@
+use std::collections::{HashSet, VecDeque};
+use std::hash::Hash;
+use std::panic::AssertUnwindSafe;
+use std::sync::{Arc, Mutex};
+
+use async_trait::async_trait;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use indexmap::IndexMap;
+
+use crate::chunk_pipeline::{process_sequentially, ChunkPipeline, DynChunkPipeline};
+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};
+use crate::errors::call_user_fn;
+use crate::interleaving::{InterleavingCallbacks, InterleavingChunkPipeline, PullOutcome};
+use crate::pipeline::DynPipeline;
+use crate::sync::{FifoLimiter, ResettableSignal};
+
+pub type GroupingFunction<T, K> = Arc<dyn Fn(&T) -> K + Send + Sync>;
+
+pub type GroupChunkProcessor<T, U, C> = Arc<
+ dyn Fn(ChunkResults<T, C>) -> BoxFuture<'static, anyhow::Result<ChunkResults<U, C>>>
+ + Send
+ + Sync,
+>;
+
+pub fn process_group_chunk_sequentially<T, U, C>(
+ processor: DynPipeline<T, U, C>,
+) -> GroupChunkProcessor<T, U, C>
+where
+ T: Send + 'static,
+ U: Send + 'static,
+ C: Send + 'static,
+{
+ Arc::new(move |items| {
+ let processor = processor.clone();
+ async move { process_sequentially(&processor, items).await }.boxed()
+ })
+}
+
+struct GroupState<K, M, O, C> {
+ // Insertion order matters: groups start in the order their keys first arrived.
+ queues: IndexMap<K, Vec<ResultWithContext<M, C>>>,
+ active: HashSet<K>,
+ completed: VecDeque<ChunkResults<O, C>>,
+ failure: Option<Arc<anyhow::Error>>,
+}
+
+struct GroupingShared<K, M, O, C> {
+ processor: GroupChunkProcessor<M, O, C>,
+ limit: Option<Arc<FifoLimiter>>,
+ state: Mutex<GroupState<K, M, O, C>>,
+ // Resolves when any group finishes, so a parked drain also wakes for a group started later.
+ group_completed: ResettableSignal,
+}
+
+impl<K, M, O, C> GroupingShared<K, M, O, C>
+where
+ K: Hash + Eq + Clone + Send + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ C: Send + 'static,
+{
+ /// Every key left in `queues` belongs to a running group, so its order no longer matters and `swap_remove` is safe.
+ fn start_group_if_queued(self: &Arc<Self>, state: &mut GroupState<K, M, O, C>, key: &K) {
+ if state.active.contains(key) {
+ return;
+ }
+ let Some(queue) = state.queues.swap_remove(key) else {
+ return;
+ };
+ if queue.is_empty() {
+ return;
+ }
+ self.spawn_group(state, key.clone(), queue);
+ }
+
+ /// One pass in key-arrival order, so a pull costs O(keys) and not O(keys²).
+ fn start_available(self: &Arc<Self>, state: &mut GroupState<K, M, O, C>) {
+ let mut ready = Vec::new();
+ let GroupState { queues, active, .. } = &mut *state;
+ queues.retain(|key, queue| {
+ if active.contains(key) || queue.is_empty() {
+ return true;
+ }
+ ready.push((key.clone(), std::mem::take(queue)));
+ false
+ });
+ for (key, queue) in ready {
+ self.spawn_group(state, key, queue);
+ }
+ }
+
+ fn spawn_group(
+ self: &Arc<Self>,
+ state: &mut GroupState<K, M, O, C>,
+ key: K,
+ queue: Vec<ResultWithContext<M, C>>,
+ ) {
+ // The key is claimed now, so per-key order holds even while the group waits for a slot.
+ state.active.insert(key.clone());
+
+ let shared = self.clone();
+ let permit = self.limit.as_ref().map(|limit| limit.acquire());
+ tokio::spawn(async move {
+ let _permit = match permit {
+ Some(permit) => Some(permit.await),
+ None => None,
+ };
+ // The processor call is guarded, so a panic before it returns its future fails the group.
+ let outcome = AssertUnwindSafe(async { (shared.processor)(queue).await })
+ .catch_unwind()
+ .await;
+ let mut state = shared.state.lock().unwrap();
+ match outcome {
+ Ok(Ok(results)) => {
+ state.completed.push_back(results);
+ state.active.remove(&key);
+ shared.start_group_if_queued(&mut state, &key);
+ }
+ Ok(Err(error)) => {
+ // Poison. The key stays active, so later items for it never start.
+ state.failure.get_or_insert_with(|| Arc::new(error));
+ }
+ Err(payload) => {
+ let message = crate::errors::panic_message(payload.as_ref());
+ state.failure.get_or_insert_with(|| {
+ Arc::new(anyhow::anyhow!("group processor panicked: {message}"))
+ });
+ }
+ }
+ drop(state);
+ shared.group_completed.resolve();
+ });
+ }
+}
+
+struct GroupingCallbacks<I, K, M, O, CI, CO> {
+ grouping: GroupingFunction<M, K>,
+ previous: DynChunkPipeline<I, M, CI, CO>,
+ shared: Arc<GroupingShared<K, M, O, CO>>,
+}
+
+#[async_trait]
+impl<I, K, M, O, CI, CO> InterleavingCallbacks<I, O, CI, CO>
+ for GroupingCallbacks<I, K, M, O, CI, CO>
+where
+ I: Send + 'static,
+ K: Hash + Eq + Clone + Send + Sync + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.previous.feed(elements);
+ }
+
+ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<O, CO>> {
+ let Some(previous) = self.previous.next().await? else {
+ return Ok(PullOutcome::Drained);
+ };
+ // The grouping function is user code, so it runs before the lock is taken.
+ let mut routed = Vec::with_capacity(previous.len());
+ let mut non_ok = Vec::new();
+ for item in previous {
+ match &item.result {
+ crate::results::Outcome::Ok(value) => {
+ let key = call_user_fn(
+ "Grouping function",
+ "groupingFn",
+ item.context.debug_context.as_ref(),
+ || (self.grouping)(value),
+ )?;
+ routed.push((key, item));
+ }
+ crate::results::Outcome::NonOk(_) => {
+ let (n, context) = item.into_ok().err().expect("non-OK");
+ non_ok.push(ResultWithContext::<M, CO>::non_ok(n, context));
+ }
+ }
+ }
+ let mut state = self.shared.state.lock().unwrap();
+ for (key, item) in routed {
+ state.queues.entry(key).or_default().push(item);
+ }
+ if !non_ok.is_empty() {
+ state.completed.push_back(non_ok);
+ }
+ self.shared.start_available(&mut state);
+ Ok(PullOutcome::Drain)
+ }
+
+ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ loop {
+ let wait = {
+ let mut state = self.shared.state.lock().unwrap();
+ if let Some(completed) = state.completed.pop_front() {
+ return Ok(Some(completed));
+ }
+ // Surface a failure only after everything that completed has drained.
+ if let Some(failure) = &state.failure {
+ return Err(crate::errors::shared_error(failure.clone()));
+ }
+ if state.active.is_empty() {
+ return Ok(None);
+ }
+ self.shared.group_completed.reset();
+ self.shared.group_completed.wait()
+ };
+ wait.await;
+ }
+ }
+}
+
+pub struct ConcurrentlyGroupingChunkPipeline<I, K, M, O, CI, CO> {
+ inner: Inner<I, K, M, O, CI, CO>,
+}
+
+type Inner<I, K, M, O, CI, CO> =
+ InterleavingChunkPipeline<I, O, CI, CO, GroupingCallbacks<I, K, M, O, CI, CO>>;
+
+impl<I, K, M, O, CI, CO> ConcurrentlyGroupingChunkPipeline<I, K, M, O, CI, CO>
+where
+ I: Send + 'static,
+ K: Hash + Eq + Clone + Send + Sync + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ pub fn new(
+ grouping: GroupingFunction<M, K>,
+ processor: GroupChunkProcessor<M, O, CO>,
+ previous: DynChunkPipeline<I, M, CI, CO>,
+ max_concurrency: Option<usize>,
+ ) -> Self {
+ let shared = Arc::new(GroupingShared {
+ processor,
+ limit: max_concurrency.map(FifoLimiter::new),
+ state: Mutex::new(GroupState {
+ queues: IndexMap::new(),
+ active: HashSet::new(),
+ completed: VecDeque::new(),
+ failure: None,
+ }),
+ group_completed: ResettableSignal::new(),
+ });
+ Self {
+ inner: InterleavingChunkPipeline::new(Arc::new(GroupingCallbacks {
+ grouping,
+ previous,
+ shared,
+ })),
+ }
+ }
+}
+
+#[async_trait]
+impl<I, K, M, O, CI, CO> ChunkPipeline<I, O, CI, CO>
+ for ConcurrentlyGroupingChunkPipeline<I, K, M, O, CI, CO>
+where
+ I: Send + 'static,
+ K: Hash + Eq + Clone + Send + Sync + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.inner.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ self.inner.next().await
+ }
+}
diff --git a/rust/ingestion-framework/src/helpers.rs b/rust/ingestion-framework/src/helpers.rs
new file mode 100644
index 000000000000..5198c2a7290d
--- /dev/null
+++ b/rust/ingestion-framework/src/helpers.rs
@@ -0,0 +1,115 @@
+use std::collections::BTreeMap;
+
+use serde_json::json;
+
+use crate::context::{OkResultWithContext, PipelineContext};
+use crate::outputs::{HasMessage, KafkaMessage};
+
+// A responder needs these headers for an event restriction. distinct_id and session_id are customer-controlled and stay out.
+const LOGGED_HEADERS: [&str; 3] = ["token", "event", "uuid"];
+
+pub fn create_kafka_debug_context(message: &KafkaMessage) -> serde_json::Value {
+ let headers: serde_json::Map<String, serde_json::Value> = message
+ .headers
+ .iter()
+ .filter(|(key, _)| LOGGED_HEADERS.contains(&key.as_str()))
+ .filter_map(|(key, value)| {
+ value
+ .as_ref()
+ .map(|v| (key.clone(), json!(String::from_utf8_lossy(v))))
+ })
+ .collect();
+ json!({
+ "topic": message.topic,
+ "partition": message.partition,
+ "offset": message.offset,
+ "headers": headers,
+ })
+}
+
+pub fn compact_offset_ranges(offsets: &[i64]) -> String {
+ let mut sorted: Vec<i64> = offsets.to_vec();
+ sorted.sort_unstable();
+ sorted.dedup();
+ let mut ranges = Vec::new();
+ let mut iter = sorted.into_iter();
+ let Some(mut start) = iter.next() else {
+ return String::new();
+ };
+ let mut end = start;
+ let fmt = |s: i64, e: i64| {
+ if s == e {
+ s.to_string()
+ } else {
+ format!("{s}-{e}")
+ }
+ };
+ for offset in iter {
+ if offset == end + 1 {
+ end = offset;
+ } else {
+ ranges.push(fmt(start, end));
+ start = offset;
+ end = offset;
+ }
+ }
+ ranges.push(fmt(start, end));
+ ranges.join(",")
+}
+
+pub fn aggregate_kafka_debug_contexts(debug_contexts: &[serde_json::Value]) -> serde_json::Value {
+ let mut by_partition: BTreeMap<String, Vec<i64>> = BTreeMap::new();
+ let mut order: Vec<String> = Vec::new();
+ for context in debug_contexts {
+ let key = format!(
+ "{}[{}]",
+ context["topic"].as_str().unwrap_or("undefined"),
+ context["partition"].as_i64().unwrap_or_default()
+ );
+ if !by_partition.contains_key(&key) {
+ order.push(key.clone());
+ }
+ by_partition
+ .entry(key)
+ .or_default()
+ .push(context["offset"].as_i64().unwrap_or_default());
+ }
+ json!({
+ "count": debug_contexts.len(),
+ "offsets": order
+ .iter()
+ .map(|key| format!("{key}:{}", compact_offset_ranges(&by_partition[key])))
+ .collect::<Vec<_>>(),
+ })
+}
+
+#[derive(Debug, Clone)]
+pub struct MessageContext {
+ pub message: KafkaMessage,
+}
+
+impl HasMessage for MessageContext {
+ fn message(&self) -> &KafkaMessage {
+ &self.message
+ }
+}
+
+/// The value is a copy of the context, as in Node, where it is `{ message }` too.
+pub fn create_batch(
+ messages: Vec<KafkaMessage>,
+) -> Vec<OkResultWithContext<MessageContext, MessageContext>> {
+ messages
+ .into_iter()
+ .map(|message| {
+ let debug = create_kafka_debug_context(&message);
+ let mut context = PipelineContext::new(MessageContext {
+ message: message.clone(),
+ });
+ context.debug_context = Some(debug);
+ OkResultWithContext {
+ value: MessageContext { message },
+ context,
+ }
+ })
+ .collect()
+}
diff --git a/rust/ingestion-framework/src/interleaving.rs b/rust/ingestion-framework/src/interleaving.rs
new file mode 100644
index 000000000000..216cdd351807
--- /dev/null
+++ b/rust/ingestion-framework/src/interleaving.rs
@@ -0,0 +1,137 @@
+//! The pull is a boxed future in a field, not a spawned task, because a spawn per `next()` costs a cross-thread wake.
+
+use std::sync::{Arc, Mutex};
+
+use async_trait::async_trait;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+
+use crate::context::{ChunkResults, OkResultWithContext};
+use crate::errors::shared_error;
+use crate::sync::ResettableSignal;
+
+pub enum PullOutcome<O, C> {
+ Emit(ChunkResults<O, C>),
+ Drain,
+ Drained,
+}
+
+#[async_trait]
+pub trait InterleavingCallbacks<I, O, CI, CO>: Send + Sync + 'static {
+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>);
+ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<O, CO>>;
+ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>>;
+}
+
+type SubPull<O, CO> = BoxFuture<'static, anyhow::Result<Option<ChunkResults<O, CO>>>>;
+
+pub struct InterleavingChunkPipeline<I, O, CI, CO, Cb> {
+ callbacks: Arc<Cb>,
+ new_input: ResettableSignal,
+ sub_pending: Mutex<Option<SubPull<O, CO>>>,
+ failure: Mutex<Option<Arc<anyhow::Error>>>,
+ _marker: std::marker::PhantomData<fn(I, CI)>,
+}
+
+impl<I, O, CI, CO, Cb> InterleavingChunkPipeline<I, O, CI, CO, Cb>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+ Cb: InterleavingCallbacks<I, O, CI, CO>,
+{
+ pub fn new(callbacks: Arc<Cb>) -> Self {
+ Self {
+ callbacks,
+ new_input: ResettableSignal::new(),
+ sub_pending: Mutex::new(None),
+ failure: Mutex::new(None),
+ _marker: std::marker::PhantomData,
+ }
+ }
+
+ pub fn callbacks(&self) -> &Arc<Cb> {
+ &self.callbacks
+ }
+
+ pub fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.callbacks.on_feed(elements);
+ self.new_input.resolve();
+ }
+
+ fn take_or_start_sub_pull(&self) -> SubPull<O, CO> {
+ if let Some(pending) = self.sub_pending.lock().unwrap().take() {
+ return pending;
+ }
+ let callbacks = self.callbacks.clone();
+ async move { callbacks.on_process_pull().await }.boxed()
+ }
+
+ fn latch(&self, error: anyhow::Error) -> Arc<anyhow::Error> {
+ let mut failure = self.failure.lock().unwrap();
+ failure.get_or_insert_with(|| Arc::new(error)).clone()
+ }
+
+ pub async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ if self.failure.lock().unwrap().is_some() {
+ return self.drain_then_reject().await;
+ }
+
+ loop {
+ // Arm a fresh wake-up before the pull, so a stale feed signal does not pull the next chunk early.
+ self.new_input.reset();
+
+ let pulled = match self.callbacks.on_source_pull().await {
+ Ok(pulled) => pulled,
+ Err(error) => {
+ self.latch(error);
+ return self.drain_then_reject().await;
+ }
+ };
+ let source_drained = match pulled {
+ PullOutcome::Emit(chunk) => return Ok(Some(chunk)),
+ PullOutcome::Drain => false,
+ PullOutcome::Drained => true,
+ };
+
+ let mut pending = self.take_or_start_sub_pull();
+ let wake = self.new_input.wait();
+ let sub_result = tokio::select! {
+ biased;
+ result = &mut pending => result,
+ _ = wake => {
+ *self.sub_pending.lock().unwrap() = Some(pending);
+ continue;
+ }
+ };
+
+ match sub_result {
+ Err(error) => {
+ let shared = self.latch(error);
+ return Err(shared_error(shared));
+ }
+ Ok(Some(chunk)) => return Ok(Some(chunk)),
+ Ok(None) => {
+ if source_drained {
+ return Ok(None);
+ }
+ }
+ }
+ }
+ }
+
+ async fn drain_then_reject(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ let pending = self.take_or_start_sub_pull();
+ if let Ok(Some(chunk)) = pending.await {
+ return Ok(Some(chunk));
+ }
+ let failure = self
+ .failure
+ .lock()
+ .unwrap()
+ .clone()
+ .expect("failure is latched");
+ Err(shared_error(failure))
+ }
+}
diff --git a/rust/ingestion-framework/src/js_json.rs b/rust/ingestion-framework/src/js_json.rs
new file mode 100644
index 000000000000..12aa717832dd
--- /dev/null
+++ b/rust/ingestion-framework/src/js_json.rs
@@ -0,0 +1,199 @@
+//! Kafka messages that must equal the Node bytes use this writer, because `serde_json` prints `1.0` and `1e21`, not `1` and `1e+21`.
+
+use std::fmt::Write;
+
+use serde_json::Value;
+
+pub fn write_value(out: &mut String, value: &Value) {
+ match value {
+ Value::Null => out.push_str("null"),
+ Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
+ Value::Number(n) => match n.as_f64() {
+ Some(f) => write_number(out, f),
+ None => out.push_str("null"),
+ },
+ Value::String(s) => write_string(out, s),
+ Value::Array(items) => {
+ out.push('[');
+ for (i, item) in items.iter().enumerate() {
+ if i > 0 {
+ out.push(',');
+ }
+ write_value(out, item);
+ }
+ out.push(']');
+ }
+ Value::Object(map) => write_object(out, map.iter()),
+ }
+}
+
+pub fn write_object<'a>(out: &mut String, entries: impl Iterator<Item = (&'a String, &'a Value)>) {
+ let mut index_keys: Vec<(u32, &String, &Value)> = Vec::new();
+ let mut other_keys: Vec<(&String, &Value)> = Vec::new();
+ for (key, value) in entries {
+ match array_index(key) {
+ Some(index) => index_keys.push((index, key, value)),
+ None => other_keys.push((key, value)),
+ }
+ }
+ index_keys.sort_by_key(|(index, _, _)| *index);
+ out.push('{');
+ let ordered = index_keys
+ .into_iter()
+ .map(|(_, key, value)| (key, value))
+ .chain(other_keys);
+ for (i, (key, value)) in ordered.enumerate() {
+ if i > 0 {
+ out.push(',');
+ }
+ write_string(out, key);
+ out.push(':');
+ write_value(out, value);
+ }
+ out.push('}');
+}
+
+/// A canonical decimal below 2^32 - 1 is an array index, which a JavaScript object lists first.
+fn array_index(key: &str) -> Option<u32> {
+ if key.is_empty() || key.len() > 10 || !key.bytes().all(|b| b.is_ascii_digit()) {
+ return None;
+ }
+ if key.len() > 1 && key.starts_with('0') {
+ return None;
+ }
+ let value: u64 = key.parse().ok()?;
+ (value < u32::MAX as u64).then_some(value as u32)
+}
+
+pub fn write_string(out: &mut String, s: &str) {
+ out.push('"');
+ for c in s.chars() {
+ match c {
+ '"' => out.push_str("\\\""),
+ '\\' => out.push_str("\\\\"),
+ '\u{08}' => out.push_str("\\b"),
+ '\u{0c}' => out.push_str("\\f"),
+ '\n' => out.push_str("\\n"),
+ '\r' => out.push_str("\\r"),
+ '\t' => out.push_str("\\t"),
+ c if (c as u32) < 0x20 => {
+ let _ = write!(out, "\\u{:04x}", c as u32);
+ }
+ c => out.push(c),
+ }
+ }
+ out.push('"');
+}
+
+/// JavaScript numbers are f64, so an integer above 2^53 prints as in Node.
+pub fn write_number(out: &mut String, f: f64) {
+ if !f.is_finite() {
+ out.push_str("null");
+ return;
+ }
+ if f == 0.0 {
+ out.push('0');
+ return;
+ }
+ if f < 0.0 {
+ out.push('-');
+ }
+ let sci = shortest_scientific(f.abs());
+ let (mantissa, exponent) = sci.split_once('e').expect("LowerExp has an exponent");
+ let digits: String = mantissa.chars().filter(|c| *c != '.').collect();
+ let k = digits.len() as i32;
+ let n = exponent.parse::<i32>().expect("integer exponent") + 1;
+ if k <= n && n <= 21 {
+ out.push_str(&digits);
+ out.extend(std::iter::repeat_n('0', (n - k) as usize));
+ } else if 0 < n && n <= 21 {
+ out.push_str(&digits[..n as usize]);
+ out.push('.');
+ out.push_str(&digits[n as usize..]);
+ } else if -6 < n && n <= 0 {
+ out.push_str("0.");
+ out.extend(std::iter::repeat_n('0', (-n) as usize));
+ out.push_str(&digits);
+ } else {
+ out.push_str(&digits[..1]);
+ if k > 1 {
+ out.push('.');
+ out.push_str(&digits[1..]);
+ }
+ let e = n - 1;
+ let _ = write!(out, "e{}{}", if e >= 0 { "+" } else { "-" }, e.abs());
+ }
+}
+
+/// ECMAScript breaks a tie between two shortest candidates toward even, so an odd last digit is checked against the exact rounding.
+fn shortest_scientific(value: f64) -> String {
+ let shortest = format!("{value:e}");
+ let mantissa = shortest.split_once('e').map_or("", |(m, _)| m);
+ let last_is_odd = mantissa.bytes().last().is_some_and(|d| (d - b'0') % 2 == 1);
+ if !last_is_odd {
+ return shortest;
+ }
+ let digits = mantissa.bytes().filter(u8::is_ascii_digit).count();
+ let exact = format!("{:.*e}", digits - 1, value);
+ // At a power of two the lower gap is narrower, so the nearest candidate can miss.
+ if exact != shortest && exact.parse::<f64>() == Ok(value) {
+ exact
+ } else {
+ shortest
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn number(f: f64) -> String {
+ let mut out = String::new();
+ write_number(&mut out, f);
+ out
+ }
+
+ // Expected strings come from `String(x)` in Node 22.
+ #[test]
+ fn numbers_match_javascript() {
+ let cases: &[(f64, &str)] = &[
+ (1.0, "1"),
+ (100.0, "100"),
+ (-1.5, "-1.5"),
+ (0.1, "0.1"),
+ (-0.0, "0"),
+ (1e21, "1e+21"),
+ (1e20, "100000000000000000000"),
+ (1.2345e-7, "1.2345e-7"),
+ (0.000001, "0.000001"),
+ (12345678901234567890.0, "12345678901234567000"),
+ (5e-324, "5e-324"),
+ (1.7976931348623157e308, "1.7976931348623157e+308"),
+ (1e15 + 0.25, "1000000000000000.2"),
+ (f64::from_bits(0x409a_ab1c_d000_0000), "1706.7781372070312"),
+ (-1417389728594571.2, "-1417389728594571.2"),
+ ];
+ for (input, expected) in cases {
+ assert_eq!(number(*input), *expected, "{input}");
+ }
+ }
+
+ #[test]
+ fn index_keys_come_first_like_a_javascript_object() {
+ let mut map = indexmap::IndexMap::new();
+ map.insert("b".to_string(), Value::from(1));
+ map.insert("10".to_string(), Value::from(2));
+ map.insert("2".to_string(), Value::from(3));
+ map.insert("01".to_string(), Value::from(4));
+ let mut out = String::new();
+ write_object(&mut out, map.iter());
+ assert_eq!(out, r#"{"2":3,"10":2,"b":1,"01":4}"#);
+ }
+
+ #[test]
+ fn strings_escape_like_javascript() {
+ let mut out = String::new();
+ write_string(&mut out, "a\"\\/\u{7}\n\u{2028}é");
+ assert_eq!(out, "\"a\\\"\\\\/\\u0007\\n\u{2028}é\"");
+ }
+}
diff --git a/rust/ingestion-framework/src/lib.rs b/rust/ingestion-framework/src/lib.rs
new file mode 100644
index 000000000000..fb3fa649e8f1
--- /dev/null
+++ b/rust/ingestion-framework/src/lib.rs
@@ -0,0 +1,60 @@
+//! Kept in step with `nodejs/src/ingestion/framework`. `PARITY.md` lists every known difference.
+
+pub mod batching;
+pub mod branching;
+pub mod budget;
+pub mod builders;
+pub mod chunk_pipeline;
+pub mod concurrent;
+pub mod context;
+pub mod errors;
+pub mod fan_out_fan_in;
+pub mod filter_map;
+pub mod grouping;
+pub mod helpers;
+pub mod interleaving;
+mod js_json;
+pub mod metrics;
+pub mod outputs;
+pub mod per_element;
+pub mod pipeline;
+pub mod result_handling;
+pub mod results;
+pub mod retry;
+pub mod side_effect_handling;
+pub mod side_effects;
+pub mod steps;
+pub mod sync;
+pub mod timeout_barrier;
+pub mod warning_handling;
+
+pub use batching::{
+ AfterBatchInput, AfterBatchOutput, BatchResult, BatchingPipeline, BatchingPipelineOptions,
+ BeforeBatchInput, BeforeBatchOutput, FeedRejectionKind, FeedResult, WithBatch,
+};
+pub use budget::BatchBudget;
+pub use builders::{
+ before_batch_step, new_batching_pipeline, new_chunk_pipeline, new_pipeline,
+ passthrough_after_batch_step, ChunkPipelineBuilder, PipelineBuilder, StartPipelineBuilder,
+};
+pub use chunk_pipeline::{ChunkPipeline, ChunkPipelineUnwrapper, DynChunkPipeline};
+pub use context::{
+ create_ok_context, ChunkResults, OkResultWithContext, PipelineContext, PipelineWarning,
+ ResultWithContext, WarningDetails,
+};
+pub use errors::{is_retriable, PipelineError};
+pub use outputs::{
+ HasMessage, HasTeamId, Headers, IngestionOutputs, KafkaMessage, MemoryOutputs, OutputMessage,
+ DLQ_OUTPUT, INGESTION_WARNINGS_OUTPUT,
+};
+pub use per_element::per_element;
+pub use pipeline::{DynPipeline, Pipeline};
+pub use result_handling::PipelineConfig;
+pub use results::{
+ dlq, drop, ok, redirect, redirect_with, timeout, NonOk, Outcome, PipelineResult,
+ PipelineResultType,
+};
+pub use retry::RetryOptions;
+pub use side_effect_handling::SideEffectHandlingConfig;
+pub use side_effects::{PromiseScheduler, SideEffect, SideEffectScheduler};
+pub use steps::{chunk_step, step, ChunkProcessingStep, ProcessingStep};
diff --git a/rust/ingestion-framework/src/metrics.rs b/rust/ingestion-framework/src/metrics.rs
new file mode 100644
index 000000000000..22650470702e
--- /dev/null
+++ b/rust/ingestion-framework/src/metrics.rs
@@ -0,0 +1,65 @@
+//! Metric names and labels match the Node side, so one dashboard reads both.
+
+use std::sync::Arc;
+
+pub const SIDE_EFFECT_RESULT_COUNTER: &str = "pipelines_side_effects_total";
+pub const PIPELINE_STEP_DURATION_HISTOGRAM: &str = "ingestion_pipeline_step_duration_seconds";
+pub const PIPELINE_RETRY_ATTEMPTS_HISTOGRAM: &str = "ingestion_pipeline_retry_attempts";
+pub const BATCH_BUDGET_EXHAUSTED_COUNTER: &str = "ingestion_batch_budget_exhausted_total";
+pub const BATCH_BUDGET_OVERRUN_HISTOGRAM: &str = "ingestion_batch_budget_overrun_seconds";
+pub const BATCH_BUDGET_CHECKPOINT_COUNTER: &str = "ingestion_batch_budget_checkpoint_total";
+pub const INGESTION_PIPELINE_RESULTS_COUNTER: &str = "ingestion_pipeline_results";
+pub const INGESTION_WARNING_COUNTER: &str = "ingestion_warnings_total";
+
+#[derive(Debug, Clone, Copy)]
+pub enum BudgetCheckpoint {
+ Step,
+ Chunk,
+}
+
+impl BudgetCheckpoint {
+ fn label(self) -> &'static str {
+ match self {
+ BudgetCheckpoint::Step => "step",
+ BudgetCheckpoint::Chunk => "chunk",
+ }
+ }
+}
+
+pub fn record_budget_checkpoint(checkpoint: BudgetCheckpoint, step_name: Arc<str>) {
+ metrics::counter!(
+ BATCH_BUDGET_CHECKPOINT_COUNTER,
+ "checkpoint" => checkpoint.label(),
+ "step_name" => step_name
+ )
+ .increment(1);
+}
+
+pub fn record_step_duration(
+ step_name: Arc<str>,
+ step_type: &'static str,
+ result: &'static str,
+ seconds: f64,
+) {
+ metrics::histogram!(
+ PIPELINE_STEP_DURATION_HISTOGRAM,
+ "step_name" => step_name,
+ "step_type" => step_type,
+ "result" => result
+ )
+ .record(seconds);
+}
+
+pub fn record_side_effect_result(ok: bool) {
+ metrics::counter!(SIDE_EFFECT_RESULT_COUNTER, "result" => if ok { "ok" } else { "error" })
+ .increment(1);
+}
+
+pub fn record_retry_attempts(name: Arc<str>, outcome: &'static str, attempts: u32) {
+ metrics::histogram!(
+ PIPELINE_RETRY_ATTEMPTS_HISTOGRAM,
+ "name" => name,
+ "outcome" => outcome
+ )
+ .record(attempts as f64);
+}
diff --git a/rust/ingestion-framework/src/outputs.rs b/rust/ingestion-framework/src/outputs.rs
new file mode 100644
index 000000000000..bd55de251692
--- /dev/null
+++ b/rust/ingestion-framework/src/outputs.rs
@@ -0,0 +1,179 @@
+use std::sync::{Arc, Mutex};
+
+use async_trait::async_trait;
+pub use bytes::Bytes;
+
+pub const DLQ_OUTPUT: &str = "dlq";
+pub const INGESTION_WARNINGS_OUTPUT: &str = "ingestion_warnings";
+
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct KafkaMessage {
+ pub topic: String,
+ pub partition: i32,
+ pub offset: i64,
+ pub key: Option<Vec<u8>>,
+ pub value: Option<Bytes>,
+ /// In wire order. A header can repeat, as in Kafka.
+ pub headers: Vec<(String, Option<Vec<u8>>)>,
+ pub timestamp: Option<i64>,
+}
+
+impl KafkaMessage {
+ /// The last value of a repeated key wins, as in `parseKafkaHeaders`.
+ pub fn header(&self, name: &str) -> Option<String> {
+ self.headers
+ .iter()
+ .rev()
+ .find(|(key, _)| key == name)
+ .and_then(|(_, value)| value.as_ref())
+ .map(|value| String::from_utf8_lossy(value).into_owned())
+ }
+}
+
+/// Headers in insertion order. `set` replaces a key in place, as a JS object does.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct Headers(pub Vec<(String, String)>);
+
+impl Headers {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn set(&mut self, key: impl Into<String>, value: impl Into<String>) {
+ let key = key.into();
+ let value = value.into();
+ match self.0.iter_mut().find(|(k, _)| *k == key) {
+ Some(entry) => entry.1 = value,
+ None => self.0.push((key, value)),
+ }
+ }
+
+ pub fn get(&self, key: &str) -> Option<&str> {
+ self.0
+ .iter()
+ .find(|(k, _)| k == key)
+ .map(|(_, v)| v.as_str())
+ }
+
+ pub fn remove(&mut self, key: &str) {
+ self.0.retain(|(k, _)| k != key);
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = &(String, String)> {
+ self.0.iter()
+ }
+
+ pub fn len(&self) -> usize {
+ self.0.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.0.is_empty()
+ }
+}
+
+impl<K: Into<String>, V: Into<String>> FromIterator<(K, V)> for Headers {
+ fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> Self {
+ let mut headers = Headers::new();
+ for (k, v) in iter {
+ headers.set(k, v);
+ }
+ headers
+ }
+}
+
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct OutputMessage {
+ pub value: Option<Bytes>,
+ pub key: Option<Vec<u8>>,
+ pub headers: Headers,
+}
+
+#[async_trait]
+pub trait IngestionOutputs: Send + Sync {
+ async fn produce(&self, output: &str, message: OutputMessage) -> anyhow::Result<()>;
+ /// Resolves on the ack, as Node's `queueMessages` does.
+ async fn queue_messages(
+ &self,
+ output: &str,
+ messages: Vec<OutputMessage>,
+ ) -> anyhow::Result<()>;
+}
+
+#[derive(Default, Clone)]
+pub struct MemoryOutputs {
+ inner: Arc<Mutex<MemoryOutputsState>>,
+}
+
+#[derive(Default)]
+struct MemoryOutputsState {
+ produced: Vec<(String, OutputMessage)>,
+ fail_outputs: Vec<String>,
+}
+
+impl MemoryOutputs {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn produced(&self) -> Vec<(String, OutputMessage)> {
+ self.inner.lock().unwrap().produced.clone()
+ }
+
+ pub fn produced_to(&self, output: &str) -> Vec<OutputMessage> {
+ self.inner
+ .lock()
+ .unwrap()
+ .produced
+ .iter()
+ .filter(|(o, _)| o == output)
+ .map(|(_, m)| m.clone())
+ .collect()
+ }
+
+ pub fn fail_output(&self, output: &str) {
+ self.inner
+ .lock()
+ .unwrap()
+ .fail_outputs
+ .push(output.to_string());
+ }
+
+ pub fn clear_failures(&self) {
+ self.inner.lock().unwrap().fail_outputs.clear();
+ }
+
+ fn record(&self, output: &str, messages: Vec<OutputMessage>) -> anyhow::Result<()> {
+ let mut state = self.inner.lock().unwrap();
+ if state.fail_outputs.iter().any(|o| o == output) {
+ anyhow::bail!("produce to {output} failed");
+ }
+ state
+ .produced
+ .extend(messages.into_iter().map(|m| (output.to_string(), m)));
+ Ok(())
+ }
+}
+
+#[async_trait]
+impl IngestionOutputs for MemoryOutputs {
+ async fn produce(&self, output: &str, message: OutputMessage) -> anyhow::Result<()> {
+ self.record(output, vec![message])
+ }
+
+ async fn queue_messages(
+ &self,
+ output: &str,
+ messages: Vec<OutputMessage>,
+ ) -> anyhow::Result<()> {
+ self.record(output, messages)
+ }
+}
+
+pub trait HasMessage {
+ fn message(&self) -> &KafkaMessage;
+}
+
+pub trait HasTeamId {
+ fn team_id(&self) -> i64;
+}
diff --git a/rust/ingestion-framework/src/per_element.rs b/rust/ingestion-framework/src/per_element.rs
new file mode 100644
index 000000000000..ca3488d65e16
--- /dev/null
+++ b/rust/ingestion-framework/src/per_element.rs
@@ -0,0 +1,75 @@
+use std::future::Future;
+use std::sync::{Arc, Mutex};
+
+use futures::stream::{FuturesOrdered, StreamExt};
+
+use crate::steps::{ChunkProcessingStep, ProcessingStep};
+use crate::sync::FifoLimiter;
+
+/// Sized up front, because above `JOIN_ALL_SMALL` `join_all` grows its output one push at a time.
+async fn join_all_presized<F: Future>(runs: Vec<F>) -> Vec<F::Output> {
+ if runs.len() <= JOIN_ALL_SMALL {
+ return futures::future::join_all(runs).await;
+ }
+ let mut results = Vec::with_capacity(runs.len());
+ let mut ordered: FuturesOrdered<F> = runs.into_iter().collect();
+ while let Some(result) = ordered.next().await {
+ results.push(result);
+ }
+ results
+}
+
+/// Mirrors `futures_util::future::join_all::SMALL`.
+const JOIN_ALL_SMALL: usize = 30;
+
+/// Steps run inside one future, not as tasks, as Node runs them on one event loop. The first error fails the chunk after the rest end.
+/// Panics on `Some(0)`, as Node's `pLimit(0)` throws.
+pub fn per_element<T, U>(
+ step: ProcessingStep<T, U>,
+ max_concurrency: Option<usize>,
+) -> ChunkProcessingStep<T, U>
+where
+ T: Send + 'static,
+ U: Send + 'static,
+{
+ let limit = max_concurrency.map(FifoLimiter::new);
+ let name = step.name().clone();
+ ChunkProcessingStep::new(&name, move |values: Vec<T>| {
+ let step = step.clone();
+ let limit = limit.clone();
+ let first_error: Arc<Mutex<Option<anyhow::Error>>> = Arc::default();
+ // Every slot is taken here, in input order, before any step starts.
+ let runs: Vec<_> = values
+ .into_iter()
+ .map(|value| {
+ let permit = limit.as_ref().map(|limit| limit.acquire());
+ let step = step.clone();
+ let first_error = first_error.clone();
+ async move {
+ let _permit = match permit {
+ Some(permit) => Some(permit.await),
+ None => None,
+ };
+ let result = step.call(value).await;
+ match result {
+ Ok(result) => Some(result),
+ Err(error) => {
+ first_error.lock().unwrap().get_or_insert(error);
+ None
+ }
+ }
+ }
+ })
+ .collect();
+ async move {
+ let results = join_all_presized(runs).await;
+ if let Some(error) = first_error.lock().unwrap().take() {
+ return Err(error);
+ }
+ Ok(results
+ .into_iter()
+ .map(|result| result.expect("no error was recorded"))
+ .collect())
+ }
+ })
+}
diff --git a/rust/ingestion-framework/src/pipeline.rs b/rust/ingestion-framework/src/pipeline.rs
new file mode 100644
index 000000000000..9b8f89491a2c
--- /dev/null
+++ b/rust/ingestion-framework/src/pipeline.rs
@@ -0,0 +1,111 @@
+use std::sync::Arc;
+use std::time::Instant;
+
+use async_trait::async_trait;
+
+use crate::context::{OkResultWithContext, ResultWithContext};
+use crate::metrics::{record_budget_checkpoint, record_step_duration, BudgetCheckpoint};
+use crate::results::{NonOk, Outcome};
+use crate::steps::ProcessingStep;
+
+/// An `Err` is the Node `throw`: it crashes the batch.
+#[async_trait]
+pub trait Pipeline<I, O, C>: Send + Sync {
+ async fn process(
+ &self,
+ input: OkResultWithContext<I, C>,
+ ) -> anyhow::Result<ResultWithContext<O, C>>;
+}
+
+pub type DynPipeline<I, O, C> = Arc<dyn Pipeline<I, O, C>>;
+
+pub struct StartPipeline;
+
+#[async_trait]
+impl<T: Send + 'static, C: Send + 'static> Pipeline<T, T, C> for StartPipeline {
+ async fn process(
+ &self,
+ input: OkResultWithContext<T, C>,
+ ) -> anyhow::Result<ResultWithContext<T, C>> {
+ Ok(input.into_result())
+ }
+}
+
+pub struct StepPipeline<I, M, O, C> {
+ step: ProcessingStep<M, O>,
+ previous: DynPipeline<I, M, C>,
+}
+
+impl<I, M, O, C> StepPipeline<I, M, O, C> {
+ pub fn new(step: ProcessingStep<M, O>, previous: DynPipeline<I, M, C>) -> Self {
+ Self { step, previous }
+ }
+}
+
+#[async_trait]
+impl<I, M, O, C> Pipeline<I, O, C> for StepPipeline<I, M, O, C>
+where
+ I: Send + 'static,
+ M: Send + 'static,
+ O: Send + 'static,
+ C: Send + 'static,
+{
+ async fn process(
+ &self,
+ input: OkResultWithContext<I, C>,
+ ) -> anyhow::Result<ResultWithContext<O, C>> {
+ let previous = self.previous.process(input).await?;
+ let (value, mut context) = match previous.into_ok() {
+ Ok(ok) => (ok.value, ok.context),
+ Err((non_ok, context)) => {
+ return Ok(ResultWithContext::<M, C>::non_ok(non_ok, context))
+ }
+ };
+
+ let step_name = self.step.name();
+ // lastStep keeps the step that spent the time; the reason names the step that never started.
+ if context.budget.as_ref().is_some_and(|b| b.exhausted()) {
+ record_budget_checkpoint(BudgetCheckpoint::Step, step_name.clone());
+ return Ok(ResultWithContext {
+ result: Outcome::NonOk(NonOk::Timeout {
+ reason: format!("budget exceeded before {step_name}"),
+ }),
+ context,
+ });
+ }
+
+ let started = Instant::now();
+ let result = match self.step.call(value).await {
+ Ok(result) => {
+ record_step_duration(
+ step_name.clone(),
+ "element",
+ result.result_type().as_label(),
+ started.elapsed().as_secs_f64(),
+ );
+ result
+ }
+ Err(error) => {
+ record_step_duration(
+ step_name.clone(),
+ "element",
+ "exception",
+ started.elapsed().as_secs_f64(),
+ );
+ tracing::error!(
+ step = %step_name,
+ error = %error,
+ debug_context = ?context.debug_context,
+ "Step threw"
+ );
+ return Err(error);
+ }
+ };
+
+ let outcome = context.absorb_step(step_name, result);
+ Ok(ResultWithContext {
+ result: outcome,
+ context,
+ })
+ }
+}
diff --git a/rust/ingestion-framework/src/result_handling.rs b/rust/ingestion-framework/src/result_handling.rs
new file mode 100644
index 000000000000..1501fed4de05
--- /dev/null
+++ b/rust/ingestion-framework/src/result_handling.rs
@@ -0,0 +1,270 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use futures::FutureExt;
+
+use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};
+use crate::context::{ChunkResults, OkResultWithContext};
+use crate::errors::SharedError;
+use crate::metrics::INGESTION_PIPELINE_RESULTS_COUNTER;
+use crate::outputs::{
+ HasMessage, Headers, IngestionOutputs, KafkaMessage, OutputMessage, DLQ_OUTPUT,
+};
+use crate::results::{NonOk, Outcome};
+use crate::side_effects::{SideEffect, SideEffectScheduler};
+
+#[derive(Clone)]
+pub struct PipelineConfig {
+ pub outputs: Arc<dyn IngestionOutputs>,
+ pub scheduler: Arc<dyn SideEffectScheduler>,
+ /// Set it only when the consumer stores no offsets after a failed side effect.
+ pub reject_on_dlq_failure: bool,
+}
+
+fn now_iso() -> String {
+ chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
+}
+
+/// Node stringifies a non-buffer, non-string value, so null is `"null"`.
+pub fn copy_and_extend_headers(message: &KafkaMessage, additional: Headers) -> Headers {
+ let mut headers = Headers::new();
+ for (key, value) in &message.headers {
+ let value = match value {
+ Some(bytes) => String::from_utf8_lossy(bytes).into_owned(),
+ None => "null".to_string(),
+ };
+ headers.set(key.clone(), value);
+ }
+ for (key, value) in additional.0 {
+ headers.set(key, value);
+ }
+ headers
+}
+
+pub fn dlq_output_message(message: &KafkaMessage, error: &str, step: &str) -> OutputMessage {
+ let headers = copy_and_extend_headers(
+ message,
+ [
+ ("dlq_reason", error.to_string()),
+ ("dlq_step", step.to_string()),
+ ("dlq_timestamp", now_iso()),
+ ("dlq_topic", message.topic.clone()),
+ ("dlq_partition", message.partition.to_string()),
+ ("dlq_offset", message.offset.to_string()),
+ ]
+ .into_iter()
+ .collect(),
+ );
+ OutputMessage {
+ value: message.value.clone(),
+ key: message.key.clone(),
+ headers,
+ }
+}
+
+async fn produce_dlq_message(
+ outputs: &dyn IngestionOutputs,
+ out: OutputMessage,
+ team_id: Option<String>,
+ step: &str,
+ rethrow_on_failure: bool,
+) -> anyhow::Result<()> {
+ if let Err(dlq_error) = outputs.produce(DLQ_OUTPUT, out).await {
+ tracing::error!(step, team_id = ?team_id, error = %dlq_error, "Failed to send event to DLQ");
+ if rethrow_on_failure {
+ return Err(dlq_error);
+ }
+ }
+ Ok(())
+}
+
+pub async fn produce_message_to_dlq(
+ outputs: &dyn IngestionOutputs,
+ message: &KafkaMessage,
+ error: &str,
+ step: &str,
+ rethrow_on_failure: bool,
+) -> anyhow::Result<()> {
+ let team_id = message.header("team_id");
+ tracing::warn!(step, team_id = ?team_id, error, "Event sent to DLQ");
+ let out = dlq_output_message(message, error, step);
+ produce_dlq_message(outputs, out, team_id, step, rethrow_on_failure).await
+}
+
+pub fn redirect_output_message(
+ message: &KafkaMessage,
+ step: &str,
+ preserve_key: bool,
+) -> OutputMessage {
+ let mut extra = Headers::new();
+ extra.set("redirect-step", step);
+ extra.set("redirect-timestamp", now_iso());
+ // Dropping the key spreads the stream across partitions, so keep the original key recoverable.
+ if !preserve_key {
+ if let Some(key) = &message.key {
+ extra.set(
+ "redirect-original-key",
+ String::from_utf8_lossy(key).into_owned(),
+ );
+ }
+ }
+ OutputMessage {
+ value: message.value.clone(),
+ key: if preserve_key {
+ message.key.clone()
+ } else {
+ None
+ },
+ headers: copy_and_extend_headers(message, extra),
+ }
+}
+
+async fn produce_redirect(
+ outputs: Arc<dyn IngestionOutputs>,
+ output: String,
+ scheduler: &dyn SideEffectScheduler,
+ out: OutputMessage,
+ await_ack: bool,
+) -> anyhow::Result<()> {
+ // Node calls `outputs.produce` before it schedules the promise, so the produce starts here.
+ let produce = async move { outputs.produce(&output, out).await }
+ .map(|result| result.map_err(|error| SharedError(Arc::new(error))))
+ .boxed()
+ .shared();
+ let scheduled = produce.clone();
+ scheduler.schedule(SideEffect::spawn(async move {
+ scheduled.await.map_err(anyhow::Error::new)
+ }));
+ if await_ack {
+ produce.await.map_err(anyhow::Error::new)?;
+ }
+ Ok(())
+}
+
+pub async fn redirect_message_to_output(
+ outputs: Arc<dyn IngestionOutputs>,
+ output: &str,
+ scheduler: &dyn SideEffectScheduler,
+ message: &KafkaMessage,
+ step: &str,
+ preserve_key: bool,
+ await_ack: bool,
+) -> anyhow::Result<()> {
+ let out = redirect_output_message(message, step, preserve_key);
+ produce_redirect(outputs, output.to_string(), scheduler, out, await_ack).await
+}
+
+fn result_details(result: &NonOk) -> (&'static str, String) {
+ match result {
+ NonOk::Drop { reason } => ("drop", reason.clone()),
+ NonOk::Dlq { reason, .. } => ("dlq", reason.clone()),
+ NonOk::Redirect {
+ output,
+ preserve_key,
+ ..
+ } => ("redirect", format!("{output}(preserve_key={preserve_key})")),
+ NonOk::Timeout { reason } => ("timeout", reason.clone()),
+ }
+}
+
+/// A timeout produces nothing, because the redelivery produces it.
+pub struct ResultHandlingPipeline<I, O, CI, CO> {
+ pipeline: DynChunkPipeline<I, O, CI, CO>,
+ config: PipelineConfig,
+}
+
+impl<I, O, CI, CO> ResultHandlingPipeline<I, O, CI, CO> {
+ pub fn new(pipeline: DynChunkPipeline<I, O, CI, CO>, config: PipelineConfig) -> Self {
+ Self { pipeline, config }
+ }
+}
+
+#[async_trait]
+impl<I, O, CI, CO> ChunkPipeline<I, O, CI, CO> for ResultHandlingPipeline<I, O, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: HasMessage + Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.pipeline.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ let Some(mut results) = self.pipeline.next().await? else {
+ return Ok(None);
+ };
+ for item in results.iter_mut() {
+ let step: Arc<str> = item
+ .context
+ .last_step
+ .clone()
+ .unwrap_or_else(|| Arc::from("unknown"));
+ let non_ok = match &item.result {
+ Outcome::Ok(_) => {
+ metrics::counter!(
+ INGESTION_PIPELINE_RESULTS_COUNTER,
+ "result" => "ok",
+ "last_step_name" => step,
+ "details" => ""
+ )
+ .increment(1);
+ continue;
+ }
+ Outcome::NonOk(non_ok) => non_ok,
+ };
+ let (result_label, details) = result_details(non_ok);
+ metrics::counter!(
+ INGESTION_PIPELINE_RESULTS_COUNTER,
+ "result" => result_label,
+ "last_step_name" => step.clone(),
+ "details" => details
+ )
+ .increment(1);
+
+ // Produces start on creation, so they are issued in item order, as in Node's loop.
+ let side_effect = match non_ok {
+ NonOk::Dlq { reason, error } => {
+ let error_text = match error {
+ Some(error) => error.to_string(),
+ None => reason.clone(),
+ };
+ let message = item.context.data.message();
+ let team_id = message.header("team_id");
+ tracing::warn!(step = %step, team_id = ?team_id, error = %error_text, "Event sent to DLQ");
+ let out = dlq_output_message(message, &error_text, &step);
+ let outputs = self.config.outputs.clone();
+ let rethrow = self.config.reject_on_dlq_failure;
+ SideEffect::spawn(async move {
+ produce_dlq_message(outputs.as_ref(), out, team_id, &step, rethrow).await
+ })
+ }
+ NonOk::Drop { reason } => {
+ let message = item.context.data.message();
+ tracing::debug!(step = %step, reason, team_id = ?message.header("team_id"), "Event dropped");
+ continue;
+ }
+ NonOk::Redirect {
+ output,
+ preserve_key,
+ await_ack,
+ ..
+ } => {
+ let out =
+ redirect_output_message(item.context.data.message(), &step, *preserve_key);
+ let output = output.clone();
+ let await_ack = *await_ack;
+ let outputs = self.config.outputs.clone();
+ let scheduler = self.config.scheduler.clone();
+ SideEffect::spawn(async move {
+ produce_redirect(outputs, output, scheduler.as_ref(), out, await_ack).await
+ })
+ }
+ NonOk::Timeout { .. } => continue,
+ };
+ item.context.side_effects.push(side_effect);
+ }
+ Ok(Some(results))
+ }
+}
diff --git a/rust/ingestion-framework/src/results.rs b/rust/ingestion-framework/src/results.rs
new file mode 100644
index 000000000000..06e2ae3ae22b
--- /dev/null
+++ b/rust/ingestion-framework/src/results.rs
@@ -0,0 +1,222 @@
+use std::fmt;
+use std::sync::Arc;
+
+use crate::context::PipelineWarning;
+use crate::side_effects::SideEffect;
+
+/// Mirrors the Node `PipelineResultType` enum, including its numeric values.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum PipelineResultType {
+ Ok = 0,
+ Dlq = 1,
+ Drop = 2,
+ Redirect = 3,
+ Timeout = 4,
+}
+
+impl PipelineResultType {
+ /// The lowercase label the Node metrics use (`PipelineResultType[type].toLowerCase()`).
+ pub fn as_label(&self) -> &'static str {
+ match self {
+ PipelineResultType::Ok => "ok",
+ PipelineResultType::Dlq => "dlq",
+ PipelineResultType::Drop => "drop",
+ PipelineResultType::Redirect => "redirect",
+ PipelineResultType::Timeout => "timeout",
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub enum NonOk {
+ Dlq {
+ reason: String,
+ error: Option<Arc<anyhow::Error>>,
+ },
+ Drop {
+ reason: String,
+ },
+ Redirect {
+ reason: String,
+ output: String,
+ preserve_key: bool,
+ await_ack: bool,
+ },
+ /// The budget cut the element off. Its message is not acked, so its source redelivers it.
+ Timeout {
+ reason: String,
+ },
+}
+
+impl NonOk {
+ pub fn reason(&self) -> &str {
+ match self {
+ NonOk::Dlq { reason, .. }
+ | NonOk::Drop { reason }
+ | NonOk::Redirect { reason, .. }
+ | NonOk::Timeout { reason } => reason,
+ }
+ }
+
+ pub fn result_type(&self) -> PipelineResultType {
+ match self {
+ NonOk::Dlq { .. } => PipelineResultType::Dlq,
+ NonOk::Drop { .. } => PipelineResultType::Drop,
+ NonOk::Redirect { .. } => PipelineResultType::Redirect,
+ NonOk::Timeout { .. } => PipelineResultType::Timeout,
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub enum Outcome<T> {
+ Ok(T),
+ NonOk(NonOk),
+}
+
+impl<T> Outcome<T> {
+ pub fn is_ok(&self) -> bool {
+ matches!(self, Outcome::Ok(_))
+ }
+
+ pub fn result_type(&self) -> PipelineResultType {
+ match self {
+ Outcome::Ok(_) => PipelineResultType::Ok,
+ Outcome::NonOk(n) => n.result_type(),
+ }
+ }
+
+ pub fn value(&self) -> Option<&T> {
+ match self {
+ Outcome::Ok(v) => Some(v),
+ Outcome::NonOk(_) => None,
+ }
+ }
+
+ pub fn into_value(self) -> Option<T> {
+ match self {
+ Outcome::Ok(v) => Some(v),
+ Outcome::NonOk(_) => None,
+ }
+ }
+
+ pub fn non_ok(&self) -> Option<&NonOk> {
+ match self {
+ Outcome::Ok(_) => None,
+ Outcome::NonOk(n) => Some(n),
+ }
+ }
+
+ pub fn reason(&self) -> Option<&str> {
+ self.non_ok().map(|n| n.reason())
+ }
+
+ pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Outcome<U> {
+ match self {
+ Outcome::Ok(v) => Outcome::Ok(f(v)),
+ Outcome::NonOk(n) => Outcome::NonOk(n),
+ }
+ }
+}
+
+/// A `SideEffect` is a task handle that cannot be copied, so the pipeline moves it where Node keeps a copy.
+pub struct PipelineResult<T> {
+ pub outcome: Outcome<T>,
+ pub side_effects: Vec<SideEffect>,
+ pub warnings: Vec<PipelineWarning>,
+}
+
+impl<T: fmt::Debug> fmt::Debug for PipelineResult<T> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("PipelineResult")
+ .field("outcome", &self.outcome)
+ .field("side_effects", &self.side_effects.len())
+ .field("warnings", &self.warnings)
+ .finish()
+ }
+}
+
+impl<T> PipelineResult<T> {
+ pub fn new(outcome: Outcome<T>) -> Self {
+ Self {
+ outcome,
+ side_effects: Vec::new(),
+ warnings: Vec::new(),
+ }
+ }
+
+ pub fn with_side_effects(mut self, side_effects: Vec<SideEffect>) -> Self {
+ self.side_effects.extend(side_effects);
+ self
+ }
+
+ pub fn with_side_effect(mut self, side_effect: SideEffect) -> Self {
+ self.side_effects.push(side_effect);
+ self
+ }
+
+ pub fn with_warnings(mut self, warnings: Vec<PipelineWarning>) -> Self {
+ self.warnings.extend(warnings);
+ self
+ }
+
+ pub fn with_warning(mut self, warning: PipelineWarning) -> Self {
+ self.warnings.push(warning);
+ self
+ }
+
+ pub fn is_ok(&self) -> bool {
+ self.outcome.is_ok()
+ }
+
+ pub fn result_type(&self) -> PipelineResultType {
+ self.outcome.result_type()
+ }
+
+ pub fn value(&self) -> Option<&T> {
+ self.outcome.value()
+ }
+}
+
+pub fn ok<T>(value: T) -> PipelineResult<T> {
+ PipelineResult::new(Outcome::Ok(value))
+}
+
+/// `dlq(reason, error?)`. Node defaults a missing error to `new Error(reason)` at DLQ time.
+pub fn dlq<T>(reason: impl Into<String>, error: Option<anyhow::Error>) -> PipelineResult<T> {
+ PipelineResult::new(Outcome::NonOk(NonOk::Dlq {
+ reason: reason.into(),
+ error: error.map(Arc::new),
+ }))
+}
+
+pub fn drop<T>(reason: impl Into<String>) -> PipelineResult<T> {
+ PipelineResult::new(Outcome::NonOk(NonOk::Drop {
+ reason: reason.into(),
+ }))
+}
+
+pub fn timeout<T>(reason: impl Into<String>) -> PipelineResult<T> {
+ PipelineResult::new(Outcome::NonOk(NonOk::Timeout {
+ reason: reason.into(),
+ }))
+}
+
+/// `redirect(reason, output)` with the Node defaults `preserveKey = true`, `awaitAck = true`.
+pub fn redirect<T>(reason: impl Into<String>, output: impl Into<String>) -> PipelineResult<T> {
+ redirect_with(reason, output, true, true)
+}
+
+pub fn redirect_with<T>(
+ reason: impl Into<String>,
+ output: impl Into<String>,
+ preserve_key: bool,
+ await_ack: bool,
+) -> PipelineResult<T> {
+ PipelineResult::new(Outcome::NonOk(NonOk::Redirect {
+ reason: reason.into(),
+ output: output.into(),
+ preserve_key,
+ await_ack,
+ }))
+}
diff --git a/rust/ingestion-framework/src/retry.rs b/rust/ingestion-framework/src/retry.rs
new file mode 100644
index 000000000000..cc478f28f8a3
--- /dev/null
+++ b/rust/ingestion-framework/src/retry.rs
@@ -0,0 +1,218 @@
+use std::sync::Arc;
+use std::time::Duration;
+
+use rand::Rng;
+use tokio::time::Instant;
+
+use crate::errors::is_retriable;
+use crate::metrics::record_retry_attempts;
+use crate::results::{dlq, PipelineResult};
+use crate::steps::{ChunkProcessingStep, ProcessingStep};
+
+/// Defaults match `defaultRetryConfig`.
+#[derive(Debug, Clone, Default)]
+pub struct RetryOptions {
+ pub tries: Option<u32>,
+ pub sleep_ms: Option<u64>,
+ pub backoff_factor: Option<f64>,
+ pub max_sleep_ms: Option<u64>,
+ pub jitter: Option<f64>,
+ pub soft_deadline_ms: Option<u64>,
+ pub name: Option<String>,
+}
+
+pub const DEFAULT_TRIES: u32 = 3;
+pub const DEFAULT_SLEEP_MS: u64 = 100;
+pub const DEFAULT_BACKOFF_FACTOR: f64 = 2.0;
+pub const DEFAULT_MAX_SLEEP_MS: u64 = 10_000;
+pub const DEFAULT_JITTER: f64 = 0.05;
+
+pub async fn retry_if_retriable<T, F, Fut>(
+ mut f: F,
+ options: &RetryOptions,
+) -> (anyhow::Result<T>, u32)
+where
+ F: FnMut() -> Fut,
+ Fut: std::future::Future<Output = anyhow::Result<T>>,
+{
+ retry_with_input((), |()| f(), options).await
+}
+
+/// Every attempt but the last gets a clone, because a retry needs the input again.
+pub async fn retry_with_input<V, T, F, Fut>(
+ input: V,
+ mut f: F,
+ options: &RetryOptions,
+) -> (anyhow::Result<T>, u32)
+where
+ V: Clone,
+ F: FnMut(V) -> Fut,
+ Fut: std::future::Future<Output = anyhow::Result<T>>,
+{
+ let tries = options.tries.unwrap_or(DEFAULT_TRIES);
+ if tries == 0 {
+ // Node's loop runs no attempt and reaches this throw.
+ return (Err(anyhow::anyhow!("Unreachable error in retry")), 0);
+ }
+ let backoff = options.backoff_factor.unwrap_or(DEFAULT_BACKOFF_FACTOR);
+ let max_sleep = options.max_sleep_ms.unwrap_or(DEFAULT_MAX_SLEEP_MS) as f64;
+ let jitter = options.jitter.unwrap_or(DEFAULT_JITTER);
+ let soft_deadline = options.soft_deadline_ms.map(Duration::from_millis);
+ let started = Instant::now();
+ let past_deadline = || soft_deadline.is_some_and(|d| started.elapsed() >= d);
+
+ let mut sleep_ms = options.sleep_ms.unwrap_or(DEFAULT_SLEEP_MS) as f64;
+ let mut input = Some(input);
+ let mut attempts = 0;
+ loop {
+ attempts += 1;
+ let value = if attempts >= tries {
+ input.take().expect("the last attempt takes the input once")
+ } else {
+ input.clone().expect("input is kept until the last attempt")
+ };
+ match f(value).await {
+ Ok(value) => return (Ok(value), attempts),
+ Err(error) => {
+ if is_retriable(&error) == Some(false) || attempts >= tries || past_deadline() {
+ return (Err(error), attempts);
+ }
+ let jittered = if jitter > 0.0 {
+ sleep_ms * (1.0 - jitter + rand::thread_rng().gen::<f64>() * jitter)
+ } else {
+ sleep_ms
+ };
+ tokio::time::sleep(node_timer_delay(jittered)).await;
+ sleep_ms = js_min(sleep_ms * backoff, max_sleep);
+ if past_deadline() {
+ return (Err(error), attempts);
+ }
+ }
+ }
+ }
+}
+
+// Node's `TIMEOUT_MAX`, the largest delay that `setTimeout` keeps.
+const NODE_TIMEOUT_MAX_MS: f64 = 2_147_483_647.0;
+
+/// Node's `setTimeout` uses 1 ms for a NaN, sub-1 or over-`TIMEOUT_MAX` delay, so bad options cannot fail a retry.
+fn node_timer_delay(ms: f64) -> Duration {
+ let ms = if (1.0..=NODE_TIMEOUT_MAX_MS).contains(&ms) {
+ ms
+ } else {
+ 1.0
+ };
+ Duration::from_secs_f64(ms / 1000.0)
+}
+
+/// `Math.min(a, b)`, which returns NaN when an input is NaN. `f64::min` returns the other input.
+fn js_min(a: f64, b: f64) -> f64 {
+ if a.is_nan() || b.is_nan() {
+ f64::NAN
+ } else {
+ a.min(b)
+ }
+}
+
+const NON_RETRIABLE_REASON: &str = "Processing error - non-retriable";
+
+pub fn with_step_retry<T, U>(
+ step: ProcessingStep<T, U>,
+ options: RetryOptions,
+) -> ProcessingStep<T, U>
+where
+ T: Clone + Send + Sync + 'static,
+ U: Send + 'static,
+{
+ let name: Arc<str> = Arc::from(
+ options
+ .name
+ .clone()
+ .unwrap_or_else(|| step.name().to_string()),
+ );
+ let step_name = step.name().to_string();
+ let options = Arc::new(options);
+ ProcessingStep::new(&step_name, move |value: T| {
+ let step = step.clone();
+ let options = options.clone();
+ let name = name.clone();
+ async move {
+ let (result, attempts) =
+ retry_with_input(value, |value| step.call(value), &options).await;
+ match result {
+ Ok(result) => {
+ record_retry_attempts(name, "completed", attempts);
+ Ok(result)
+ }
+ Err(error) => {
+ let retriable = is_retriable(&error);
+ tracing::error!(step = %name, error = %error, ?retriable, "Step failed");
+ if retriable == Some(false) {
+ record_retry_attempts(name, "non_retriable", attempts);
+ Ok(dlq(NON_RETRIABLE_REASON, Some(error)))
+ } else {
+ record_retry_attempts(name, "exhausted", attempts);
+ Err(error)
+ }
+ }
+ }
+ }
+ })
+}
+
+pub fn with_chunk_retry<T, U>(
+ step: ChunkProcessingStep<T, U>,
+ options: RetryOptions,
+) -> ChunkProcessingStep<T, U>
+where
+ T: Clone + Send + Sync + 'static,
+ U: Send + 'static,
+{
+ let name: Arc<str> = Arc::from(
+ options
+ .name
+ .clone()
+ .unwrap_or_else(|| step.name().to_string()),
+ );
+ let step_name = step.name().to_string();
+ let options = Arc::new(options);
+ ChunkProcessingStep::new(&step_name, move |values: Vec<T>| {
+ let step = step.clone();
+ let options = options.clone();
+ let name = name.clone();
+ async move {
+ let count = values.len();
+ let (result, attempts) =
+ retry_with_input(values, |values| step.call(values), &options).await;
+ match result {
+ Ok(result) => {
+ record_retry_attempts(name, "completed", attempts);
+ Ok(result)
+ }
+ Err(error) => {
+ let retriable = is_retriable(&error);
+ tracing::error!(step = %name, error = %error, chunk_size = count, ?retriable, "Chunk step failed");
+ if retriable == Some(false) {
+ record_retry_attempts(name, "non_retriable", attempts);
+ let shared = Arc::new(error);
+ Ok((0..count)
+ .map(|_| {
+ let mut r: PipelineResult<U> = dlq(NON_RETRIABLE_REASON, None);
+ if let crate::results::Outcome::NonOk(
+ crate::results::NonOk::Dlq { error, .. },
+ ) = &mut r.outcome
+ {
+ *error = Some(shared.clone());
+ }
+ r
+ })
+ .collect())
+ } else {
+ record_retry_attempts(name, "exhausted", attempts);
+ Err(error)
+ }
+ }
+ }
+ }
+ })
+}
diff --git a/rust/ingestion-framework/src/side_effect_handling.rs b/rust/ingestion-framework/src/side_effect_handling.rs
new file mode 100644
index 000000000000..637612b2c521
--- /dev/null
+++ b/rust/ingestion-framework/src/side_effect_handling.rs
@@ -0,0 +1,118 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};
+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};
+use crate::metrics::record_side_effect_result;
+use crate::pipeline::{DynPipeline, Pipeline};
+use crate::side_effects::{SideEffect, SideEffectScheduler};
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct SideEffectHandlingConfig {
+ pub await_side_effects: bool,
+}
+
+async fn handle(
+ scheduler: &dyn SideEffectScheduler,
+ config: SideEffectHandlingConfig,
+ side_effects: Vec<SideEffect>,
+) {
+ if side_effects.is_empty() {
+ return;
+ }
+ if config.await_side_effects {
+ let settled =
+ futures::future::join_all(side_effects.into_iter().map(SideEffect::wait)).await;
+ for result in settled {
+ record_side_effect_result(result.is_ok());
+ }
+ } else {
+ for side_effect in side_effects {
+ scheduler.schedule(side_effect);
+ }
+ }
+}
+
+pub struct SideEffectHandlingPipeline<I, O, CI, CO> {
+ sub: DynChunkPipeline<I, O, CI, CO>,
+ scheduler: Arc<dyn SideEffectScheduler>,
+ config: SideEffectHandlingConfig,
+}
+
+impl<I, O, CI, CO> SideEffectHandlingPipeline<I, O, CI, CO> {
+ pub fn new(
+ sub: DynChunkPipeline<I, O, CI, CO>,
+ scheduler: Arc<dyn SideEffectScheduler>,
+ config: SideEffectHandlingConfig,
+ ) -> Self {
+ Self {
+ sub,
+ scheduler,
+ config,
+ }
+ }
+}
+
+#[async_trait]
+impl<I, O, CI, CO> ChunkPipeline<I, O, CI, CO> for SideEffectHandlingPipeline<I, O, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.sub.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ let Some(mut results) = self.sub.next().await? else {
+ return Ok(None);
+ };
+ let side_effects: Vec<SideEffect> = results
+ .iter_mut()
+ .flat_map(|r| std::mem::take(&mut r.context.side_effects))
+ .collect();
+ handle(self.scheduler.as_ref(), self.config, side_effects).await;
+ Ok(Some(results))
+ }
+}
+
+pub struct SideEffectHandlingProcessor<I, O, C> {
+ sub: DynPipeline<I, O, C>,
+ scheduler: Arc<dyn SideEffectScheduler>,
+ config: SideEffectHandlingConfig,
+}
+
+impl<I, O, C> SideEffectHandlingProcessor<I, O, C> {
+ pub fn new(
+ sub: DynPipeline<I, O, C>,
+ scheduler: Arc<dyn SideEffectScheduler>,
+ config: SideEffectHandlingConfig,
+ ) -> Self {
+ Self {
+ sub,
+ scheduler,
+ config,
+ }
+ }
+}
+
+#[async_trait]
+impl<I, O, C> Pipeline<I, O, C> for SideEffectHandlingProcessor<I, O, C>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ C: Send + 'static,
+{
+ async fn process(
+ &self,
+ input: OkResultWithContext<I, C>,
+ ) -> anyhow::Result<ResultWithContext<O, C>> {
+ let mut result = self.sub.process(input).await?;
+ let side_effects = std::mem::take(&mut result.context.side_effects);
+ handle(self.scheduler.as_ref(), self.config, side_effects).await;
+ Ok(result)
+ }
+}
diff --git a/rust/ingestion-framework/src/side_effects.rs b/rust/ingestion-framework/src/side_effects.rs
new file mode 100644
index 000000000000..efdd69c5d261
--- /dev/null
+++ b/rust/ingestion-framework/src/side_effects.rs
@@ -0,0 +1,245 @@
+//! `spawn` polls once on the caller's thread, as a Node async function runs to its first `await`, so produces start in creation order.
+
+use std::collections::BTreeMap;
+use std::future::Future;
+use std::panic::AssertUnwindSafe;
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
+
+use futures::future::{BoxFuture, Shared};
+use futures::FutureExt;
+use tokio::task::JoinHandle;
+
+enum State {
+ Running(JoinHandle<anyhow::Result<()>>),
+ Settled(Option<anyhow::Result<()>>),
+}
+
+pub struct SideEffect {
+ state: State,
+}
+
+fn panic_error() -> anyhow::Error {
+ anyhow::anyhow!("side effect task failed: panicked")
+}
+
+impl SideEffect {
+ pub fn spawn<F>(fut: F) -> Self
+ where
+ F: Future<Output = anyhow::Result<()>> + Send + 'static,
+ {
+ let mut fut = Box::pin(fut);
+ let mut cx = Context::from_waker(futures::task::noop_waker_ref());
+ // A Node async function that throws before its first await rejects instead of throwing.
+ match std::panic::catch_unwind(AssertUnwindSafe(|| fut.as_mut().poll(&mut cx))) {
+ Ok(Poll::Ready(result)) => Self::settled(result),
+ // The task polls again with a real waker, so the no-op waker loses no wake-up.
+ Ok(Poll::Pending) => Self {
+ state: State::Running(tokio::spawn(fut)),
+ },
+ Err(_) => Self::settled(Err(panic_error())),
+ }
+ }
+
+ fn settled(result: anyhow::Result<()>) -> Self {
+ Self {
+ state: State::Settled(Some(result)),
+ }
+ }
+
+ pub fn resolved() -> Self {
+ Self::settled(Ok(()))
+ }
+
+ pub fn rejected(error: anyhow::Error) -> Self {
+ Self::settled(Err(error))
+ }
+
+ pub fn is_finished(&self) -> bool {
+ match &self.state {
+ State::Running(handle) => handle.is_finished(),
+ State::Settled(_) => true,
+ }
+ }
+
+ /// `None` while the work runs. After a `Some`, no result is left and `wait()` succeeds.
+ pub fn try_take(&mut self) -> Option<anyhow::Result<()>> {
+ match &mut self.state {
+ State::Settled(result) => result.take(),
+ State::Running(handle) => {
+ if !handle.is_finished() {
+ return None;
+ }
+ // Unconstrained: a long prune exhausts the coop budget and reads done tasks as pending.
+ let result = match tokio::task::unconstrained(handle).now_or_never()? {
+ Ok(result) => result,
+ Err(join_error) => Err(join_failure(join_error)),
+ };
+ self.state = State::Settled(None);
+ Some(result)
+ }
+ }
+ }
+
+ pub async fn wait(self) -> anyhow::Result<()> {
+ match self.state {
+ State::Settled(result) => result.unwrap_or(Ok(())),
+ State::Running(handle) => match handle.await {
+ Ok(result) => result,
+ Err(join_error) => Err(join_failure(join_error)),
+ },
+ }
+ }
+
+ pub fn into_future(self) -> BoxFuture<'static, anyhow::Result<()>> {
+ self.wait().boxed()
+ }
+}
+
+fn join_failure(join_error: tokio::task::JoinError) -> anyhow::Error {
+ anyhow::anyhow!("side effect task failed: {join_error}")
+}
+
+impl std::fmt::Debug for SideEffect {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("SideEffect")
+ .field("finished", &self.is_finished())
+ .finish()
+ }
+}
+
+pub trait SideEffectScheduler: Send + Sync {
+ fn schedule(&self, side_effect: SideEffect);
+}
+
+/// Unlike Node, it latches the first failure, so the consumer never stores offsets for a batch whose side effect failed early.
+#[derive(Default, Clone)]
+pub struct PromiseScheduler {
+ inner: Arc<Mutex<SchedulerState>>,
+}
+
+// Prune at this size, then at twice the surviving size, so schedule() is amortized O(1).
+const MIN_PRUNE_AT: usize = 64;
+
+type Pending = Shared<BoxFuture<'static, Result<(), Arc<anyhow::Error>>>>;
+
+struct SchedulerState {
+ pending: BTreeMap<u64, Pending>,
+ next_id: u64,
+ prune_at: usize,
+ failure: Option<Arc<anyhow::Error>>,
+}
+
+impl Default for SchedulerState {
+ fn default() -> Self {
+ Self {
+ pending: BTreeMap::new(),
+ next_id: 0,
+ prune_at: MIN_PRUNE_AT,
+ failure: None,
+ }
+ }
+}
+
+fn poll_settled(pending: &Pending) -> Option<Result<(), Arc<anyhow::Error>>> {
+ if let Some(result) = pending.peek() {
+ return Some(result.clone());
+ }
+ // Unconstrained: a long prune exhausts the coop budget and reads done tasks as pending.
+ tokio::task::unconstrained(pending.clone()).now_or_never()
+}
+
+impl SchedulerState {
+ fn latch(&mut self, error: Arc<anyhow::Error>) {
+ if self.failure.is_none() {
+ self.failure = Some(error);
+ }
+ }
+
+ fn prune(&mut self) {
+ let mut failures = Vec::new();
+ self.pending
+ .retain(|_, pending| match poll_settled(pending) {
+ None => true,
+ Some(Ok(())) => false,
+ Some(Err(error)) => {
+ failures.push(error);
+ false
+ }
+ });
+ for error in failures {
+ self.latch(error);
+ }
+ self.prune_at = (self.pending.len() * 2).max(MIN_PRUNE_AT);
+ }
+}
+
+impl PromiseScheduler {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn pending_count(&self) -> usize {
+ let mut state = self.inner.lock().unwrap();
+ state.prune();
+ state.pending.len()
+ }
+
+ /// Entries leave only when they settle, so an overlapping call also waits for them and a cancelled call leaves them.
+ pub async fn wait_for_all(&self) -> anyhow::Result<()> {
+ let snapshot: Vec<(u64, Pending)> = self
+ .inner
+ .lock()
+ .unwrap()
+ .pending
+ .iter()
+ .map(|(id, pending)| (*id, pending.clone()))
+ .collect();
+ let mut first_failure = None;
+ for (id, pending) in snapshot {
+ let result = pending.await;
+ let mut state = self.inner.lock().unwrap();
+ state.pending.remove(&id);
+ if let Err(error) = result {
+ // The latch keeps the failure if this call is cancelled before it returns.
+ state.latch(error.clone());
+ first_failure.get_or_insert(error);
+ }
+ }
+ // An overlapping call can take the latched failure first, so keep the one seen here.
+ match self.inner.lock().unwrap().failure.take().or(first_failure) {
+ Some(error) => Err(crate::errors::shared_error(error)),
+ None => Ok(()),
+ }
+ }
+
+ pub async fn wait_for_all_settled(&self) {
+ let _ = self.wait_for_all().await;
+ }
+}
+
+impl SideEffectScheduler for PromiseScheduler {
+ fn schedule(&self, mut side_effect: SideEffect) {
+ // Settled on creation, so it skips the set, as a settled Node promise leaves it.
+ match side_effect.try_take() {
+ Some(Ok(())) => return,
+ Some(Err(error)) => {
+ self.inner.lock().unwrap().latch(Arc::new(error));
+ return;
+ }
+ None => {}
+ }
+ let pending = side_effect
+ .wait()
+ .map(|r| r.map_err(Arc::new))
+ .boxed()
+ .shared();
+ let mut state = self.inner.lock().unwrap();
+ let id = state.next_id;
+ state.next_id += 1;
+ state.pending.insert(id, pending);
+ if state.pending.len() >= state.prune_at {
+ state.prune();
+ }
+ }
+}
diff --git a/rust/ingestion-framework/src/steps.rs b/rust/ingestion-framework/src/steps.rs
new file mode 100644
index 000000000000..25c435910579
--- /dev/null
+++ b/rust/ingestion-framework/src/steps.rs
@@ -0,0 +1,136 @@
+//! Rust closures have no `.name`, so a step carries its name.
+
+use std::future::Future;
+use std::sync::Arc;
+
+use futures::future::BoxFuture;
+use futures::FutureExt;
+
+use crate::results::PipelineResult;
+
+pub type StepFuture<U> = BoxFuture<'static, anyhow::Result<PipelineResult<U>>>;
+pub type ChunkStepFuture<U> = BoxFuture<'static, anyhow::Result<Vec<PipelineResult<U>>>>;
+
+/// An `Err` is the Node `throw`: it propagates and is not converted into a DLQ.
+pub struct ProcessingStep<T, U> {
+ name: Arc<str>,
+ f: Arc<dyn Fn(T) -> StepFuture<U> + Send + Sync>,
+}
+
+impl<T, U> Clone for ProcessingStep<T, U> {
+ fn clone(&self) -> Self {
+ Self {
+ name: self.name.clone(),
+ f: self.f.clone(),
+ }
+ }
+}
+
+impl<T: 'static, U: 'static> ProcessingStep<T, U> {
+ pub fn new<F, Fut>(name: &str, f: F) -> Self
+ where
+ F: Fn(T) -> Fut + Send + Sync + 'static,
+ Fut: Future<Output = anyhow::Result<PipelineResult<U>>> + Send + 'static,
+ {
+ Self {
+ name: Arc::from(name),
+ f: Arc::new(move |value| f(value).boxed()),
+ }
+ }
+
+ pub fn sync<F>(name: &str, f: F) -> Self
+ where
+ F: Fn(T) -> anyhow::Result<PipelineResult<U>> + Send + Sync + 'static,
+ U: Send,
+ {
+ Self {
+ name: Arc::from(name),
+ f: Arc::new(move |value| futures::future::ready(f(value)).boxed()),
+ }
+ }
+
+ pub fn name(&self) -> &Arc<str> {
+ &self.name
+ }
+
+ pub fn call(&self, value: T) -> StepFuture<U> {
+ (self.f)(value)
+ }
+
+ pub fn renamed(mut self, name: &str) -> Self {
+ self.name = Arc::from(name);
+ self
+ }
+}
+
+pub struct ChunkProcessingStep<T, U> {
+ name: Arc<str>,
+ f: Arc<dyn Fn(Vec<T>) -> ChunkStepFuture<U> + Send + Sync>,
+}
+
+impl<T, U> Clone for ChunkProcessingStep<T, U> {
+ fn clone(&self) -> Self {
+ Self {
+ name: self.name.clone(),
+ f: self.f.clone(),
+ }
+ }
+}
+
+impl<T: 'static, U: 'static> ChunkProcessingStep<T, U> {
+ pub fn new<F, Fut>(name: &str, f: F) -> Self
+ where
+ F: Fn(Vec<T>) -> Fut + Send + Sync + 'static,
+ Fut: Future<Output = anyhow::Result<Vec<PipelineResult<U>>>> + Send + 'static,
+ {
+ Self {
+ name: Arc::from(name),
+ f: Arc::new(move |values| f(values).boxed()),
+ }
+ }
+
+ pub fn sync<F>(name: &str, f: F) -> Self
+ where
+ F: Fn(Vec<T>) -> anyhow::Result<Vec<PipelineResult<U>>> + Send + Sync + 'static,
+ U: Send,
+ {
+ Self {
+ name: Arc::from(name),
+ f: Arc::new(move |values| futures::future::ready(f(values)).boxed()),
+ }
+ }
+
+ pub fn name(&self) -> &Arc<str> {
+ &self.name
+ }
+
+ pub fn call(&self, values: Vec<T>) -> ChunkStepFuture<U> {
+ (self.f)(values)
+ }
+}
+
+impl<T: Send + 'static, U: Send + 'static> ChunkProcessingStep<T, U> {
+ pub fn per_item(step: ProcessingStep<T, U>) -> Self {
+ crate::per_element::per_element(step, None)
+ }
+}
+
+pub fn step<T, U, F, Fut>(name: &str, f: F) -> ProcessingStep<T, U>
+where
+ T: 'static,
+ U: 'static,
+ F: Fn(T) -> Fut + Send + Sync + 'static,
+ Fut: Future<Output = anyhow::Result<PipelineResult<U>>> + Send + 'static,
+{
+ ProcessingStep::new(name, f)
+}
+
+pub fn chunk_step<T, U, F, Fut>(name: &str, f: F) -> ChunkProcessingStep<T, U>
+where
+ T: 'static,
+ U: 'static,
+ F: Fn(Vec<T>) -> Fut + Send + Sync + 'static,
+ Fut: Future<Output = anyhow::Result<Vec<PipelineResult<U>>>> + Send + 'static,
+{
+ ChunkProcessingStep::new(name, f)
+}
diff --git a/rust/ingestion-framework/src/sync.rs b/rust/ingestion-framework/src/sync.rs
new file mode 100644
index 000000000000..70f80e0f3224
--- /dev/null
+++ b/rust/ingestion-framework/src/sync.rs
@@ -0,0 +1,135 @@
+use std::collections::VecDeque;
+use std::future::Future;
+use std::sync::{Arc, Mutex};
+
+use tokio::sync::{oneshot, watch};
+
+/// A re-armable wake-up. A `resolve()` between `reset()` and `wait()` is not lost.
+pub struct ResettableSignal {
+ tx: watch::Sender<u64>,
+ baseline: Mutex<u64>,
+}
+
+impl Default for ResettableSignal {
+ fn default() -> Self {
+ let (tx, _) = watch::channel(0);
+ Self {
+ tx,
+ baseline: Mutex::new(0),
+ }
+ }
+}
+
+impl ResettableSignal {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn resolve(&self) {
+ self.tx.send_modify(|generation| *generation += 1);
+ }
+
+ pub fn reset(&self) {
+ *self.baseline.lock().unwrap() = *self.tx.borrow();
+ }
+
+ pub fn wait(&self) -> impl Future<Output = ()> + Send + 'static {
+ let baseline = *self.baseline.lock().unwrap();
+ let mut rx = self.tx.subscribe();
+ async move {
+ let _ = rx.wait_for(|generation| *generation != baseline).await;
+ }
+ }
+}
+
+/// A caller queues when it calls `acquire()`, not when it first polls, so admission order is call order.
+pub struct FifoLimiter {
+ max: usize,
+ state: Mutex<LimiterState>,
+}
+
+struct LimiterState {
+ running: usize,
+ waiters: VecDeque<oneshot::Sender<LimiterPermit>>,
+}
+
+/// Reserved at `acquire()`, so a dropped future or a permit dropped in transit releases the slot.
+pub struct LimiterPermit {
+ limiter: Option<Arc<FifoLimiter>>,
+}
+
+impl FifoLimiter {
+ /// Panics when `max` is 0, as `pLimit(0)` throws when the Node pipeline is built.
+ pub fn new(max: usize) -> Arc<Self> {
+ assert!(
+ max > 0,
+ "Expected `concurrency` to be a number from 1 and up"
+ );
+ Arc::new(Self {
+ max,
+ state: Mutex::new(LimiterState {
+ running: 0,
+ waiters: VecDeque::new(),
+ }),
+ })
+ }
+
+ pub fn acquire(self: &Arc<Self>) -> impl Future<Output = LimiterPermit> + Send + 'static {
+ enum Reserved {
+ Now(LimiterPermit),
+ Queued(oneshot::Receiver<LimiterPermit>),
+ }
+ let reserved = {
+ let mut state = self.state.lock().unwrap();
+ if state.running < self.max && state.waiters.is_empty() {
+ state.running += 1;
+ Reserved::Now(LimiterPermit {
+ limiter: Some(self.clone()),
+ })
+ } else {
+ let (tx, rx) = oneshot::channel();
+ state.waiters.push_back(tx);
+ Reserved::Queued(rx)
+ }
+ };
+ async move {
+ match reserved {
+ Reserved::Now(permit) => permit,
+ // Every queued sender gets a send, so this receive does not fail.
+ Reserved::Queued(rx) => rx.await.unwrap_or(LimiterPermit { limiter: None }),
+ }
+ }
+ }
+
+ fn release(self: &Arc<Self>) {
+ loop {
+ let waiter = {
+ let mut state = self.state.lock().unwrap();
+ match state.waiters.pop_front() {
+ Some(waiter) => waiter,
+ None => {
+ state.running -= 1;
+ return;
+ }
+ }
+ };
+ // The slot moves to the next waiter. A waiter that is gone hands it back.
+ match waiter.send(LimiterPermit {
+ limiter: Some(self.clone()),
+ }) {
+ Ok(()) => return,
+ Err(mut permit) => {
+ permit.limiter = None;
+ }
+ }
+ }
+ }
+}
+
+impl Drop for LimiterPermit {
+ fn drop(&mut self) {
+ if let Some(limiter) = self.limiter.take() {
+ limiter.release();
+ }
+ }
+}
diff --git a/rust/ingestion-framework/src/timeout_barrier.rs b/rust/ingestion-framework/src/timeout_barrier.rs
new file mode 100644
index 000000000000..97b3e36d8f7f
--- /dev/null
+++ b/rust/ingestion-framework/src/timeout_barrier.rs
@@ -0,0 +1,56 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+use crate::budget::BatchBudget;
+use crate::context::{OkResultWithContext, ResultWithContext};
+use crate::metrics::{record_budget_checkpoint, BudgetCheckpoint};
+use crate::pipeline::{DynPipeline, Pipeline};
+use crate::results::{NonOk, Outcome};
+
+const TIMEOUT_BARRIER: &str = "timeoutBarrier";
+
+/// An element with time left continues with an unlimited budget, so it always finishes.
+pub struct TimeoutBarrierPipeline<I, O, C> {
+ previous: DynPipeline<I, O, C>,
+}
+
+impl<I, O, C> TimeoutBarrierPipeline<I, O, C> {
+ pub fn new(previous: DynPipeline<I, O, C>) -> Self {
+ Self { previous }
+ }
+}
+
+#[async_trait]
+impl<I, O, C> Pipeline<I, O, C> for TimeoutBarrierPipeline<I, O, C>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ C: Send + 'static,
+{
+ async fn process(
+ &self,
+ input: OkResultWithContext<I, C>,
+ ) -> anyhow::Result<ResultWithContext<O, C>> {
+ let mut result = self.previous.process(input).await?;
+ let Some(budget) = result.context.budget.as_ref() else {
+ return Ok(result);
+ };
+ if !result.is_ok() {
+ return Ok(result);
+ }
+ if budget.exhausted() {
+ record_budget_checkpoint(BudgetCheckpoint::Step, Arc::from(TIMEOUT_BARRIER));
+ return Ok(ResultWithContext {
+ result: Outcome::NonOk(NonOk::Timeout {
+ reason: format!("budget exceeded before {TIMEOUT_BARRIER}"),
+ }),
+ context: result.context,
+ });
+ }
+ if !budget.is_unlimited() {
+ result.context.budget = Some(BatchBudget::unlimited());
+ }
+ Ok(result)
+ }
+}
diff --git a/rust/ingestion-framework/src/warning_handling.rs b/rust/ingestion-framework/src/warning_handling.rs
new file mode 100644
index 000000000000..6d52995e07f9
--- /dev/null
+++ b/rust/ingestion-framework/src/warning_handling.rs
@@ -0,0 +1,190 @@
+use std::collections::HashMap;
+use std::sync::{Arc, OnceLock};
+
+use async_trait::async_trait;
+
+use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};
+use crate::context::{ChunkResults, OkResultWithContext, PipelineWarning};
+use crate::js_json;
+use crate::metrics::INGESTION_WARNING_COUNTER;
+use crate::outputs::{HasTeamId, IngestionOutputs, OutputMessage, INGESTION_WARNINGS_OUTPUT};
+use crate::results::{NonOk, Outcome};
+use crate::side_effects::SideEffect;
+
+/// Node logs a failed produce and does not reject, so that is `Ok(false)`. `Err` fails the side effect, as Node's emit rejects.
+#[async_trait]
+pub trait IngestionWarningEmitter: Send + Sync {
+ async fn emit(&self, team_id: i64, warning: PipelineWarning) -> anyhow::Result<bool>;
+}
+
+pub struct OutputsWarningEmitter {
+ pub outputs: Arc<dyn IngestionOutputs>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct WarningClassification {
+ pub category: &'static str,
+ pub severity: &'static str,
+}
+
+// The Node registry exports this file, and a Node test fails when the two differ.
+const WARNING_TYPES_JSON: &str =
+ include_str!("../../common/ingestion_warnings/warning_types.generated.json");
+
+pub fn warning_classification(warning_type: &str) -> Option<WarningClassification> {
+ #[derive(serde::Deserialize)]
+ struct Entry<'a> {
+ #[serde(rename = "type")]
+ warning_type: &'a str,
+ category: &'a str,
+ severity: &'a str,
+ }
+ #[derive(serde::Deserialize)]
+ struct Registry<'a> {
+ #[serde(borrow)]
+ types: Vec<Entry<'a>>,
+ }
+ static REGISTRY: OnceLock<HashMap<&'static str, WarningClassification>> = OnceLock::new();
+ REGISTRY
+ .get_or_init(|| {
+ let registry: Registry<'static> =
+ serde_json::from_str(WARNING_TYPES_JSON).expect("valid warning types JSON");
+ registry
+ .types
+ .into_iter()
+ .map(|entry| {
+ let classification = WarningClassification {
+ category: entry.category,
+ severity: entry.severity,
+ };
+ (entry.warning_type, classification)
+ })
+ .collect()
+ })
+ .get(warning_type)
+ .copied()
+}
+
+/// Byte for byte as Node. A key the caller already set keeps its place and takes the new value, as in an object spread.
+pub fn serialize_ingestion_warning(
+ team_id: i64,
+ warning: &PipelineWarning,
+) -> anyhow::Result<String> {
+ let Some(classification) = warning_classification(&warning.warning_type) else {
+ anyhow::bail!(
+ "ingestion warning type {:?} is not in INGESTION_WARNING_TYPES",
+ warning.warning_type
+ );
+ };
+ let mut details = warning.details.clone();
+ details.insert("category".into(), classification.category.into());
+ details.insert("severity".into(), classification.severity.into());
+ if let Some(step) = warning.pipeline_step.as_deref().filter(|s| !s.is_empty()) {
+ details.insert("pipelineStep".into(), step.into());
+ }
+ let mut details_json = String::new();
+ js_json::write_object(&mut details_json, details.iter());
+
+ let mut out = String::with_capacity(details_json.len() + 128);
+ out.push_str("{\"team_id\":");
+ js_json::write_number(&mut out, team_id as f64);
+ out.push_str(",\"type\":");
+ js_json::write_string(&mut out, &warning.warning_type);
+ out.push_str(",\"source\":");
+ js_json::write_string(
+ &mut out,
+ warning.source.as_deref().unwrap_or("plugin-server"),
+ );
+ out.push_str(",\"details\":");
+ js_json::write_string(&mut out, &details_json);
+ out.push_str(",\"timestamp\":");
+ js_json::write_string(
+ &mut out,
+ &chrono::Utc::now()
+ .format("%Y-%m-%d %H:%M:%S%.3f")
+ .to_string(),
+ );
+ out.push('}');
+ Ok(out)
+}
+
+#[async_trait]
+impl IngestionWarningEmitter for OutputsWarningEmitter {
+ async fn emit(&self, team_id: i64, warning: PipelineWarning) -> anyhow::Result<bool> {
+ metrics::counter!(INGESTION_WARNING_COUNTER, "type" => warning.warning_type.clone(), "emitted" => "true")
+ .increment(1);
+ let value = serialize_ingestion_warning(team_id, &warning)?.into_bytes();
+ match self
+ .outputs
+ .queue_messages(
+ INGESTION_WARNINGS_OUTPUT,
+ vec![OutputMessage {
+ value: Some(value.into()),
+ ..Default::default()
+ }],
+ )
+ .await
+ {
+ Ok(()) => Ok(true),
+ Err(error) => {
+ tracing::warn!(team_id, error = %error, "Failed to produce ingestion warning");
+ Ok(false)
+ }
+ }
+ }
+}
+
+/// A timed-out result drops its warnings, because the redelivery emits them again.
+pub struct IngestionWarningHandlingChunkPipeline<I, O, CI, CO> {
+ emitter: Arc<dyn IngestionWarningEmitter>,
+ previous: DynChunkPipeline<I, O, CI, CO>,
+}
+
+impl<I, O, CI, CO> IngestionWarningHandlingChunkPipeline<I, O, CI, CO> {
+ pub fn new(
+ emitter: Arc<dyn IngestionWarningEmitter>,
+ previous: DynChunkPipeline<I, O, CI, CO>,
+ ) -> Self {
+ Self { emitter, previous }
+ }
+}
+
+#[async_trait]
+impl<I, O, CI, CO> ChunkPipeline<I, O, CI, CO>
+ for IngestionWarningHandlingChunkPipeline<I, O, CI, CO>
+where
+ I: Send + 'static,
+ O: Send + 'static,
+ CI: Send + 'static,
+ CO: HasTeamId + Send + 'static,
+{
+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {
+ self.previous.feed(elements)
+ }
+
+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {
+ let Some(mut results) = self.previous.next().await? else {
+ return Ok(None);
+ };
+ for item in results.iter_mut() {
+ if item.context.warnings.is_empty() {
+ continue;
+ }
+ let warnings = std::mem::take(&mut item.context.warnings);
+ if matches!(item.result, Outcome::NonOk(NonOk::Timeout { .. })) {
+ continue;
+ }
+ let team_id = item.context.data.team_id();
+ for warning in warnings {
+ let emitter = self.emitter.clone();
+ // The emit starts on creation, so warnings are queued in item order.
+ item.context
+ .side_effects
+ .push(SideEffect::spawn(async move {
+ emitter.emit(team_id, warning).await.map(|_| ())
+ }));
+ }
+ }
+ Ok(Some(results))
+ }
+}
diff --git a/rust/ingestion-framework/tests/common/mod.rs b/rust/ingestion-framework/tests/common/mod.rs
new file mode 100644
index 000000000000..4223dda424f4
--- /dev/null
+++ b/rust/ingestion-framework/tests/common/mod.rs
@@ -0,0 +1,153 @@
+#![allow(dead_code)]
+
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use ingestion_framework::{create_ok_context, ChunkPipeline, OkResultWithContext};
+
+pub type Ctx = ();
+
+pub fn items<T>(values: impl IntoIterator<Item = T>) -> Vec<OkResultWithContext<T, Ctx>> {
+ values
+ .into_iter()
+ .map(|v| create_ok_context(v, ()))
+ .collect()
+}
+
+/// A paused tokio clock advances when all tasks wait, like `jest.advanceTimersByTimeAsync`.
+pub async fn consume_all<I, O, CI, CO>(pipeline: &dyn ChunkPipeline<I, O, CI, CO>) -> Vec<O> {
+ let mut values = Vec::new();
+ while let Some(chunk) = pipeline.next().await.expect("next") {
+ for r in chunk {
+ if let Some(v) = r.result.into_value() {
+ values.push(v);
+ }
+ }
+ }
+ values
+}
+
+/// The bound is 1 ms past `advance_ms`, because tokio fires a timer and this timeout in the same millisecond tick.
+pub async fn consume_all_within<I, O, CI, CO>(
+ pipeline: &dyn ChunkPipeline<I, O, CI, CO>,
+ advance_ms: u64,
+) -> Vec<O> {
+ within_ms(advance_ms, consume_all(pipeline)).await
+}
+
+pub async fn within_ms<T>(advance_ms: u64, fut: impl std::future::Future<Output = T>) -> T {
+ tokio::time::timeout(Duration::from_millis(advance_ms + 1), fut)
+ .await
+ .unwrap_or_else(|_| panic!("did not finish within {advance_ms} ms of virtual time"))
+}
+
+pub async fn collect_chunks<I, O, CI, CO>(
+ pipeline: &dyn ChunkPipeline<I, O, CI, CO>,
+) -> Vec<Vec<O>> {
+ let mut chunks = Vec::new();
+ while let Some(chunk) = pipeline.next().await.expect("next") {
+ chunks.push(
+ chunk
+ .into_iter()
+ .filter_map(|r| r.result.into_value())
+ .collect(),
+ );
+ }
+ chunks
+}
+
+pub async fn sleep_ms(ms: u64) {
+ tokio::time::sleep(Duration::from_millis(ms)).await
+}
+
+pub fn shared<T>(value: T) -> Arc<Mutex<T>> {
+ Arc::new(Mutex::new(value))
+}
+
+use async_trait::async_trait;
+use ingestion_framework::{IngestionOutputs, KafkaMessage, OutputMessage};
+
+#[derive(Clone)]
+pub struct MockOutputs {
+ pub calls: Arc<Mutex<Vec<(String, OutputMessage)>>>,
+ gate: Arc<tokio::sync::watch::Sender<bool>>,
+}
+
+impl Default for MockOutputs {
+ fn default() -> Self {
+ let (tx, _) = tokio::sync::watch::channel(true);
+ Self {
+ calls: Arc::default(),
+ gate: Arc::new(tx),
+ }
+ }
+}
+
+impl MockOutputs {
+ pub fn close_gate(&self) {
+ self.gate.send_replace(false);
+ }
+ pub fn open_gate(&self) {
+ self.gate.send_replace(true);
+ }
+ pub fn calls(&self) -> Vec<(String, OutputMessage)> {
+ self.calls.lock().unwrap().clone()
+ }
+}
+
+#[async_trait]
+impl IngestionOutputs for MockOutputs {
+ async fn produce(&self, output: &str, message: OutputMessage) -> anyhow::Result<()> {
+ self.calls
+ .lock()
+ .unwrap()
+ .push((output.to_string(), message));
+ let mut rx = self.gate.subscribe();
+ let _ = rx.wait_for(|open| *open).await;
+ Ok(())
+ }
+ async fn queue_messages(
+ &self,
+ output: &str,
+ messages: Vec<OutputMessage>,
+ ) -> anyhow::Result<()> {
+ for m in messages {
+ self.produce(output, m).await?;
+ }
+ Ok(())
+ }
+}
+
+pub fn test_message(key: Option<&[u8]>) -> KafkaMessage {
+ KafkaMessage {
+ topic: "test-topic".into(),
+ partition: 0,
+ offset: 0,
+ key: key.map(|k| k.to_vec()),
+ value: Some(b"test-value".to_vec().into()),
+ headers: vec![],
+ timestamp: Some(0),
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct MsgCtx {
+ pub message: KafkaMessage,
+}
+
+impl ingestion_framework::HasMessage for MsgCtx {
+ fn message(&self) -> &KafkaMessage {
+ &self.message
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct TeamCtx {
+ pub team_id: i64,
+}
+
+impl ingestion_framework::HasTeamId for TeamCtx {
+ fn team_id(&self) -> i64 {
+ self.team_id
+ }
+}
diff --git a/rust/ingestion-framework/tests/docs_01_introduction.rs b/rust/ingestion-framework/tests/docs_01_introduction.rs
new file mode 100644
index 000000000000..77a0acf2cb05
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_01_introduction.rs
@@ -0,0 +1,255 @@
+mod common;
+
+use std::sync::Arc;
+
+use common::{items, shared};
+use ingestion_framework::{
+ chunk_step, create_ok_context, dlq, drop, new_chunk_pipeline, new_pipeline, ok, redirect, step,
+ PipelineResultType, ProcessingStep,
+};
+
+#[tokio::test]
+async fn steps_use_the_factory_pattern_for_dependency_injection_and_naming() {
+ fn create_uppercase_step() -> ProcessingStep<String, String> {
+ step("uppercaseStep", |input: String| async move {
+ Ok(ok(input.to_uppercase()))
+ })
+ }
+ struct MultiplierConfig {
+ factor: i64,
+ }
+ fn create_multiply_step(config: MultiplierConfig) -> ProcessingStep<i64, i64> {
+ step("multiplyStep", move |input: i64| async move {
+ Ok(ok(input * config.factor))
+ })
+ }
+
+ let string_pipeline = new_pipeline::<String, ()>()
+ .pipe(create_uppercase_step())
+ .build();
+ let number_pipeline = new_pipeline::<i64, ()>()
+ .pipe(create_multiply_step(MultiplierConfig { factor: 3 }))
+ .build();
+
+ let r = string_pipeline
+ .process(create_ok_context("hello".into(), ()))
+ .await
+ .unwrap();
+ assert_eq!(r.value().map(String::as_str), Some("HELLO"));
+ let r = number_pipeline
+ .process(create_ok_context(5, ()))
+ .await
+ .unwrap();
+ assert_eq!(r.value(), Some(&15));
+}
+
+#[tokio::test]
+async fn steps_accumulate_data_through_the_pipeline() {
+ #[derive(Clone)]
+ struct Input {
+ event_id: String,
+ }
+ struct Validated {
+ event_id: String,
+ is_valid: bool,
+ }
+ struct Enriched {
+ event_id: String,
+ is_valid: bool,
+ timestamp: i64,
+ }
+ let pipeline = new_pipeline::<Input, ()>()
+ .pipe(step("validateStep", |i: Input| async move {
+ Ok(ok(Validated {
+ event_id: i.event_id,
+ is_valid: true,
+ }))
+ }))
+ .pipe(step("enrichStep", |i: Validated| async move {
+ Ok(ok(Enriched {
+ event_id: i.event_id,
+ is_valid: i.is_valid,
+ timestamp: chrono::Utc::now().timestamp_millis(),
+ }))
+ }))
+ .pipe(step("finalizeStep", |i: Enriched| async move {
+ Ok(ok(format!(
+ "Event {} valid={} at {}",
+ i.event_id, i.is_valid, i.timestamp
+ )))
+ }))
+ .build();
+
+ let r = pipeline
+ .process(create_ok_context(
+ Input {
+ event_id: "evt-123".into(),
+ },
+ (),
+ ))
+ .await
+ .unwrap();
+ let summary = r.value().unwrap();
+ assert!(summary.contains("evt-123"));
+ assert!(summary.contains("valid=true"));
+}
+
+#[tokio::test]
+async fn a_step_can_return_ok_to_pass_data_forward() {
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe(step("processDataStep", |i: String| async move {
+ Ok(ok(i.to_uppercase()))
+ }))
+ .build();
+ let r = pipeline
+ .process(create_ok_context("hello".into(), ()))
+ .await
+ .unwrap();
+ assert_eq!(r.value().unwrap(), "HELLO");
+}
+
+#[tokio::test]
+async fn a_pipeline_transforms_input_through_a_sequence_of_steps() {
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe(step("uppercaseStep", |i: String| async move {
+ Ok(ok(i.to_uppercase()))
+ }))
+ .pipe(step("addExclamationStep", |i: String| async move {
+ Ok(ok(i + "!"))
+ }))
+ .build();
+ let r = pipeline
+ .process(create_ok_context("hello".into(), ()))
+ .await
+ .unwrap();
+ assert_eq!(r.value().unwrap(), "HELLO!");
+}
+
+#[tokio::test]
+async fn a_step_can_return_dlq_to_signal_an_error() {
+ let pipeline = new_pipeline::<i64, ()>()
+ .pipe(step("validateDataStep", |v: i64| async move {
+ if v < 0 {
+ return Ok(dlq(
+ "Invalid value: must be non-negative",
+ Some(anyhow::anyhow!("Validation failed")),
+ ));
+ }
+ Ok(ok(v))
+ }))
+ .build();
+ let r = pipeline.process(create_ok_context(-1, ())).await.unwrap();
+ assert_eq!(r.result.result_type(), PipelineResultType::Dlq);
+ assert_eq!(r.result.result_type() as i32, 1);
+}
+
+#[tokio::test]
+async fn a_step_can_return_drop_to_silently_discard_an_item() {
+ let pipeline = new_pipeline::<&'static str, ()>()
+ .pipe(step("filterInternalStep", |t: &'static str| async move {
+ if t == "internal" {
+ return Ok(drop("Internal events are not processed"));
+ }
+ Ok(ok(t))
+ }))
+ .build();
+ let r = pipeline
+ .process(create_ok_context("internal", ()))
+ .await
+ .unwrap();
+ assert_eq!(r.result.result_type() as i32, 2);
+}
+
+#[tokio::test]
+async fn a_step_can_return_redirect_to_send_an_item_elsewhere() {
+ let pipeline = new_pipeline::<&'static str, ()>()
+ .pipe(step("routeByPriorityStep", |p: &'static str| async move {
+ if p == "high" {
+ return Ok(redirect("High priority event", "high_priority"));
+ }
+ Ok(ok(p))
+ }))
+ .build();
+ let r = pipeline
+ .process(create_ok_context("high", ()))
+ .await
+ .unwrap();
+ assert_eq!(r.result.result_type() as i32, 3);
+}
+
+#[tokio::test]
+async fn non_ok_results_short_circuit_the_pipeline_ok_results_propagate() {
+ let pipeline = new_pipeline::<&'static str, ()>()
+ .pipe(step("decisionStep", |action: &'static str| async move {
+ Ok(match action {
+ "dlq" => dlq("Error condition", None),
+ "drop" => drop("Filtered out"),
+ "redirect" => redirect("Redirecting", "other"),
+ _ => ok(action),
+ })
+ }))
+ .pipe(step("processStep", |action: &'static str| async move {
+ if action != "ok" {
+ anyhow::bail!("Unexpected action: {action}");
+ }
+ Ok(ok((action, true)))
+ }))
+ .build();
+
+ let r = pipeline.process(create_ok_context("ok", ())).await.unwrap();
+ assert!(r.is_ok());
+ assert_eq!(r.context.last_step(), Some("processStep"));
+ for (action, expected) in [("dlq", 1), ("drop", 2), ("redirect", 3)] {
+ let r = pipeline
+ .process(create_ok_context(action, ()))
+ .await
+ .unwrap();
+ assert_eq!(r.result.result_type() as i32, expected);
+ assert_eq!(r.context.last_step(), Some("decisionStep"));
+ }
+}
+
+#[tokio::test]
+async fn chunk_pipelines_are_created_using_new_chunk_pipeline_builder() {
+ let pipeline = new_chunk_pipeline::<String, ()>()
+ .pipe_chunk(chunk_step(
+ "uppercaseChunkStep",
+ |items: Vec<String>| async move {
+ Ok(items.into_iter().map(|s| ok(s.to_uppercase())).collect())
+ },
+ ))
+ .build();
+ pipeline.feed(items(["a", "b", "c"].map(String::from)));
+ let results = pipeline.next().await.unwrap().unwrap();
+ let values: Vec<_> = results.iter().map(|r| r.value().unwrap().clone()).collect();
+ assert_eq!(values, ["A", "B", "C"]);
+}
+
+#[tokio::test]
+async fn pipe_chunk_processes_all_items_in_a_single_call() {
+ let call_counts = shared(Vec::<usize>::new());
+ let counts = call_counts.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("chunkEnrichStep", move |items: Vec<i64>| {
+ counts.lock().unwrap().push(items.len());
+ async move { Ok(items.into_iter().map(|n| ok(n * 10)).collect()) }
+ }))
+ .build();
+ pipeline.feed(items([1, 2, 3, 4, 5]));
+ pipeline.next().await.unwrap();
+ assert_eq!(*call_counts.lock().unwrap(), vec![5]);
+}
+
+#[tokio::test]
+async fn feed_accepts_a_batch_and_next_returns_processed_chunks() {
+ let pipeline = new_chunk_pipeline::<&'static str, ()>()
+ .pipe_chunk(chunk_step(
+ "passthroughStep",
+ |items: Vec<&'static str>| async move { Ok(items.into_iter().map(ok).collect()) },
+ ))
+ .build();
+ pipeline.feed(items(["x", "y"]));
+ assert_eq!(pipeline.next().await.unwrap().unwrap().len(), 2);
+ assert!(pipeline.next().await.unwrap().is_none());
+ let _ = Arc::new(());
+}
diff --git a/rust/ingestion-framework/tests/docs_02_chunk_pipelines.rs b/rust/ingestion-framework/tests/docs_02_chunk_pipelines.rs
new file mode 100644
index 000000000000..66346a6c2ceb
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_02_chunk_pipelines.rs
@@ -0,0 +1,131 @@
+mod common;
+
+use common::{collect_chunks, items, shared};
+use ingestion_framework::{chunk_step, dlq, new_chunk_pipeline, ok, step};
+
+#[tokio::test]
+async fn chunk_pipelines_process_multiple_items_at_once() {
+ let calls = shared(0);
+ let c = calls.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("chunkDoubleStep", move |items: Vec<i64>| {
+ *c.lock().unwrap() += 1;
+ async move { Ok(items.into_iter().map(|n| ok(n * 2)).collect()) }
+ }))
+ .build();
+ pipeline.feed(items([1, 2, 3, 4, 5]));
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(*calls.lock().unwrap(), 1);
+ let v: Vec<_> = results.iter().map(|r| *r.value().unwrap()).collect();
+ assert_eq!(v, [2, 4, 6, 8, 10]);
+}
+
+#[tokio::test]
+async fn feed_accepts_items_and_next_returns_results_then_null() {
+ let pipeline = new_chunk_pipeline::<String, ()>()
+ .pipe_chunk(chunk_step(
+ "uppercaseStep",
+ |items: Vec<String>| async move {
+ Ok(items.into_iter().map(|s| ok(s.to_uppercase())).collect())
+ },
+ ))
+ .build();
+ pipeline.feed(items(["a", "b", "c"].map(String::from)));
+ let v: Vec<_> = pipeline
+ .next()
+ .await
+ .unwrap()
+ .unwrap()
+ .into_iter()
+ .map(|r| r.result.into_value().unwrap())
+ .collect();
+ assert_eq!(v, ["A", "B", "C"]);
+ assert!(pipeline.next().await.unwrap().is_none());
+}
+
+#[tokio::test]
+async fn chunk_steps_must_return_same_number_of_results_as_inputs() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("validChunkStep", |items: Vec<i64>| async move {
+ Ok(items.into_iter().map(|n| ok(n * 2)).collect())
+ }))
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ assert_eq!(pipeline.next().await.unwrap().unwrap().len(), 3);
+}
+
+#[tokio::test]
+async fn mismatched_result_count_throws_an_error() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("badChunkStep", |items: Vec<i64>| async move {
+ Ok(vec![ok(items[0])])
+ }))
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ let err = pipeline.next().await.unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("different number of results than input values"),
+ "{err}"
+ );
+}
+
+#[tokio::test]
+async fn chunk_steps_receive_only_ok_items_from_previous_steps() {
+ let received = shared(Vec::<i64>::new());
+ let r = received.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("filterStep", |items: Vec<i64>| async move {
+ Ok(items
+ .into_iter()
+ .map(|n| {
+ if n > 3 {
+ dlq(format!("Number {n} is too large"), None)
+ } else {
+ ok(n)
+ }
+ })
+ .collect())
+ }))
+ .pipe_chunk(chunk_step("processStep", move |items: Vec<i64>| {
+ r.lock().unwrap().extend(items.iter().copied());
+ async move { Ok(items.into_iter().map(|n| ok(n * 10)).collect()) }
+ }))
+ .build();
+ pipeline.feed(items([1, 2, 3, 4, 5]));
+ pipeline.next().await.unwrap();
+ assert_eq!(*received.lock().unwrap(), vec![1, 2, 3]);
+}
+
+#[tokio::test]
+async fn two_batches_emerge_as_a_single_chunk() {
+ let lengths = shared(Vec::<usize>::new());
+ let l = lengths.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("recordingStep", move |items: Vec<i64>| {
+ l.lock().unwrap().push(items.len());
+ async move { Ok(items.into_iter().map(ok).collect()) }
+ }))
+ .build();
+ pipeline.feed(items([1, 2]));
+ pipeline.feed(items([3, 4, 5]));
+ assert_eq!(pipeline.next().await.unwrap().unwrap().len(), 5);
+ assert_eq!(*lengths.lock().unwrap(), vec![5]);
+}
+
+#[tokio::test]
+async fn one_batch_spreads_across_multiple_chunks() {
+ let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()
+ .concurrently_per_group(
+ |e: &(&'static str, i64)| e.0,
+ |group| {
+ group.sequentially(|b| b.pipe(step("identityStep", |e| async move { Ok(ok(e)) })))
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(items([("alice", 1), ("bob", 2), ("alice", 3), ("bob", 4)]));
+ let chunks = collect_chunks(pipeline.as_ref()).await;
+ assert_eq!(chunks.len(), 2);
+ assert_eq!(chunks.iter().map(Vec::len).collect::<Vec<_>>(), [2, 2]);
+}
diff --git a/rust/ingestion-framework/tests/docs_03_concurrent_processing.rs b/rust/ingestion-framework/tests/docs_03_concurrent_processing.rs
new file mode 100644
index 000000000000..6fd7c3c11c32
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_03_concurrent_processing.rs
@@ -0,0 +1,106 @@
+mod common;
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use common::{consume_all_within, items, shared, sleep_ms};
+use ingestion_framework::{new_chunk_pipeline, ok, step};
+
+#[tokio::test(start_paused = true)]
+async fn concurrently_processes_items_in_parallel() {
+ let processed = shared(Vec::<i64>::new());
+ let p = processed.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently(
+ move |b| {
+ b.pipe(step("slowStep", move |n: i64| {
+ let p = p.clone();
+ async move {
+ sleep_ms(100).await;
+ p.lock().unwrap().push(n);
+ Ok(ok(n * 2))
+ }
+ }))
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ let started = tokio::time::Instant::now();
+ let values = consume_all_within(pipeline.as_ref(), 100).await;
+ assert_eq!(processed.lock().unwrap().len(), 3);
+ assert_eq!(values, [2, 4, 6]);
+ // In parallel: the three 100 ms steps take 100 ms in total, not 300 ms.
+ assert!(started.elapsed().as_millis() < 200);
+}
+
+#[tokio::test(start_paused = true)]
+async fn items_are_returned_one_by_one_as_they_complete() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently(
+ |b| {
+ b.pipe(step("slowStep", |n: i64| async move {
+ sleep_ms(10).await;
+ Ok(ok(n * 10))
+ }))
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ let chunks = common::collect_chunks(pipeline.as_ref()).await;
+ assert_eq!(chunks, vec![vec![10], vec![20], vec![30]]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn result_ordering_matches_input_order() {
+ let order = shared(Vec::<i64>::new());
+ let o = order.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently(
+ move |b| {
+ b.pipe(step("variableDelayStep", move |n: i64| {
+ let o = o.clone();
+ async move {
+ sleep_ms(((4 - n) * 10) as u64).await;
+ o.lock().unwrap().push(n);
+ Ok(ok(n * 10))
+ }
+ }))
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ let values = consume_all_within(pipeline.as_ref(), 30).await;
+ assert_eq!(*order.lock().unwrap(), vec![3, 2, 1]);
+ assert_eq!(values, [10, 20, 30]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn max_concurrency_caps_how_many_items_run_at_once() {
+ let running = Arc::new(AtomicUsize::new(0));
+ let peak = Arc::new(AtomicUsize::new(0));
+ let (r, p) = (running.clone(), peak.clone());
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently(
+ move |b| {
+ b.pipe(step("trackingStep", move |n: i64| {
+ let (r, p) = (r.clone(), p.clone());
+ async move {
+ let now = r.fetch_add(1, Ordering::SeqCst) + 1;
+ p.fetch_max(now, Ordering::SeqCst);
+ sleep_ms(100).await;
+ r.fetch_sub(1, Ordering::SeqCst);
+ Ok(ok(n * 2))
+ }
+ }))
+ },
+ Some(2),
+ )
+ .build();
+ pipeline.feed(items([1, 2, 3, 4, 5]));
+ let values = consume_all_within(pipeline.as_ref(), 300).await;
+ assert_eq!(peak.load(Ordering::SeqCst), 2);
+ assert_eq!(values, [2, 4, 6, 8, 10]);
+}
diff --git a/rust/ingestion-framework/tests/docs_04_sequential_processing.rs b/rust/ingestion-framework/tests/docs_04_sequential_processing.rs
new file mode 100644
index 000000000000..611a4e76572c
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_04_sequential_processing.rs
@@ -0,0 +1,67 @@
+mod common;
+
+use common::{collect_chunks, consume_all, items, shared, sleep_ms};
+use ingestion_framework::{new_chunk_pipeline, ok, step};
+
+#[tokio::test(start_paused = true)]
+async fn sequentially_processes_items_one_at_a_time() {
+ let order = shared(Vec::<i64>::new());
+ let o = order.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .sequentially(move |b| {
+ b.pipe(step("orderTrackingStep", move |n: i64| {
+ let o = o.clone();
+ async move {
+ sleep_ms(10).await;
+ o.lock().unwrap().push(n);
+ Ok(ok(n))
+ }
+ }))
+ })
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ let started = tokio::time::Instant::now();
+ let values = consume_all(pipeline.as_ref()).await;
+ assert_eq!(*order.lock().unwrap(), vec![1, 2, 3]);
+ assert_eq!(values, [1, 2, 3]);
+ assert!(started.elapsed().as_millis() >= 30);
+}
+
+#[tokio::test(start_paused = true)]
+async fn all_items_are_returned_together_after_sequential_processing() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .sequentially(|b| {
+ b.pipe(step("slowStep", |n: i64| async move {
+ sleep_ms(10).await;
+ Ok(ok(n * 10))
+ }))
+ })
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ assert_eq!(
+ collect_chunks(pipeline.as_ref()).await,
+ vec![vec![10, 20, 30]]
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn result_ordering_matches_input_order() {
+ let order = shared(Vec::<i64>::new());
+ let o = order.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .sequentially(move |b| {
+ b.pipe(step("trackingStep", move |n: i64| {
+ let o = o.clone();
+ async move {
+ sleep_ms(10).await;
+ o.lock().unwrap().push(n);
+ Ok(ok(n * 10))
+ }
+ }))
+ })
+ .build();
+ pipeline.feed(items([3, 1, 2]));
+ let values = consume_all(pipeline.as_ref()).await;
+ assert_eq!(*order.lock().unwrap(), vec![3, 1, 2]);
+ assert_eq!(values, [30, 10, 20]);
+}
diff --git a/rust/ingestion-framework/tests/docs_05_grouping.rs b/rust/ingestion-framework/tests/docs_05_grouping.rs
new file mode 100644
index 000000000000..454970ec4636
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_05_grouping.rs
@@ -0,0 +1,161 @@
+mod common;
+
+use std::collections::HashMap;
+
+use common::{collect_chunks, consume_all_within, items, shared, sleep_ms};
+use ingestion_framework::{new_chunk_pipeline, ok, step};
+
+type Event = (&'static str, i64);
+
+fn events(list: &[Event]) -> Vec<ingestion_framework::OkResultWithContext<Event, ()>> {
+ items(list.iter().copied())
+}
+
+#[tokio::test(start_paused = true)]
+async fn concurrently_per_group_partitions_items_by_a_key_function() {
+ let groups = shared(HashMap::<&'static str, Vec<i64>>::new());
+ let g = groups.clone();
+ let pipeline = new_chunk_pipeline::<Event, ()>()
+ .concurrently_per_group(
+ |e: &Event| e.0,
+ move |group| {
+ group.sequentially(move |b| {
+ b.pipe(step("processEventStep", move |e: Event| {
+ let g = g.clone();
+ async move {
+ g.lock().unwrap().entry(e.0).or_default().push(e.1);
+ sleep_ms(5).await;
+ Ok(ok(e))
+ }
+ }))
+ })
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(events(&[
+ ("alice", 1),
+ ("bob", 2),
+ ("alice", 3),
+ ("bob", 4),
+ ]));
+ consume_all_within(pipeline.as_ref(), 20).await;
+ let groups = groups.lock().unwrap();
+ assert_eq!(groups["alice"], vec![1, 3]);
+ assert_eq!(groups["bob"], vec![2, 4]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn items_in_the_same_group_are_processed_sequentially() {
+ let order = shared(Vec::<i64>::new());
+ let o = order.clone();
+ let pipeline = new_chunk_pipeline::<Event, ()>()
+ .concurrently_per_group(
+ |e: &Event| e.0,
+ move |group| {
+ group.sequentially(move |b| {
+ b.pipe(step("processEventStep", move |e: Event| {
+ let o = o.clone();
+ async move {
+ o.lock().unwrap().push(e.1);
+ sleep_ms(10).await;
+ Ok(ok(e))
+ }
+ }))
+ })
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(events(&[("alice", 1), ("alice", 2), ("alice", 3)]));
+ consume_all_within(pipeline.as_ref(), 30).await;
+ assert_eq!(*order.lock().unwrap(), vec![1, 2, 3]);
+}
+
+fn delay_for(user: &str) -> u64 {
+ match user {
+ "alice" => 30,
+ "bob" => 10,
+ "charlie" => 20,
+ _ => 0,
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn items_in_different_groups_can_process_concurrently() {
+ let pipeline = new_chunk_pipeline::<Event, ()>()
+ .concurrently_per_group(
+ |e: &Event| e.0,
+ |group| {
+ group.sequentially(|b| {
+ b.pipe(step("variableDelayStep", |e: Event| async move {
+ sleep_ms(delay_for(e.0)).await;
+ Ok(ok(e))
+ }))
+ })
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(events(&[
+ ("alice", 1),
+ ("bob", 2),
+ ("charlie", 3),
+ ("alice", 4),
+ ("bob", 5),
+ ("charlie", 6),
+ ]));
+ let chunks: Vec<Vec<String>> = collect_chunks(pipeline.as_ref())
+ .await
+ .into_iter()
+ .map(|c| c.into_iter().map(|(u, id)| format!("{u}:{id}")).collect())
+ .collect();
+ assert_eq!(
+ chunks,
+ vec![
+ vec!["bob:2", "bob:5"],
+ vec!["charlie:3", "charlie:6"],
+ vec!["alice:1", "alice:4"],
+ ]
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn complex_group_keys_with_token_and_distinct_id() {
+ type IngestionEvent = (&'static str, &'static str, &'static str);
+ let groups = shared(HashMap::<String, Vec<&'static str>>::new());
+ let g = groups.clone();
+ let pipeline = new_chunk_pipeline::<IngestionEvent, ()>()
+ .concurrently_per_group(
+ |e: &IngestionEvent| format!("{}:{}", e.0, e.1),
+ move |group| {
+ group.sequentially(move |b| {
+ b.pipe(step("processEventStep", move |e: IngestionEvent| {
+ let g = g.clone();
+ async move {
+ g.lock()
+ .unwrap()
+ .entry(format!("{}:{}", e.0, e.1))
+ .or_default()
+ .push(e.2);
+ sleep_ms(5).await;
+ Ok(ok(e))
+ }
+ }))
+ })
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(items([
+ ("token-A", "user-1", "login"),
+ ("token-B", "user-1", "signup"),
+ ("token-A", "user-2", "pageview"),
+ ("token-A", "user-1", "purchase"),
+ ]));
+ consume_all_within(pipeline.as_ref(), 10).await;
+ let groups = groups.lock().unwrap();
+ assert_eq!(groups["token-A:user-1"], vec!["login", "purchase"]);
+ assert_eq!(groups["token-B:user-1"], vec!["signup"]);
+ assert_eq!(groups["token-A:user-2"], vec!["pageview"]);
+}
diff --git a/rust/ingestion-framework/tests/docs_06_gathering.rs b/rust/ingestion-framework/tests/docs_06_gathering.rs
new file mode 100644
index 000000000000..f960767ca759
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_06_gathering.rs
@@ -0,0 +1,81 @@
+mod common;
+
+use common::{collect_chunks, items, sleep_ms};
+use ingestion_framework::{new_chunk_pipeline, ok, step, ProcessingStep};
+
+fn variable_delay_numbers() -> ProcessingStep<i64, i64> {
+ step("variableDelayStep", |n: i64| async move {
+ sleep_ms(match n {
+ 1 => 30,
+ 2 => 10,
+ _ => 20,
+ })
+ .await;
+ Ok(ok(n * 10))
+ })
+}
+
+#[tokio::test(start_paused = true)]
+async fn gather_after_concurrently_collects_items_into_one_chunk() {
+ let streaming = new_chunk_pipeline::<i64, ()>()
+ .concurrently(|b| b.pipe(variable_delay_numbers()), None)
+ .build();
+ streaming.feed(items([1, 2, 3]));
+ assert_eq!(
+ collect_chunks(streaming.as_ref()).await,
+ vec![vec![10], vec![20], vec![30]]
+ );
+
+ let gathering = new_chunk_pipeline::<i64, ()>()
+ .concurrently(|b| b.pipe(variable_delay_numbers()), None)
+ .gather()
+ .build();
+ gathering.feed(items([1, 2, 3]));
+ assert_eq!(
+ collect_chunks(gathering.as_ref()).await,
+ vec![vec![10, 20, 30]]
+ );
+}
+
+type Event = (&'static str, i64);
+
+fn variable_delay_events() -> ProcessingStep<Event, Event> {
+ step("variableDelayStep", |e: Event| async move {
+ sleep_ms(if e.0 == "alice" { 30 } else { 10 }).await;
+ Ok(ok(e))
+ })
+}
+
+#[tokio::test(start_paused = true)]
+async fn gather_after_concurrently_per_group_collects_groups_into_one_chunk() {
+ let events = [("alice", 1), ("bob", 2), ("alice", 3), ("bob", 4)];
+ let streaming = new_chunk_pipeline::<Event, ()>()
+ .concurrently_per_group(
+ |e: &Event| e.0,
+ |g| g.sequentially(|b| b.pipe(variable_delay_events())),
+ None,
+ )
+ .build();
+ streaming.feed(items(events));
+ assert_eq!(
+ collect_chunks(streaming.as_ref()).await,
+ vec![
+ vec![("bob", 2), ("bob", 4)],
+ vec![("alice", 1), ("alice", 3)]
+ ]
+ );
+
+ let gathering = new_chunk_pipeline::<Event, ()>()
+ .concurrently_per_group(
+ |e: &Event| e.0,
+ |g| g.sequentially(|b| b.pipe(variable_delay_events())),
+ None,
+ )
+ .gather()
+ .build();
+ gathering.feed(items(events));
+ assert_eq!(
+ collect_chunks(gathering.as_ref()).await,
+ vec![vec![("bob", 2), ("bob", 4), ("alice", 1), ("alice", 3)]]
+ );
+}
diff --git a/rust/ingestion-framework/tests/docs_07_result_handling.rs b/rust/ingestion-framework/tests/docs_07_result_handling.rs
new file mode 100644
index 000000000000..7185a522dd09
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_07_result_handling.rs
@@ -0,0 +1,242 @@
+mod common;
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use common::{test_message, MockOutputs, MsgCtx};
+use ingestion_framework::{
+ chunk_step, create_ok_context, dlq, drop, new_chunk_pipeline, ok, redirect, redirect_with,
+ PipelineConfig, PipelineResultType, PromiseScheduler, SideEffectHandlingConfig, DLQ_OUTPUT,
+};
+
+fn config(outputs: &MockOutputs, scheduler: &PromiseScheduler) -> PipelineConfig {
+ PipelineConfig {
+ outputs: Arc::new(outputs.clone()),
+ scheduler: Arc::new(scheduler.clone()),
+ reject_on_dlq_failure: false,
+ }
+}
+
+const AWAIT: SideEffectHandlingConfig = SideEffectHandlingConfig {
+ await_side_effects: true,
+};
+
+#[tokio::test]
+async fn dlq_sends_failed_items_to_dead_letter_topic_with_error_details() {
+ let outputs = MockOutputs::default();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()
+ .pipe_chunk(chunk_step(
+ "validationStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|d| {
+ if d == "invalid" {
+ dlq(
+ "Validation failed",
+ Some(anyhow::anyhow!("Invalid data format")),
+ )
+ } else {
+ ok(d)
+ }
+ })
+ .collect())
+ },
+ ))
+ .message_aware(|b| b)
+ .handle_results(config(&outputs, &scheduler))
+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)
+ .build();
+ let message = test_message(None);
+ pipeline.feed(vec![create_ok_context(
+ "invalid",
+ MsgCtx {
+ message: message.clone(),
+ },
+ )]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(results.len(), 1);
+ assert_eq!(results[0].result.result_type(), PipelineResultType::Dlq);
+ let calls = outputs.calls();
+ assert_eq!(calls.len(), 1);
+ let (output, produced) = &calls[0];
+ assert_eq!(output, DLQ_OUTPUT);
+ assert_eq!(produced.value, message.value);
+ assert_eq!(produced.key, message.key);
+ assert_eq!(
+ produced.headers.get("dlq_reason"),
+ Some("Invalid data format")
+ );
+ assert_eq!(produced.headers.get("dlq_step"), Some("validationStep"));
+ assert!(produced.headers.get("dlq_timestamp").is_some());
+}
+
+#[tokio::test]
+async fn drop_discards_items_without_sending_to_kafka() {
+ let outputs = MockOutputs::default();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()
+ .pipe_chunk(chunk_step(
+ "filterStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|t| {
+ if t.starts_with("internal-") {
+ drop("Internal event filtered")
+ } else {
+ ok(t)
+ }
+ })
+ .collect())
+ },
+ ))
+ .message_aware(|b| b)
+ .handle_results(config(&outputs, &scheduler))
+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)
+ .build();
+ pipeline.feed(vec![create_ok_context(
+ "internal-heartbeat",
+ MsgCtx {
+ message: test_message(None),
+ },
+ )]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(results.len(), 1);
+ assert_eq!(results[0].result.result_type(), PipelineResultType::Drop);
+ assert!(outputs.calls().is_empty());
+}
+
+#[tokio::test]
+async fn redirect_sends_items_to_a_named_output_with_optional_key_preservation() {
+ let outputs = MockOutputs::default();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()
+ .pipe_chunk(chunk_step(
+ "routingStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|p| match p {
+ "high" => {
+ redirect_with("High priority routing", "high_priority", true, true)
+ }
+ "broadcast" => redirect_with("Broadcast routing", "broadcast", false, true),
+ _ => ok(p),
+ })
+ .collect())
+ },
+ ))
+ .message_aware(|b| b)
+ .handle_results(config(&outputs, &scheduler))
+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)
+ .build();
+ let message1 = test_message(Some(b"my-partition-key"));
+ let message2 = test_message(Some(b"another-key"));
+ pipeline.feed(vec![
+ create_ok_context(
+ "high",
+ MsgCtx {
+ message: message1.clone(),
+ },
+ ),
+ create_ok_context(
+ "broadcast",
+ MsgCtx {
+ message: message2.clone(),
+ },
+ ),
+ ]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(results.len(), 2);
+ assert!(results
+ .iter()
+ .all(|r| r.result.result_type() == PipelineResultType::Redirect));
+
+ // Node produces in item order, so the call order is the feed order.
+ let calls = outputs.calls();
+ assert_eq!(calls.len(), 2);
+ let (high_out, high) = &calls[0];
+ let (broadcast_out, broadcast) = &calls[1];
+ assert_eq!(high_out, "high_priority");
+ assert_eq!(high.value, message1.value);
+ assert_eq!(high.key.as_deref(), Some(&b"my-partition-key"[..]));
+ assert_eq!(high.headers.get("redirect-step"), Some("routingStep"));
+ assert!(high.headers.get("redirect-timestamp").is_some());
+ assert_eq!(broadcast_out, "broadcast");
+ assert_eq!(broadcast.value, message2.value);
+ assert_eq!(broadcast.key, None);
+ assert_eq!(broadcast.headers.get("redirect-step"), Some("routingStep"));
+ assert!(broadcast.headers.get("redirect-timestamp").is_some());
+}
+
+#[tokio::test]
+async fn redirect_with_await_ack_true_blocks_on_the_produce_ack() {
+ let outputs = MockOutputs::default();
+ outputs.close_gate();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()
+ .pipe_chunk(chunk_step(
+ "routingStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|_| redirect::<&str>("Overflow", "overflow"))
+ .collect())
+ },
+ ))
+ .message_aware(|b| b)
+ .handle_results(config(&outputs, &scheduler))
+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)
+ .build();
+ pipeline.feed(vec![create_ok_context(
+ "x",
+ MsgCtx {
+ message: test_message(None),
+ },
+ )]);
+ let mut next = Box::pin(pipeline.next());
+ assert!(tokio::time::timeout(Duration::from_millis(50), &mut next)
+ .await
+ .is_err());
+ outputs.open_gate();
+ let results = next.await.unwrap().unwrap();
+ assert_eq!(
+ results[0].result.result_type(),
+ PipelineResultType::Redirect
+ );
+}
+
+#[tokio::test]
+async fn redirect_with_await_ack_false_does_not_block_on_the_produce_ack() {
+ let outputs = MockOutputs::default();
+ outputs.close_gate();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()
+ .pipe_chunk(chunk_step(
+ "routingStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|_| redirect_with::<&str>("Overflow", "overflow", true, false))
+ .collect())
+ },
+ ))
+ .message_aware(|b| b)
+ .handle_results(config(&outputs, &scheduler))
+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)
+ .build();
+ pipeline.feed(vec![create_ok_context(
+ "x",
+ MsgCtx {
+ message: test_message(None),
+ },
+ )]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(
+ results[0].result.result_type(),
+ PipelineResultType::Redirect
+ );
+ assert_eq!(outputs.calls().len(), 1);
+}
diff --git a/rust/ingestion-framework/tests/docs_08_side_effects.rs b/rust/ingestion-framework/tests/docs_08_side_effects.rs
new file mode 100644
index 000000000000..04f728787404
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_08_side_effects.rs
@@ -0,0 +1,181 @@
+mod common;
+
+use std::sync::Arc;
+
+use common::{items, shared, sleep_ms};
+use ingestion_framework::{
+ create_ok_context, new_chunk_pipeline, new_pipeline, ok, step, ChunkProcessingStep,
+ PromiseScheduler, SideEffect, SideEffectHandlingConfig,
+};
+
+async fn wait_all(side_effects: Vec<SideEffect>) {
+ for s in side_effects {
+ s.wait().await.unwrap();
+ }
+}
+
+#[tokio::test]
+async fn steps_can_add_side_effects_to_context() {
+ let log = shared(Vec::<String>::new());
+ let l = log.clone();
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe(step("processWithLoggingStep", move |input: String| {
+ let l = l.clone();
+ async move {
+ let result = input.to_uppercase();
+ let message = format!("Processed: {input} -> {result}");
+ let effect = SideEffect::spawn(async move {
+ l.lock().unwrap().push(message);
+ Ok(())
+ });
+ Ok(ok((result, true)).with_side_effect(effect))
+ }
+ }))
+ .build();
+ let mut r = pipeline
+ .process(create_ok_context("hello".into(), ()))
+ .await
+ .unwrap();
+ assert!(r.is_ok());
+ assert_eq!(r.context.side_effects.len(), 1);
+ // The side effect started when it was created, as a Node promise does.
+ assert!(log
+ .lock()
+ .unwrap()
+ .contains(&"Processed: hello -> HELLO".to_string()));
+ wait_all(std::mem::take(&mut r.context.side_effects)).await;
+ assert!(log
+ .lock()
+ .unwrap()
+ .contains(&"Processed: hello -> HELLO".to_string()));
+}
+
+#[tokio::test]
+async fn side_effects_accumulate_through_the_pipeline() {
+ let audit = shared(Vec::<&'static str>::new());
+ let audited = |name: &'static str, f: fn(i64) -> i64| {
+ let a = audit.clone();
+ step(name, move |v: i64| {
+ let a = a.clone();
+ async move {
+ let effect = SideEffect::spawn(async move {
+ a.lock().unwrap().push(match name {
+ "step1" => "step1 completed",
+ "step2" => "step2 completed",
+ _ => "step3 completed",
+ });
+ Ok(())
+ });
+ Ok(ok(f(v)).with_side_effect(effect))
+ }
+ })
+ };
+ let pipeline = new_pipeline::<i64, ()>()
+ .pipe(audited("step1", |v| v * 2))
+ .pipe(audited("step2", |v| v + 10))
+ .pipe(audited("step3", |v| v * v))
+ .build();
+ let mut r = pipeline.process(create_ok_context(5, ())).await.unwrap();
+ assert_eq!(r.context.side_effects.len(), 3);
+ wait_all(std::mem::take(&mut r.context.side_effects)).await;
+ let audit = audit.lock().unwrap();
+ assert_eq!(audit.len(), 3);
+ for expected in ["step1 completed", "step2 completed", "step3 completed"] {
+ assert!(audit.contains(&expected));
+ }
+}
+
+#[tokio::test]
+async fn a_step_can_add_multiple_side_effects() {
+ let sinks = shared((0, 0, 0));
+ let s = sinks.clone();
+ let pipeline = new_pipeline::<(&'static str, &'static str), ()>()
+ .pipe(step(
+ "processEventStep",
+ move |e: (&'static str, &'static str)| {
+ let (s1, s2, s3) = (s.clone(), s.clone(), s.clone());
+ async move {
+ Ok(ok((e, true))
+ .with_side_effect(SideEffect::spawn(async move {
+ s1.lock().unwrap().0 += 1;
+ Ok(())
+ }))
+ .with_side_effect(SideEffect::spawn(async move {
+ s2.lock().unwrap().1 += 1;
+ Ok(())
+ }))
+ .with_side_effect(SideEffect::spawn(async move {
+ s3.lock().unwrap().2 += 1;
+ Ok(())
+ })))
+ }
+ },
+ ))
+ .build();
+ let mut r = pipeline
+ .process(create_ok_context(("purchase", "user-1"), ()))
+ .await
+ .unwrap();
+ assert_eq!(r.context.side_effects.len(), 3);
+ wait_all(std::mem::take(&mut r.context.side_effects)).await;
+ assert_eq!(*sinks.lock().unwrap(), (1, 1, 1));
+}
+
+fn delayed_effect_step(
+ completed: Arc<std::sync::Mutex<Vec<&'static str>>>,
+ delay: u64,
+) -> ChunkProcessingStep<&'static str, &'static str> {
+ ChunkProcessingStep::per_item(step("processStep", move |id: &'static str| {
+ let completed = completed.clone();
+ async move {
+ let effect = SideEffect::spawn(async move {
+ sleep_ms(delay).await;
+ completed.lock().unwrap().push(id);
+ Ok(())
+ });
+ Ok(ok(id).with_side_effect(effect))
+ }
+ }))
+}
+
+#[tokio::test(start_paused = true)]
+async fn handle_side_effects_with_await_true_blocks_until_completion() {
+ let completed = shared(Vec::new());
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<&'static str, ()>()
+ .pipe_chunk(delayed_effect_step(completed.clone(), 10))
+ .handle_side_effects(
+ Arc::new(scheduler),
+ SideEffectHandlingConfig {
+ await_side_effects: true,
+ },
+ )
+ .build();
+ pipeline.feed(items(["a", "b", "c"]));
+ pipeline.next().await.unwrap();
+ let mut done = completed.lock().unwrap().clone();
+ done.sort();
+ assert_eq!(done, ["a", "b", "c"]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn handle_side_effects_with_await_false_schedules_for_background_execution() {
+ let completed = shared(Vec::new());
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<&'static str, ()>()
+ .pipe_chunk(delayed_effect_step(completed.clone(), 50))
+ .handle_side_effects(
+ Arc::new(scheduler.clone()),
+ SideEffectHandlingConfig {
+ await_side_effects: false,
+ },
+ )
+ .build();
+ pipeline.feed(items(["x", "y"]));
+ pipeline.next().await.unwrap();
+ assert!(completed.lock().unwrap().is_empty());
+ scheduler.wait_for_all().await.unwrap();
+ let mut done = completed.lock().unwrap().clone();
+ done.sort();
+ assert_eq!(done, ["x", "y"]);
+}
diff --git a/rust/ingestion-framework/tests/docs_09_ingestion_warnings.rs b/rust/ingestion-framework/tests/docs_09_ingestion_warnings.rs
new file mode 100644
index 000000000000..5ba6a633d016
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_09_ingestion_warnings.rs
@@ -0,0 +1,311 @@
+mod common;
+
+use std::sync::Arc;
+
+use common::{shared, MockOutputs, TeamCtx};
+use ingestion_framework::warning_handling::OutputsWarningEmitter;
+use ingestion_framework::{
+ chunk_step, create_ok_context, new_chunk_pipeline, ok, PipelineWarning, PromiseScheduler,
+ SideEffect, SideEffectHandlingConfig, INGESTION_WARNINGS_OUTPUT,
+};
+use serde_json::json;
+
+fn team() -> TeamCtx {
+ TeamCtx { team_id: 2 }
+}
+
+#[tokio::test]
+async fn steps_can_add_warnings_to_results() {
+ let pipeline = new_chunk_pipeline::<(&'static str, Option<()>), TeamCtx>()
+ .pipe_chunk(chunk_step(
+ "validationStep",
+ |items: Vec<(&'static str, Option<()>)>| async move {
+ Ok(items
+ .into_iter()
+ .map(|item| {
+ let mut r = ok(item);
+ if item.1.is_none() {
+ r = r.with_warning(PipelineWarning::new(
+ "schema_validation_failed",
+ json!({"eventName": item.0}),
+ ));
+ }
+ r
+ })
+ .collect())
+ },
+ ))
+ .build();
+ pipeline.feed(vec![create_ok_context(("pageview", None), team())]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(results.len(), 1);
+ assert!(results[0].is_ok());
+ assert_eq!(
+ results[0].context.warnings,
+ vec![PipelineWarning::new(
+ "schema_validation_failed",
+ json!({"eventName": "pageview"})
+ )]
+ );
+}
+
+#[tokio::test]
+async fn warnings_accumulate_through_the_pipeline() {
+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()
+ .pipe_chunk(chunk_step(
+ "timestampCheckStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|n| {
+ ok(n).with_warning(PipelineWarning::new(
+ "ignored_invalid_timestamp",
+ json!({"eventName": n}),
+ ))
+ })
+ .collect())
+ },
+ ))
+ .pipe_chunk(chunk_step(
+ "propertiesCheckStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|n| {
+ ok(n).with_warning(PipelineWarning::new(
+ "schema_validation_failed",
+ json!({"eventName": n}),
+ ))
+ })
+ .collect())
+ },
+ ))
+ .build();
+ pipeline.feed(vec![create_ok_context("click", team())]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ let types: Vec<_> = results[0]
+ .context
+ .warnings
+ .iter()
+ .map(|w| w.warning_type.as_str())
+ .collect();
+ assert_eq!(
+ types,
+ ["ignored_invalid_timestamp", "schema_validation_failed"]
+ );
+}
+
+#[tokio::test]
+async fn a_step_can_add_multiple_warnings() {
+ struct Event {
+ timestamp: &'static str,
+ group_key: String,
+ process_person: serde_json::Value,
+ }
+ let pipeline = new_chunk_pipeline::<Event, TeamCtx>()
+ .pipe_chunk(chunk_step(
+ "comprehensiveValidationStep",
+ |items: Vec<Event>| async move {
+ Ok(items
+ .into_iter()
+ .map(|item| {
+ let mut warnings = Vec::new();
+ if item.group_key.len() > 400 {
+ warnings.push(PipelineWarning::new(
+ "group_key_too_long",
+ json!({"groupKey": &item.group_key[..20], "max": 400}),
+ ));
+ }
+ if chrono::DateTime::parse_from_rfc3339(item.timestamp).is_err() {
+ warnings.push(PipelineWarning::new(
+ "ignored_invalid_timestamp",
+ json!({"value": item.timestamp}),
+ ));
+ }
+ if !item.process_person.is_null() && !item.process_person.is_boolean() {
+ warnings.push(PipelineWarning::new(
+ "invalid_process_person_profile",
+ json!({"received": "string"}),
+ ));
+ }
+ ok(item).with_warnings(warnings)
+ })
+ .collect())
+ },
+ ))
+ .build();
+ pipeline.feed(vec![create_ok_context(
+ Event {
+ timestamp: "not-a-date",
+ group_key: "g".repeat(500),
+ process_person: json!("yes"),
+ },
+ team(),
+ )]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ let types: Vec<_> = results[0]
+ .context
+ .warnings
+ .iter()
+ .map(|w| w.warning_type.clone())
+ .collect();
+ assert_eq!(types.len(), 3);
+ for t in [
+ "group_key_too_long",
+ "ignored_invalid_timestamp",
+ "invalid_process_person_profile",
+ ] {
+ assert!(types.contains(&t.to_string()));
+ }
+}
+
+#[tokio::test]
+async fn handle_ingestion_warnings_converts_warnings_to_side_effects() {
+ let outputs = MockOutputs::default();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()
+ .pipe_chunk(chunk_step(
+ "warningStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|n| {
+ ok(n).with_warning(PipelineWarning::new(
+ "client_ingestion_warning",
+ json!({"eventName": n}),
+ ))
+ })
+ .collect())
+ },
+ ))
+ .team_aware(|b| b)
+ .handle_ingestion_warnings(Arc::new(OutputsWarningEmitter {
+ outputs: Arc::new(outputs.clone()),
+ }))
+ .handle_side_effects(
+ Arc::new(scheduler),
+ SideEffectHandlingConfig {
+ await_side_effects: true,
+ },
+ )
+ .build();
+ pipeline.feed(vec![create_ok_context("pageview", TeamCtx { team_id: 42 })]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert!(results[0].context.warnings.is_empty());
+ let calls = outputs.calls();
+ assert_eq!(calls.len(), 1);
+ assert_eq!(calls[0].0, INGESTION_WARNINGS_OUTPUT);
+ let body: serde_json::Value =
+ serde_json::from_slice(calls[0].1.value.as_ref().unwrap()).unwrap();
+ assert_eq!(body["team_id"], 42);
+ assert_eq!(body["type"], "client_ingestion_warning");
+}
+
+#[tokio::test]
+async fn handle_ingestion_warnings_preserves_existing_side_effects() {
+ let outputs = MockOutputs::default();
+ let log = shared(Vec::<String>::new());
+ let l = log.clone();
+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()
+ .pipe_chunk(chunk_step(
+ "stepWithBothSideEffectsAndWarnings",
+ move |items: Vec<&'static str>| {
+ let l = l.clone();
+ async move {
+ Ok(items
+ .into_iter()
+ .map(|n| {
+ let l = l.clone();
+ ok(n)
+ .with_side_effect(SideEffect::spawn(async move {
+ l.lock().unwrap().push(format!("processed: {n}"));
+ Ok(())
+ }))
+ .with_warning(PipelineWarning::new(
+ "client_ingestion_warning",
+ json!({}),
+ ))
+ })
+ .collect())
+ }
+ },
+ ))
+ .team_aware(|b| b)
+ .handle_ingestion_warnings(Arc::new(OutputsWarningEmitter {
+ outputs: Arc::new(outputs.clone()),
+ }))
+ .handle_side_effects(
+ Arc::new(PromiseScheduler::new()),
+ SideEffectHandlingConfig {
+ await_side_effects: true,
+ },
+ )
+ .build();
+ pipeline.feed(vec![create_ok_context("click", team())]);
+ pipeline.next().await.unwrap();
+ assert!(log
+ .lock()
+ .unwrap()
+ .contains(&"processed: click".to_string()));
+ assert_eq!(outputs.calls().len(), 1);
+}
+
+#[tokio::test]
+async fn warnings_can_include_a_key_for_debouncing() {
+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()
+ .pipe_chunk(chunk_step(
+ "userWarningStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|d| {
+ let mut w = PipelineWarning::new(
+ "cannot_merge_already_identified",
+ json!({"distinctId": d}),
+ );
+ w.key = Some(d.to_string());
+ ok(d).with_warning(w)
+ })
+ .collect())
+ },
+ ))
+ .build();
+ pipeline.feed(vec![create_ok_context("user-123", team())]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ let mut expected = PipelineWarning::new(
+ "cannot_merge_already_identified",
+ json!({"distinctId": "user-123"}),
+ );
+ expected.key = Some("user-123".into());
+ assert_eq!(results[0].context.warnings, vec![expected]);
+}
+
+#[tokio::test]
+async fn always_send_bypasses_debouncing_for_critical_warnings() {
+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()
+ .pipe_chunk(chunk_step(
+ "criticalWarningStep",
+ |items: Vec<&'static str>| async move {
+ Ok(items
+ .into_iter()
+ .map(|n| {
+ let mut w =
+ PipelineWarning::new("message_size_too_large", json!({"eventName": n}));
+ w.key = Some("quota".into());
+ w.always_send = true;
+ ok(n).with_warning(w)
+ })
+ .collect())
+ },
+ ))
+ .build();
+ pipeline.feed(vec![create_ok_context("important_event", team())]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ let mut expected = PipelineWarning::new(
+ "message_size_too_large",
+ json!({"eventName": "important_event"}),
+ );
+ expected.key = Some("quota".into());
+ expected.always_send = true;
+ assert_eq!(results[0].context.warnings, vec![expected]);
+}
diff --git a/rust/ingestion-framework/tests/docs_10_branching.rs b/rust/ingestion-framework/tests/docs_10_branching.rs
new file mode 100644
index 000000000000..42b77209f7de
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_10_branching.rs
@@ -0,0 +1,112 @@
+//! Branch names are strings, so only the run-time DLQ path exists.
+
+mod common;
+
+use common::shared;
+use ingestion_framework::{create_ok_context, new_pipeline, ok, step, PipelineResultType};
+
+type Event = (&'static str, &'static str);
+
+#[tokio::test]
+async fn branching_routes_items_to_different_sub_pipelines() {
+ let calls = shared(Vec::<&'static str>::new());
+ let (c1, c2, c3) = (calls.clone(), calls.clone(), calls.clone());
+ let pipeline = new_pipeline::<Event, ()>()
+ .branching(
+ |e: &Event| e.0.to_string(),
+ move |b| {
+ b.branch("pageview", move |s| {
+ s.pipe(step("pageviewStep", move |e: Event| {
+ c1.lock().unwrap().push("pageview");
+ async move { Ok(ok(format!("PAGE: {}", e.1))) }
+ }))
+ })
+ .branch("click", move |s| {
+ s.pipe(step("clickStep", move |e: Event| {
+ c2.lock().unwrap().push("click");
+ async move { Ok(ok(format!("CLICK: {}", e.1))) }
+ }))
+ })
+ .branch("custom", move |s| {
+ s.pipe(step("customStep", move |e: Event| {
+ c3.lock().unwrap().push("custom");
+ async move { Ok(ok(e.1.to_string())) }
+ }))
+ })
+ },
+ )
+ .build();
+ let page = pipeline
+ .process(create_ok_context(("pageview", "home"), ()))
+ .await
+ .unwrap();
+ let click = pipeline
+ .process(create_ok_context(("click", "button"), ()))
+ .await
+ .unwrap();
+ assert_eq!(page.value().unwrap(), "PAGE: home");
+ assert_eq!(click.value().unwrap(), "CLICK: button");
+ assert_eq!(*calls.lock().unwrap(), vec!["pageview", "click"]);
+}
+
+#[tokio::test]
+async fn unknown_branch_routes_to_dlq() {
+ let pipeline = new_pipeline::<Event, ()>()
+ .branching(
+ |e: &Event| e.0.to_string(),
+ |b| {
+ b.branch("pageview", |s| {
+ s.pipe(step("pageviewStep", |e: Event| async move { Ok(ok(e)) }))
+ })
+ },
+ )
+ .build();
+ let r = pipeline
+ .process(create_ok_context(("unknown", "test"), ()))
+ .await
+ .unwrap();
+ assert_eq!(r.result.result_type(), PipelineResultType::Dlq);
+ assert_eq!(r.result.reason(), Some("Unknown branch: unknown"));
+}
+
+#[tokio::test]
+async fn branches_must_converge_to_the_same_output_type() {
+ #[derive(Debug, PartialEq)]
+ enum Output {
+ Number(i64),
+ Str(String),
+ }
+ let pipeline = new_pipeline::<(&'static str, &'static str), ()>()
+ .branching(
+ |i: &(&'static str, &'static str)| i.0.to_string(),
+ |b| {
+ b.branch("toNumber", |s| {
+ s.pipe(step(
+ "toNumberStep",
+ |i: (&'static str, &'static str)| async move {
+ Ok(ok(Output::Number(i.1.parse().unwrap())))
+ },
+ ))
+ })
+ .branch("toString", |s| {
+ s.pipe(step(
+ "toStringStep",
+ |i: (&'static str, &'static str)| async move {
+ Ok(ok(Output::Str(i.1.to_uppercase())))
+ },
+ ))
+ })
+ },
+ )
+ .build();
+ let n = pipeline
+ .process(create_ok_context(("toNumber", "42"), ()))
+ .await
+ .unwrap();
+ let s = pipeline
+ .process(create_ok_context(("toString", "hello"), ()))
+ .await
+ .unwrap();
+ assert_eq!(n.value(), Some(&Output::Number(42)));
+ assert_eq!(s.value(), Some(&Output::Str("HELLO".into())));
+}
diff --git a/rust/ingestion-framework/tests/docs_11_retries.rs b/rust/ingestion-framework/tests/docs_11_retries.rs
new file mode 100644
index 000000000000..9cefb7ff7063
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_11_retries.rs
@@ -0,0 +1,230 @@
+mod common;
+
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+
+use common::{items, sleep_ms, within_ms};
+use ingestion_framework::{
+ chunk_step, create_ok_context, new_chunk_pipeline, new_pipeline, ok, step, PipelineError,
+ PipelineResultType, RetryOptions,
+};
+
+fn opts(tries: u32, sleep_ms: u64) -> RetryOptions {
+ RetryOptions {
+ tries: Some(tries),
+ sleep_ms: Some(sleep_ms),
+ ..Default::default()
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn retriable_errors_are_retried_until_the_step_succeeds() {
+ let attempts = Arc::new(AtomicU32::new(0));
+ let a = attempts.clone();
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe_with_retry(
+ step("flakyStep", move |v: String| {
+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;
+ async move {
+ if n < 3 {
+ return Err(PipelineError::retriable("Temporary failure"));
+ }
+ Ok(ok(v.to_uppercase()))
+ }
+ }),
+ opts(5, 100),
+ )
+ .build();
+ let r = within_ms(300, pipeline.process(create_ok_context("hello".into(), ())))
+ .await
+ .unwrap();
+ assert_eq!(attempts.load(Ordering::SeqCst), 3);
+ assert_eq!(r.value().unwrap(), "HELLO");
+}
+
+#[tokio::test(start_paused = true)]
+async fn non_retriable_errors_go_directly_to_dlq() {
+ let attempts = Arc::new(AtomicU32::new(0));
+ let a = attempts.clone();
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe_with_retry(
+ step("validationStep", move |_v: String| {
+ a.fetch_add(1, Ordering::SeqCst);
+ async move {
+ Err::<ingestion_framework::PipelineResult<String>, _>(
+ PipelineError::non_retriable("Validation failed - invalid format"),
+ )
+ }
+ }),
+ opts(5, 100),
+ )
+ .build();
+ let r = within_ms(
+ 0,
+ pipeline.process(create_ok_context("bad-data".into(), ())),
+ )
+ .await
+ .unwrap();
+ assert_eq!(attempts.load(Ordering::SeqCst), 1);
+ assert_eq!(r.result.result_type(), PipelineResultType::Dlq);
+ assert!(r.result.reason().unwrap().contains("non-retriable"));
+}
+
+#[tokio::test(start_paused = true)]
+async fn after_max_retries_the_error_is_rethrown() {
+ let attempts = Arc::new(AtomicU32::new(0));
+ let a = attempts.clone();
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe_with_retry(
+ step("alwaysFailsStep", move |_v: String| {
+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;
+ async move {
+ Err::<ingestion_framework::PipelineResult<String>, _>(PipelineError::retriable(
+ format!("Attempt {n} failed"),
+ ))
+ }
+ }),
+ opts(3, 100),
+ )
+ .build();
+ let err = within_ms(300, pipeline.process(create_ok_context("data".into(), ())))
+ .await
+ .unwrap_err();
+ assert_eq!(attempts.load(Ordering::SeqCst), 3);
+ assert_eq!(err.to_string(), "Attempt 3 failed");
+}
+
+#[tokio::test(start_paused = true)]
+async fn errors_without_is_retriable_are_retried_then_rethrown() {
+ let attempts = Arc::new(AtomicU32::new(0));
+ let a = attempts.clone();
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe_with_retry(
+ step("unexpectedErrorStep", move |_v: String| {
+ a.fetch_add(1, Ordering::SeqCst);
+ async move {
+ Err::<ingestion_framework::PipelineResult<String>, _>(anyhow::anyhow!(
+ "Unexpected internal error"
+ ))
+ }
+ }),
+ opts(3, 100),
+ )
+ .build();
+ let err = within_ms(300, pipeline.process(create_ok_context("data".into(), ())))
+ .await
+ .unwrap_err();
+ assert_eq!(attempts.load(Ordering::SeqCst), 3);
+ assert_eq!(err.to_string(), "Unexpected internal error");
+}
+
+#[tokio::test(start_paused = true)]
+async fn tries_and_sleep_ms_control_attempts_and_backoff() {
+ let attempts = Arc::new(AtomicU32::new(0));
+ let a = attempts.clone();
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe_with_retry(
+ step("slowlyRecoveringStep", move |v: String| {
+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;
+ async move {
+ if n < 4 {
+ return Err(PipelineError::retriable("Still recovering"));
+ }
+ Ok(ok(v))
+ }
+ }),
+ opts(5, 50),
+ )
+ .build();
+ let handle =
+ tokio::spawn(async move { pipeline.process(create_ok_context("data".into(), ())).await });
+ async fn advance(ms: u64) {
+ sleep_ms(ms).await;
+ for _ in 0..10 {
+ tokio::task::yield_now().await;
+ }
+ }
+ advance(0).await;
+ assert_eq!(attempts.load(Ordering::SeqCst), 1);
+ advance(50).await;
+ assert_eq!(attempts.load(Ordering::SeqCst), 2);
+ advance(100).await;
+ assert_eq!(attempts.load(Ordering::SeqCst), 3);
+ advance(200).await;
+ assert_eq!(attempts.load(Ordering::SeqCst), 4);
+ assert!(handle.await.unwrap().unwrap().is_ok());
+}
+
+#[tokio::test(start_paused = true)]
+async fn defaults_to_3_tries_and_100ms_sleep() {
+ let attempts = Arc::new(AtomicU32::new(0));
+ let a = attempts.clone();
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe_with_retry(
+ step("flakyStep", move |v: String| {
+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;
+ async move {
+ if n < 3 {
+ return Err(PipelineError::retriable("retry"));
+ }
+ Ok(ok(v))
+ }
+ }),
+ RetryOptions::default(),
+ )
+ .build();
+ let started = tokio::time::Instant::now();
+ let r = within_ms(300, pipeline.process(create_ok_context("data".into(), ())))
+ .await
+ .unwrap();
+ assert!(r.is_ok());
+ assert_eq!(attempts.load(Ordering::SeqCst), 3);
+ // 100 ms + 200 ms nominal, minus at most 5% jitter each.
+ let elapsed = started.elapsed().as_millis();
+ assert!((285..=300).contains(&elapsed), "{elapsed}");
+}
+
+#[tokio::test(start_paused = true)]
+async fn pipe_chunk_retries_the_chunk_step_until_it_succeeds() {
+ let attempts = Arc::new(AtomicU32::new(0));
+ let a = attempts.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk_with_retry(
+ chunk_step("flakyChunkStep", move |events: Vec<i64>| {
+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;
+ async move {
+ if n < 3 {
+ return Err(PipelineError::retriable("Downstream unavailable"));
+ }
+ Ok(events.into_iter().map(ok).collect())
+ }
+ }),
+ opts(5, 100),
+ )
+ .build();
+ pipeline.feed(items([1, 2]));
+ let results = within_ms(300, pipeline.next()).await.unwrap().unwrap();
+ assert_eq!(attempts.load(Ordering::SeqCst), 3);
+ assert_eq!(results.len(), 2);
+ assert!(results.iter().all(|r| r.is_ok()));
+}
+
+#[tokio::test(start_paused = true)]
+async fn pipe_chunk_sends_one_dlq_per_input_on_a_non_retriable_error() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk_with_retry(
+ chunk_step("failingChunkStep", |_events: Vec<i64>| async move {
+ Err::<Vec<ingestion_framework::PipelineResult<i64>>, _>(
+ PipelineError::non_retriable("Chunk permanently invalid"),
+ )
+ }),
+ opts(5, 100),
+ )
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ let results = within_ms(0, pipeline.next()).await.unwrap().unwrap();
+ assert_eq!(results.len(), 3);
+ assert!(results
+ .iter()
+ .all(|r| r.result.result_type() == PipelineResultType::Dlq));
+}
diff --git a/rust/ingestion-framework/tests/docs_12_filter_map.rs b/rust/ingestion-framework/tests/docs_12_filter_map.rs
new file mode 100644
index 000000000000..f6f37561a87b
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_12_filter_map.rs
@@ -0,0 +1,172 @@
+mod common;
+
+use std::sync::Arc;
+
+use common::{shared, test_message, MockOutputs};
+use ingestion_framework::warning_handling::OutputsWarningEmitter;
+use ingestion_framework::{
+ chunk_step, create_ok_context, dlq, new_chunk_pipeline, ok, redirect, HasMessage, HasTeamId,
+ KafkaMessage, OkResultWithContext, PipelineConfig, PipelineWarning, PromiseScheduler,
+ SideEffectHandlingConfig, DLQ_OUTPUT, INGESTION_WARNINGS_OUTPUT,
+};
+use serde_json::json;
+
+#[derive(Debug, Clone)]
+struct MsgCtx {
+ message: KafkaMessage,
+}
+
+#[derive(Debug, Clone)]
+struct TeamMsgCtx {
+ message: KafkaMessage,
+ team_id: Option<i64>,
+}
+
+impl From<MsgCtx> for TeamMsgCtx {
+ fn from(c: MsgCtx) -> Self {
+ Self {
+ message: c.message,
+ team_id: None,
+ }
+ }
+}
+impl HasMessage for TeamMsgCtx {
+ fn message(&self) -> &KafkaMessage {
+ &self.message
+ }
+}
+impl HasTeamId for TeamMsgCtx {
+ fn team_id(&self) -> i64 {
+ self.team_id
+ .expect("team set inside the filter_map sub-pipeline")
+ }
+}
+
+#[derive(Debug, Clone)]
+struct RawEvent {
+ team_id: i64,
+ name: &'static str,
+}
+
+#[tokio::test]
+async fn filter_map_enables_team_aware_processing_with_non_ok_passthrough() {
+ let processed = shared(Vec::<(&'static str, i64)>::new());
+ let p = processed.clone();
+ let outputs = MockOutputs::default();
+ let warning_outputs = MockOutputs::default();
+ let scheduler = PromiseScheduler::new();
+ let wo = warning_outputs.clone();
+
+ let pipeline = new_chunk_pipeline::<RawEvent, MsgCtx>()
+ .pipe_chunk(chunk_step(
+ "teamLookupStep",
+ |events: Vec<RawEvent>| async move {
+ Ok(events
+ .into_iter()
+ .map(|e| {
+ if e.team_id == 999 {
+ dlq("Team not found", Some(anyhow::anyhow!("Unknown team")))
+ } else {
+ ok((e.clone(), e.team_id))
+ }
+ })
+ .collect())
+ },
+ ))
+ .gather()
+ .filter_map(
+ |el: OkResultWithContext<(RawEvent, i64), MsgCtx>| {
+ let team_id = el.value.1;
+ OkResultWithContext {
+ value: el.value,
+ context: el.context.map_data(|c| TeamMsgCtx {
+ message: c.message,
+ team_id: Some(team_id),
+ }),
+ }
+ },
+ move |b| {
+ b.team_aware(move |b| {
+ b.pipe_chunk(chunk_step(
+ "processEventStep",
+ move |events: Vec<(RawEvent, i64)>| {
+ let p = p.clone();
+ async move {
+ Ok(events
+ .into_iter()
+ .map(|item| {
+ if item.0.name == "overflow" {
+ return redirect("Event overflow", "overflow");
+ }
+ let mut r = ok(item.clone());
+ if item.0.name == "deprecated_event" {
+ r = r.with_warning(PipelineWarning::new(
+ "event_dropped_by_transformation",
+ json!({"eventName": item.0.name}),
+ ));
+ }
+ p.lock().unwrap().push((item.0.name, item.1));
+ r
+ })
+ .collect())
+ }
+ },
+ ))
+ .gather()
+ })
+ .handle_ingestion_warnings(Arc::new(OutputsWarningEmitter {
+ outputs: Arc::new(wo),
+ }))
+ },
+ )
+ .message_aware(|b| b)
+ .handle_results(PipelineConfig {
+ outputs: Arc::new(outputs.clone()),
+ scheduler: Arc::new(scheduler.clone()),
+ reject_on_dlq_failure: false,
+ })
+ .handle_side_effects(
+ Arc::new(scheduler.clone()),
+ SideEffectHandlingConfig {
+ await_side_effects: true,
+ },
+ )
+ .build();
+
+ let feed = |team_id, name| {
+ create_ok_context(
+ RawEvent { team_id, name },
+ MsgCtx {
+ message: test_message(None),
+ },
+ )
+ };
+ pipeline.feed(vec![
+ feed(1, "pageview"),
+ feed(999, "will_fail"),
+ feed(2, "overflow"),
+ feed(3, "deprecated_event"),
+ ]);
+
+ let mut all = Vec::new();
+ while let Some(chunk) = pipeline.next().await.unwrap() {
+ all.extend(chunk);
+ }
+ assert_eq!(all.len(), 4);
+ assert_eq!(all.iter().filter(|r| r.is_ok()).count(), 2);
+ assert_eq!(
+ *processed.lock().unwrap(),
+ vec![("pageview", 1), ("deprecated_event", 3)]
+ );
+ let produced: Vec<String> = outputs.calls().into_iter().map(|c| c.0).collect();
+ assert!(produced.contains(&DLQ_OUTPUT.to_string()));
+ assert!(produced.contains(&"overflow".to_string()));
+ let warnings = warning_outputs.calls();
+ assert_eq!(warnings.len(), 1);
+ assert_eq!(warnings[0].0, INGESTION_WARNINGS_OUTPUT);
+ let value = String::from_utf8(warnings[0].1.value.clone().unwrap().to_vec()).unwrap();
+ assert!(
+ value.contains("\"type\":\"event_dropped_by_transformation\""),
+ "{value}"
+ );
+}
diff --git a/rust/ingestion-framework/tests/docs_13_conventions.rs b/rust/ingestion-framework/tests/docs_13_conventions.rs
new file mode 100644
index 000000000000..1b6a1f60f520
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_13_conventions.rs
@@ -0,0 +1,340 @@
+mod common;
+
+use std::sync::Arc;
+
+use common::{shared, test_message, MockOutputs};
+use ingestion_framework::warning_handling::OutputsWarningEmitter;
+use ingestion_framework::{
+ create_ok_context, dlq, new_chunk_pipeline, new_pipeline, ok, step, HasMessage, HasTeamId,
+ KafkaMessage, OkResultWithContext, PipelineBuilder, PipelineConfig, ProcessingStep,
+ PromiseScheduler, SideEffect, SideEffectHandlingConfig, StartPipelineBuilder,
+};
+
+#[tokio::test]
+async fn steps_are_created_via_factory_functions() {
+ fn create_process_step() -> ProcessingStep<String, (String, bool)> {
+ step("processStep", |v: String| async move {
+ Ok(ok((v.to_uppercase(), true)))
+ })
+ }
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe(create_process_step())
+ .build();
+ let r = pipeline
+ .process(create_ok_context("hello".into(), ()))
+ .await
+ .unwrap();
+ assert_eq!(r.value(), Some(&("HELLO".to_string(), true)));
+}
+
+#[tokio::test]
+async fn pipelines_are_created_via_factory_functions_to_ensure_single_caller() {
+ fn create_my_pipeline() -> ingestion_framework::DynPipeline<i64, (i64, bool), ()> {
+ new_pipeline::<i64, ()>()
+ .pipe(step(
+ "processStep",
+ |id: i64| async move { Ok(ok((id, true))) },
+ ))
+ .build()
+ }
+ let p1 = create_my_pipeline();
+ let p2 = create_my_pipeline();
+ assert!(!Arc::ptr_eq(&p1, &p2));
+ assert_eq!(
+ p1.process(create_ok_context(1, ()))
+ .await
+ .unwrap()
+ .value()
+ .unwrap()
+ .0,
+ 1
+ );
+ assert_eq!(
+ p2.process(create_ok_context(2, ()))
+ .await
+ .unwrap()
+ .value()
+ .unwrap()
+ .0,
+ 2
+ );
+}
+
+#[tokio::test]
+async fn factory_functions_accept_configuration_parameters() {
+ fn create_multiply_step(multiplier: i64) -> ProcessingStep<i64, (i64, i64)> {
+ step("multiplyStep", move |v: i64| async move {
+ Ok(ok((v, v * multiplier)))
+ })
+ }
+ let pipeline = new_pipeline::<i64, ()>()
+ .pipe(create_multiply_step(3))
+ .build();
+ let r = pipeline.process(create_ok_context(10, ())).await.unwrap();
+ assert_eq!(r.value().unwrap().1, 30);
+}
+
+#[tokio::test]
+async fn factory_functions_can_inject_multiple_dependencies() {
+ struct UserService;
+ impl UserService {
+ async fn get_user(&self, id: &str) -> String {
+ format!("User {id}")
+ }
+ }
+ struct AuditService;
+ impl AuditService {
+ async fn log(&self, _msg: String) {}
+ }
+ fn create_enrich_step(
+ users: Arc<UserService>,
+ audit: Arc<AuditService>,
+ ) -> ProcessingStep<String, (String, String, bool)> {
+ step("enrichStep", move |user_id: String| {
+ let (users, audit) = (users.clone(), audit.clone());
+ async move {
+ let name = users.get_user(&user_id).await;
+ let message = format!("Enriched user {user_id}");
+ let effect = SideEffect::spawn(async move {
+ audit.log(message).await;
+ Ok(())
+ });
+ Ok(ok((user_id, name, true)).with_side_effect(effect))
+ }
+ })
+ }
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe(create_enrich_step(
+ Arc::new(UserService),
+ Arc::new(AuditService),
+ ))
+ .build();
+ let r = pipeline
+ .process(create_ok_context("123".into(), ()))
+ .await
+ .unwrap();
+ assert_eq!(r.value().unwrap().1, "User 123");
+}
+
+#[tokio::test]
+async fn steps_extend_input_types_with_new_properties() {
+ // Node extends with intersection types (`T & ValidateOutput`). Rust wraps the input.
+ struct Validated<T> {
+ inner: T,
+ is_valid: bool,
+ }
+ struct Enriched<T> {
+ inner: T,
+ enriched_at: i64,
+ }
+ let pipeline = new_pipeline::<String, ()>()
+ .pipe(step("validateStep", |raw: String| async move {
+ let is_valid = !raw.is_empty();
+ Ok(ok(Validated {
+ inner: raw,
+ is_valid,
+ }))
+ }))
+ .pipe(step("enrichStep", |v: Validated<String>| async move {
+ Ok(ok(Enriched {
+ inner: v,
+ enriched_at: chrono::Utc::now().timestamp_millis(),
+ }))
+ }))
+ .build();
+ let r = pipeline
+ .process(create_ok_context("test data".into(), ()))
+ .await
+ .unwrap();
+ let v = r.value().unwrap();
+ assert_eq!(v.inner.inner, "test data");
+ assert!(v.inner.is_valid);
+ assert!(v.enriched_at > 0);
+}
+
+#[tokio::test]
+async fn terminal_transforms_produce_different_output_shapes() {
+ let pipeline = new_pipeline::<(&'static str, bool, i64), ()>()
+ .pipe(step(
+ "summarizeStep",
+ |e: (&'static str, bool, i64)| async move {
+ Ok(ok(format!("Event {}: valid={} at {}", e.0, e.1, e.2)))
+ },
+ ))
+ .build();
+ let r = pipeline
+ .process(create_ok_context(("evt-1", true, 1000), ()))
+ .await
+ .unwrap();
+ assert_eq!(r.value().unwrap(), "Event evt-1: valid=true at 1000");
+}
+
+#[tokio::test]
+async fn terminal_steps_return_void_with_side_effects() {
+ let emitted = shared(Vec::<&'static str>::new());
+ let e = emitted.clone();
+ let pipeline = new_pipeline::<&'static str, ()>()
+ .pipe(step("emitStep", move |id: &'static str| {
+ let e = e.clone();
+ async move {
+ Ok(ok(()).with_side_effect(SideEffect::spawn(async move {
+ e.lock().unwrap().push(id);
+ Ok(())
+ })))
+ }
+ }))
+ .build();
+ let mut r = pipeline
+ .process(create_ok_context("evt-1", ()))
+ .await
+ .unwrap();
+ assert_eq!(r.value(), Some(&()));
+ for s in std::mem::take(&mut r.context.side_effects) {
+ s.wait().await.unwrap();
+ }
+ assert!(emitted.lock().unwrap().contains(&"evt-1"));
+}
+
+#[tokio::test]
+async fn simple_terminal_steps_return_ok_unit() {
+ let pipeline = new_pipeline::<bool, ()>()
+ .pipe(step("skipStep", |_skip: bool| async move { Ok(ok(())) }))
+ .build();
+ assert!(pipeline
+ .process(create_ok_context(true, ()))
+ .await
+ .unwrap()
+ .is_ok());
+}
+
+#[derive(Debug, Clone)]
+struct MsgCtx {
+ message: KafkaMessage,
+}
+#[derive(Debug, Clone)]
+struct TeamMsgCtx {
+ message: KafkaMessage,
+ team_id: Option<i64>,
+}
+impl From<MsgCtx> for TeamMsgCtx {
+ fn from(c: MsgCtx) -> Self {
+ Self {
+ message: c.message,
+ team_id: None,
+ }
+ }
+}
+impl HasMessage for MsgCtx {
+ fn message(&self) -> &KafkaMessage {
+ &self.message
+ }
+}
+impl HasMessage for TeamMsgCtx {
+ fn message(&self) -> &KafkaMessage {
+ &self.message
+ }
+}
+impl HasTeamId for TeamMsgCtx {
+ fn team_id(&self) -> i64 {
+ self.team_id.unwrap_or_default()
+ }
+}
+
+#[derive(Debug, Clone)]
+struct PreTeam {
+ team_id: i64,
+ event_name: String,
+}
+
+#[tokio::test]
+async fn pipelines_are_organized_into_preprocessing_and_processing_phases() {
+ let phase_log = shared(Vec::<&'static str>::new());
+ let scheduler = PromiseScheduler::new();
+
+ let (l1, l2, l3) = (phase_log.clone(), phase_log.clone(), phase_log.clone());
+ let pre_team = move |b: StartPipelineBuilder<KafkaMessage, MsgCtx>| -> PipelineBuilder<KafkaMessage, PreTeam, MsgCtx> {
+ b.pipe(step("parseStep", move |m: KafkaMessage| {
+ l1.lock().unwrap().push("pre-team: parse");
+ async move {
+ match serde_json::from_slice::<serde_json::Value>(m.value.as_deref().unwrap_or(b"{}")) {
+ Ok(v) => Ok(ok(PreTeam {
+ team_id: v["teamId"].as_i64().unwrap_or_default(),
+ event_name: v["eventName"].as_str().unwrap_or_default().to_string(),
+ })),
+ Err(e) => Ok(dlq("Failed to parse message", Some(e.into()))),
+ }
+ }
+ }))
+ };
+ let post_team = move |b: StartPipelineBuilder<PreTeam, TeamMsgCtx>| {
+ b.pipe(step("validateStep", move |p: PreTeam| {
+ l2.lock().unwrap().push("post-team: validate");
+ async move { Ok(ok((p, true))) }
+ }))
+ };
+ let processing = move |b: StartPipelineBuilder<(PreTeam, bool), TeamMsgCtx>| {
+ b.pipe(step(
+ "processStep",
+ move |(p, _validated): (PreTeam, bool)| {
+ l3.lock().unwrap().push("processing: process");
+ async move { Ok(ok(format!("{} for Team {}", p.event_name, p.team_id))) }
+ },
+ ))
+ };
+
+ let pipeline =
+ new_chunk_pipeline::<KafkaMessage, MsgCtx>()
+ .message_aware(move |b| {
+ b.concurrently(pre_team, None).filter_map(
+ |el: OkResultWithContext<PreTeam, MsgCtx>| {
+ let team_id = el.value.team_id;
+ OkResultWithContext {
+ value: el.value,
+ context: el.context.map_data(|c| TeamMsgCtx {
+ message: c.message,
+ team_id: Some(team_id),
+ }),
+ }
+ },
+ move |b| {
+ b.team_aware(move |b| {
+ b.concurrently(post_team, None).concurrently_per_group(
+ |item: &(PreTeam, bool)| item.0.team_id,
+ move |g| g.sequentially(processing),
+ None,
+ )
+ })
+ .handle_ingestion_warnings(Arc::new(OutputsWarningEmitter {
+ outputs: Arc::new(MockOutputs::default()),
+ }))
+ },
+ )
+ })
+ .handle_results(PipelineConfig {
+ outputs: Arc::new(MockOutputs::default()),
+ scheduler: Arc::new(scheduler.clone()),
+ reject_on_dlq_failure: false,
+ })
+ .handle_side_effects(
+ Arc::new(scheduler),
+ SideEffectHandlingConfig {
+ await_side_effects: true,
+ },
+ )
+ .build();
+
+ let mut message = test_message(None);
+ message.value = Some(br#"{"teamId":42,"eventName":"pageview"}"#.to_vec().into());
+ pipeline.feed(vec![create_ok_context(message.clone(), MsgCtx { message })]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(
+ *phase_log.lock().unwrap(),
+ vec![
+ "pre-team: parse",
+ "post-team: validate",
+ "processing: process"
+ ]
+ );
+ assert_eq!(results.len(), 1);
+ assert_eq!(results[0].value().unwrap(), "pageview for Team 42");
+}
diff --git a/rust/ingestion-framework/tests/docs_14_batching.rs b/rust/ingestion-framework/tests/docs_14_batching.rs
new file mode 100644
index 000000000000..491b594d0c4a
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_14_batching.rs
@@ -0,0 +1,203 @@
+mod common;
+
+use common::{items, shared};
+use ingestion_framework::{
+ new_batching_pipeline, ok, step, AfterBatchInput, BatchingPipelineOptions, BeforeBatchInput,
+ BeforeBatchOutput, FeedRejectionKind, FeedResult,
+};
+
+fn unbounded() -> BatchingPipelineOptions {
+ BatchingPipelineOptions {
+ concurrent_batches: usize::MAX,
+ }
+}
+
+fn passthrough_before<CB: Send + 'static>(
+ name: &str,
+ make: impl Fn(u64) -> CB + Send + Sync + 'static,
+) -> ingestion_framework::ProcessingStep<BeforeBatchInput<i64, ()>, BeforeBatchOutput<i64, (), CB>>
+{
+ ingestion_framework::before_batch_step(name, make)
+}
+
+#[tokio::test]
+async fn each_feed_forms_one_batch_with_sequential_batch_id_and_message_id_tagging() {
+ let before_ids = shared(Vec::<u64>::new());
+ let after_ids = shared(Vec::<u64>::new());
+ let (b, a) = (before_ids.clone(), after_ids.clone());
+ let pipeline = new_batching_pipeline::<i64, _, (), (), u64, ()>(
+ move |builder| {
+ builder.pipe(step(
+ "recordingBefore",
+ move |input: BeforeBatchInput<i64, ()>| {
+ b.lock().unwrap().push(input.batch_id);
+ async move {
+ Ok(ok(BeforeBatchOutput {
+ batch_context: input.batch_id,
+ elements: input.elements,
+ batch_id: input.batch_id,
+ }))
+ }
+ },
+ ))
+ },
+ |builder| builder,
+ move |builder| {
+ builder.pipe(step(
+ "recordingAfter",
+ move |input: AfterBatchInput<_, (), u64>| {
+ a.lock().unwrap().push(input.batch_id);
+ async move { Ok(ok(input)) }
+ },
+ ))
+ },
+ unbounded(),
+ );
+ pipeline.feed(items([10, 20]), (), None).await.unwrap();
+ pipeline.feed(items([30]), (), None).await.unwrap();
+ let mut message_ids = Vec::new();
+ while let Some(batch) = pipeline.next().await.unwrap() {
+ for element in batch.elements {
+ message_ids.push(element.context.message_id.unwrap());
+ }
+ }
+ assert_eq!(*before_ids.lock().unwrap(), vec![0, 1]);
+ assert_eq!(*after_ids.lock().unwrap(), vec![0, 1]);
+ assert_eq!(message_ids, vec![0, 1, 2]);
+}
+
+#[tokio::test]
+async fn batch_context_flows_from_before_batch_to_after_batch() {
+ let seen = shared(None::<String>);
+ let s = seen.clone();
+ let pipeline = new_batching_pipeline::<i64, _, (), (), String, ()>(
+ |builder| {
+ builder.pipe(passthrough_before("attachStore", |id| {
+ format!("store-{id}")
+ }))
+ },
+ |builder| builder,
+ move |builder| {
+ builder.pipe(step(
+ "readStore",
+ move |input: AfterBatchInput<_, (), String>| {
+ *s.lock().unwrap() = Some((*input.batch_context).clone());
+ async move { Ok(ok(input)) }
+ },
+ ))
+ },
+ unbounded(),
+ );
+ pipeline.feed(items([1]), (), None).await.unwrap();
+ pipeline.next().await.unwrap();
+ assert_eq!(seen.lock().unwrap().as_deref(), Some("store-0"));
+}
+
+fn simple_pipeline(
+ concurrent_batches: usize,
+ before_ran: Option<std::sync::Arc<std::sync::Mutex<bool>>>,
+) -> ingestion_framework::BatchingPipeline<
+ i64,
+ ingestion_framework::WithBatch<i64, ()>,
+ (),
+ (),
+ (),
+ (),
+> {
+ new_batching_pipeline(
+ move |builder| {
+ builder.pipe(step(
+ "recordingBefore",
+ move |input: BeforeBatchInput<i64, ()>| {
+ if let Some(r) = &before_ran {
+ *r.lock().unwrap() = true;
+ }
+ async move {
+ Ok(ok(BeforeBatchOutput {
+ batch_context: (),
+ elements: input.elements,
+ batch_id: input.batch_id,
+ }))
+ }
+ },
+ ))
+ },
+ |builder| builder,
+ |builder| {
+ builder.pipe(ingestion_framework::passthrough_after_batch_step(
+ "passThroughAfter",
+ ))
+ },
+ BatchingPipelineOptions { concurrent_batches },
+ )
+}
+
+#[tokio::test]
+async fn feed_reports_at_capacity_backpressure_when_concurrent_batches_is_exceeded() {
+ let pipeline = simple_pipeline(1, None);
+ assert_eq!(
+ pipeline.feed(items([1]), (), None).await.unwrap(),
+ FeedResult::Ok
+ );
+ match pipeline.feed(items([2]), (), None).await.unwrap() {
+ FeedResult::Rejected { kind, .. } => assert_eq!(kind, FeedRejectionKind::AtCapacity),
+ other => panic!("expected rejection, got {other:?}"),
+ }
+ let drained = pipeline.next().await.unwrap().unwrap();
+ assert!(drained.elements.iter().all(|e| e.is_ok()));
+ assert_eq!(
+ pipeline.feed(items([3]), (), None).await.unwrap(),
+ FeedResult::Ok
+ );
+}
+
+#[tokio::test]
+async fn an_empty_feed_is_a_no_op_that_runs_no_hooks_and_consumes_no_capacity() {
+ let ran = shared(false);
+ let pipeline = simple_pipeline(1, Some(ran.clone()));
+ assert_eq!(
+ pipeline.feed(vec![], (), None).await.unwrap(),
+ FeedResult::Ok
+ );
+ assert!(!*ran.lock().unwrap());
+ assert!(pipeline.next().await.unwrap().is_none());
+ assert_eq!(
+ pipeline.feed(items([1]), (), None).await.unwrap(),
+ FeedResult::Ok
+ );
+ assert!(pipeline.next().await.unwrap().is_some());
+}
+
+#[tokio::test]
+async fn a_before_batch_that_changes_the_element_count_throws() {
+ let pipeline = new_batching_pipeline::<i64, _, (), (), (), ()>(
+ |builder| {
+ builder.pipe(step(
+ "filteringBefore",
+ |input: BeforeBatchInput<i64, ()>| async move {
+ let mut elements = input.elements;
+ elements.remove(0);
+ Ok(ok(BeforeBatchOutput {
+ batch_context: (),
+ elements,
+ batch_id: input.batch_id,
+ }))
+ },
+ ))
+ },
+ |builder| builder,
+ |builder| {
+ builder.pipe(ingestion_framework::passthrough_after_batch_step(
+ "passThroughAfter",
+ ))
+ },
+ BatchingPipelineOptions {
+ concurrent_batches: 1,
+ },
+ );
+ let err = pipeline.feed(items([1, 2]), (), None).await.unwrap_err();
+ assert!(
+ err.to_string().contains("changed element count (2 -> 1)"),
+ "{err}"
+ );
+}
diff --git a/rust/ingestion-framework/tests/docs_15_consuming_pipelines.rs b/rust/ingestion-framework/tests/docs_15_consuming_pipelines.rs
new file mode 100644
index 000000000000..ade132c63b38
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_15_consuming_pipelines.rs
@@ -0,0 +1,99 @@
+mod common;
+
+use std::sync::{Arc, Mutex};
+
+use common::items;
+use ingestion_framework::{
+ chunk_step, drop, new_chunk_pipeline, ok, ChunkPipelineUnwrapper, SideEffect,
+};
+
+#[tokio::test]
+async fn feed_then_next_drives_the_pipeline_until_it_returns_null() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("doubleStep", |events: Vec<i64>| async move {
+ Ok(events.into_iter().map(|e| ok(e * 2)).collect())
+ }))
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ let mut values = Vec::new();
+ while let Some(results) = pipeline.next().await.unwrap() {
+ values.extend(results.into_iter().filter_map(|r| r.result.into_value()));
+ }
+ assert_eq!(values, [2, 4, 6]);
+ assert!(pipeline.next().await.unwrap().is_none());
+}
+
+#[tokio::test]
+async fn chunk_pipeline_unwrapper_returns_plain_values_and_drops_non_ok_results() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("filterStep", |events: Vec<i64>| async move {
+ Ok(events
+ .into_iter()
+ .map(|e| {
+ if e % 2 == 0 {
+ drop("even filtered")
+ } else {
+ ok(e)
+ }
+ })
+ .collect())
+ }))
+ .build();
+ let unwrapper = ChunkPipelineUnwrapper::new(pipeline);
+ unwrapper.feed(items([1, 2, 3]));
+ let mut values = Vec::new();
+ while let Some(chunk) = unwrapper.next().await.unwrap() {
+ values.extend(chunk);
+ }
+ assert_eq!(values, [1, 3]);
+}
+
+#[derive(Clone, Default)]
+struct Capture(Arc<Mutex<Vec<String>>>);
+
+impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Capture {
+ fn on_event(
+ &self,
+ event: &tracing::Event<'_>,
+ _ctx: tracing_subscriber::layer::Context<'_, S>,
+ ) {
+ struct V<'a>(&'a mut String);
+ impl tracing::field::Visit for V<'_> {
+ fn record_debug(&mut self, _f: &tracing::field::Field, value: &dyn std::fmt::Debug) {
+ self.0.push_str(&format!("{value:?}"));
+ }
+ }
+ let mut s = String::new();
+ event.record(&mut V(&mut s));
+ self.0.lock().unwrap().push(s);
+ }
+}
+
+#[tokio::test]
+async fn chunk_pipeline_unwrapper_warns_about_unhandled_side_effects() {
+ use tracing_subscriber::layer::SubscriberExt;
+ let capture = Capture::default();
+ let subscriber = tracing_subscriber::registry().with(capture.clone());
+ let _guard = tracing::subscriber::set_default(subscriber);
+
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step(
+ "stepWithSideEffect",
+ |events: Vec<i64>| async move {
+ Ok(events
+ .into_iter()
+ .map(|e| ok(e).with_side_effect(SideEffect::resolved()))
+ .collect())
+ },
+ ))
+ .build();
+ let unwrapper = ChunkPipelineUnwrapper::new(pipeline);
+ unwrapper.feed(items([1]));
+ unwrapper.next().await.unwrap();
+ assert!(capture
+ .0
+ .lock()
+ .unwrap()
+ .iter()
+ .any(|m| m.contains("remaining side effects")));
+}
diff --git a/rust/ingestion-framework/tests/docs_17_fan_out_fan_in.rs b/rust/ingestion-framework/tests/docs_17_fan_out_fan_in.rs
new file mode 100644
index 000000000000..01af8f3dc910
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_17_fan_out_fan_in.rs
@@ -0,0 +1,147 @@
+mod common;
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use common::items;
+use ingestion_framework::{drop, new_chunk_pipeline, ok, step, PipelineError, RetryOptions};
+
+#[tokio::test]
+async fn uploads_attachments_concurrently_and_folds_the_results_back_in() {
+ let in_flight = Arc::new(AtomicUsize::new(0));
+ let high_water = Arc::new(AtomicUsize::new(0));
+ let (f, h) = (in_flight.clone(), high_water.clone());
+ let pipeline = new_chunk_pipeline::<(&'static str, Vec<&'static str>), ()>()
+ .fan_out(
+ "attachmentsFanOut",
+ |e: &(&'static str, Vec<&'static str>)| {
+ e.1.iter().map(|a| (e.0, *a)).collect::<Vec<_>>()
+ },
+ )
+ .via(move |sub| {
+ sub.concurrently(
+ move |b| {
+ b.pipe(step(
+ "uploadStep",
+ move |u: (&'static str, &'static str)| {
+ let (f, h) = (f.clone(), h.clone());
+ async move {
+ let now = f.fetch_add(1, Ordering::SeqCst) + 1;
+ h.fetch_max(now, Ordering::SeqCst);
+ tokio::task::yield_now().await;
+ f.fetch_sub(1, Ordering::SeqCst);
+ Ok(ok(format!("s3://bucket/{}", u.1)))
+ }
+ },
+ ))
+ },
+ Some(2),
+ )
+ })
+ .fan_in(
+ "collectUrlsFanIn",
+ |e: (&'static str, Vec<&'static str>), urls: Vec<String>| {
+ let mut urls = urls;
+ urls.sort();
+ (e.0, urls)
+ },
+ )
+ .build();
+ pipeline.feed(items([
+ ("e1", vec!["a.png", "b.png", "c.png"]),
+ ("e2", vec![]),
+ ]));
+ let mut collected = common::consume_all(pipeline.as_ref()).await;
+ collected.sort();
+ assert_eq!(
+ collected,
+ vec![
+ (
+ "e1",
+ vec![
+ "s3://bucket/a.png".to_string(),
+ "s3://bucket/b.png".into(),
+ "s3://bucket/c.png".into()
+ ]
+ ),
+ ("e2", vec![]),
+ ]
+ );
+ assert!(high_water.load(Ordering::SeqCst) <= 2);
+}
+
+#[tokio::test]
+async fn retries_a_transient_sub_step_failure_without_disturbing_the_parent() {
+ let attempts = Arc::new(AtomicUsize::new(0));
+ let a = attempts.clone();
+ let pipeline = new_chunk_pipeline::<(&'static str, Vec<i64>), ()>()
+ .fan_out("partsFanOut", |j: &(&'static str, Vec<i64>)| j.1.clone())
+ .via(move |sub| {
+ sub.concurrently(
+ move |b| {
+ b.pipe_with_retry(
+ step("flakyStep", move |part: i64| {
+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;
+ async move {
+ if n == 1 {
+ return Err(PipelineError::retriable("transient"));
+ }
+ Ok(ok(part))
+ }
+ }),
+ RetryOptions {
+ tries: Some(3),
+ sleep_ms: Some(1),
+ ..Default::default()
+ },
+ )
+ },
+ None,
+ )
+ })
+ .fan_in(
+ "sumFanIn",
+ |j: (&'static str, Vec<i64>), parts: Vec<i64>| (j.0, parts.iter().sum::<i64>()),
+ )
+ .build();
+ pipeline.feed(items([("job", vec![7])]));
+ let mut results = Vec::new();
+ while let Some(chunk) = pipeline.next().await.unwrap() {
+ results.extend(chunk);
+ }
+ assert_eq!(results.len(), 1);
+ assert_eq!(results[0].value(), Some(&("job", 7)));
+ assert_eq!(attempts.load(Ordering::SeqCst), 2);
+}
+
+#[tokio::test]
+async fn drops_a_sub_element_without_failing_its_parent() {
+ let pipeline = new_chunk_pipeline::<(&'static str, Vec<i64>), ()>()
+ .fan_out("partsFanOut", |j: &(&'static str, Vec<i64>)| j.1.clone())
+ .via(|sub| {
+ sub.concurrently(
+ |b| {
+ b.pipe(step("validatePartStep", |part: i64| async move {
+ Ok(if part < 0 {
+ drop("negative part")
+ } else {
+ ok(part)
+ })
+ }))
+ },
+ None,
+ )
+ })
+ .fan_in(
+ "sumFanIn",
+ |j: (&'static str, Vec<i64>), parts: Vec<i64>| (j.0, parts.iter().sum::<i64>()),
+ )
+ .build();
+ pipeline.feed(items([
+ ("partly-valid", vec![1, -2, 3]),
+ ("all-valid", vec![1, 2]),
+ ]));
+ let mut collected = common::consume_all(pipeline.as_ref()).await;
+ collected.sort();
+ assert_eq!(collected, vec![("all-valid", 3), ("partly-valid", 4)]);
+}
diff --git a/rust/ingestion-framework/tests/docs_18_batch_budgets.rs b/rust/ingestion-framework/tests/docs_18_batch_budgets.rs
new file mode 100644
index 000000000000..972ab0197a1a
--- /dev/null
+++ b/rust/ingestion-framework/tests/docs_18_batch_budgets.rs
@@ -0,0 +1,301 @@
+//! `tokio::time::advance` inside a step replaces the synchronous `jest.advanceTimersByTime`.
+
+mod common;
+
+use std::time::Duration;
+
+use common::{items, shared};
+use ingestion_framework::{
+ before_batch_step, chunk_step, new_batching_pipeline, ok, passthrough_after_batch_step, step,
+ BatchBudget, BatchingPipelineOptions, ChunkResults, NonOk, Outcome, WithBatch,
+};
+use tokio::time::Instant;
+
+#[derive(Debug, Clone, Copy)]
+struct Event {
+ key: &'static str,
+ seq: i64,
+}
+
+type V = WithBatch<Event, ()>;
+
+fn kinds<C>(results: &ChunkResults<V, C>) -> Vec<&'static str> {
+ results
+ .iter()
+ .map(|r| match &r.result {
+ Outcome::Ok(_) => "ok",
+ Outcome::NonOk(NonOk::Timeout { .. }) => "timeout",
+ _ => "other",
+ })
+ .collect()
+}
+
+fn reasons<C>(results: &ChunkResults<V, C>) -> Vec<Option<String>> {
+ results
+ .iter()
+ .map(|r| match &r.result {
+ Outcome::NonOk(NonOk::Timeout { reason }) => Some(reason.clone()),
+ _ => None,
+ })
+ .collect()
+}
+
+fn events(
+ key: &'static str,
+ seqs: impl IntoIterator<Item = i64>,
+) -> Vec<ingestion_framework::OkResultWithContext<Event, ()>> {
+ items(seqs.into_iter().map(|seq| Event { key, seq }))
+}
+
+async fn advance_1s() {
+ tokio::time::advance(Duration::from_millis(1000)).await;
+}
+
+fn one() -> BatchingPipelineOptions {
+ BatchingPipelineOptions {
+ concurrent_batches: 1,
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn stops_each_element_at_its_next_step_once_the_budget_is_exhausted() {
+ let ran = shared(Vec::<String>::new());
+ let (r1, r2) = (ran.clone(), ran.clone());
+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(
+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),
+ move |b| {
+ b.sequentially(move |s| {
+ s.pipe(step("enrichStep", move |e: V| {
+ r1.lock().unwrap().push(format!("enrich-{}", e.seq));
+ async move {
+ if e.seq == 2 {
+ advance_1s().await;
+ }
+ Ok(ok(e))
+ }
+ }))
+ .pipe(step("emitStep", move |e: V| {
+ r2.lock().unwrap().push(format!("emit-{}", e.seq));
+ async move { Ok(ok(e)) }
+ }))
+ })
+ },
+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),
+ one(),
+ );
+ pipeline
+ .feed(
+ events("a", [1, 2, 3]),
+ (),
+ Some(BatchBudget::soft_deadline(
+ Instant::now() + Duration::from_millis(1000),
+ )),
+ )
+ .await
+ .unwrap();
+ let batch = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(*ran.lock().unwrap(), vec!["enrich-1", "emit-1", "enrich-2"]);
+ assert_eq!(kinds(&batch.elements), ["ok", "timeout", "timeout"]);
+ assert_eq!(
+ reasons(&batch.elements),
+ [
+ None,
+ Some("budget exceeded before emitStep".into()),
+ Some("budget exceeded before enrichStep".into())
+ ]
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn runs_a_chunk_step_on_the_elements_whose_batch_still_has_time() {
+ let chunked = shared(Vec::<Vec<i64>>::new());
+ let c = chunked.clone();
+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(
+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),
+ move |b| {
+ b.pipe_chunk(chunk_step("writeChunk", move |events: Vec<V>| {
+ c.lock()
+ .unwrap()
+ .push(events.iter().map(|e| e.seq).collect());
+ async move { Ok(events.into_iter().map(ok).collect()) }
+ }))
+ },
+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),
+ BatchingPipelineOptions {
+ concurrent_batches: 2,
+ },
+ );
+ pipeline
+ .feed(
+ events("a", [1]),
+ (),
+ Some(BatchBudget::soft_deadline(
+ Instant::now() - Duration::from_millis(1),
+ )),
+ )
+ .await
+ .unwrap();
+ pipeline
+ .feed(
+ events("b", [2]),
+ (),
+ Some(BatchBudget::soft_deadline(
+ Instant::now() + Duration::from_secs(60),
+ )),
+ )
+ .await
+ .unwrap();
+ let spent = pipeline.next().await.unwrap().unwrap();
+ let fresh = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(*chunked.lock().unwrap(), vec![vec![2]]);
+ assert_eq!(kinds(&spent.elements), ["timeout"]);
+ assert_eq!(
+ reasons(&spent.elements),
+ [Some("budget exceeded before writeChunk".into())]
+ );
+ assert_eq!(kinds(&fresh.elements), ["ok"]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn runs_an_element_past_the_barrier_to_completion_after_the_budget_expires() {
+ let ran = shared(Vec::<String>::new());
+ let (r1, r2, r3) = (ran.clone(), ran.clone(), ran.clone());
+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(
+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),
+ move |b| {
+ b.sequentially(move |s| {
+ s.pipe(step("prepareEvent", move |e: V| {
+ r1.lock().unwrap().push(format!("prepare-{}", e.seq));
+ async move { Ok(ok(e)) }
+ }))
+ .timeout_barrier()
+ .pipe(step("writePerson", move |e: V| {
+ r2.lock().unwrap().push(format!("write-{}", e.seq));
+ async move {
+ advance_1s().await;
+ Ok(ok(e))
+ }
+ }))
+ .pipe(step("emitEvent", move |e: V| {
+ r3.lock().unwrap().push(format!("emit-{}", e.seq));
+ async move { Ok(ok(e)) }
+ }))
+ })
+ },
+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),
+ one(),
+ );
+ pipeline
+ .feed(
+ events("a", [1, 2]),
+ (),
+ Some(BatchBudget::soft_deadline(
+ Instant::now() + Duration::from_millis(1000),
+ )),
+ )
+ .await
+ .unwrap();
+ let batch = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(*ran.lock().unwrap(), vec!["prepare-1", "write-1", "emit-1"]);
+ assert_eq!(kinds(&batch.elements), ["ok", "timeout"]);
+ assert_eq!(
+ reasons(&batch.elements),
+ [None, Some("budget exceeded before prepareEvent".into())]
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn returns_one_result_per_message_and_still_runs_after_batch_when_everything_times_out() {
+ let flushed = shared(0usize);
+ let f = flushed.clone();
+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(
+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),
+ |b| b.sequentially(|s| s.pipe(step("emitStep", |e: V| async move { Ok(ok(e)) }))),
+ move |b| {
+ b.pipe(step(
+ "flushBatch",
+ move |input: ingestion_framework::AfterBatchInput<V, (), ()>| {
+ *f.lock().unwrap() = input.elements.len();
+ async move { Ok(ok(input)) }
+ },
+ ))
+ },
+ one(),
+ );
+ let mut fed = events("a", [1, 2]);
+ fed.extend(events("b", [1]));
+ pipeline
+ .feed(
+ fed,
+ (),
+ Some(BatchBudget::soft_deadline(
+ Instant::now() - Duration::from_millis(1),
+ )),
+ )
+ .await
+ .unwrap();
+ let batch = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(batch.elements.len(), 3);
+ assert_eq!(kinds(&batch.elements), ["timeout", "timeout", "timeout"]);
+ assert_eq!(*flushed.lock().unwrap(), 3);
+}
+
+#[tokio::test(start_paused = true)]
+async fn completes_a_prefix_of_each_groups_feed_order() {
+ let processed = shared(Vec::<i64>::new());
+ let p = processed.clone();
+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(
+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),
+ move |b| {
+ b.concurrently_per_group(
+ |e: &V| e.key,
+ move |g| {
+ g.sequentially(move |s| {
+ s.pipe(step("processStep", move |e: V| {
+ let count = {
+ let mut p = p.lock().unwrap();
+ p.push(e.seq);
+ p.len()
+ };
+ async move {
+ if count == 2 {
+ advance_1s().await;
+ }
+ Ok(ok(e))
+ }
+ }))
+ })
+ },
+ None,
+ )
+ },
+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),
+ one(),
+ );
+ pipeline
+ .feed(
+ events("a", [1, 2, 3, 4]),
+ (),
+ Some(BatchBudget::soft_deadline(
+ Instant::now() + Duration::from_millis(1000),
+ )),
+ )
+ .await
+ .unwrap();
+ let batch = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(*processed.lock().unwrap(), vec![1, 2]);
+ assert_eq!(kinds(&batch.elements), ["ok", "ok", "timeout", "timeout"]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn never_cancels_anything_under_the_default_unlimited_budget() {
+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(
+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),
+ |b| b.sequentially(|s| s.pipe(step("emitStep", |e: V| async move { Ok(ok(e)) }))),
+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),
+ one(),
+ );
+ pipeline.feed(events("a", [1]), (), None).await.unwrap();
+ let batch = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(kinds(&batch.elements), ["ok"]);
+}
diff --git a/rust/ingestion-framework/tests/per_element_step.rs b/rust/ingestion-framework/tests/per_element_step.rs
new file mode 100644
index 000000000000..2713b3c10afd
--- /dev/null
+++ b/rust/ingestion-framework/tests/per_element_step.rs
@@ -0,0 +1,48 @@
+use std::sync::{Arc, Mutex};
+
+use ingestion_framework::{ok, per_element, step, PipelineResult};
+
+#[derive(Default)]
+struct Track {
+ running: usize,
+ peak: usize,
+ started: Vec<i64>,
+}
+
+#[tokio::test]
+async fn keeps_input_order_and_never_runs_more_than_max_concurrency_steps_at_once() {
+ let track = Arc::new(Mutex::new(Track::default()));
+ let t = track.clone();
+ let chunk = per_element(
+ step("slowStep", move |value: i64| {
+ let t = t.clone();
+ async move {
+ {
+ let mut t = t.lock().unwrap();
+ t.started.push(value);
+ t.running += 1;
+ t.peak = t.peak.max(t.running);
+ }
+ // Later values wait fewer ticks and finish first, so output order cannot follow completion order.
+ for _ in 0..(8 - value) {
+ tokio::task::yield_now().await;
+ }
+ t.lock().unwrap().running -= 1;
+ Ok(ok(value * 10))
+ }
+ }),
+ Some(3),
+ );
+
+ let results: Vec<PipelineResult<i64>> = chunk.call(vec![1, 2, 3, 4, 5, 6, 7]).await.unwrap();
+
+ let values: Vec<i64> = results
+ .into_iter()
+ .map(|r| r.outcome.into_value().unwrap())
+ .collect();
+ assert_eq!(values, [10, 20, 30, 40, 50, 60, 70]);
+ let track = track.lock().unwrap();
+ assert_eq!(track.peak, 3);
+ assert_eq!(track.started, [1, 2, 3, 4, 5, 6, 7]);
+ assert_eq!(chunk.name().as_ref(), "slowStep");
+}
diff --git a/rust/ingestion-framework/tests/review_regressions.rs b/rust/ingestion-framework/tests/review_regressions.rs
new file mode 100644
index 000000000000..10330958b8c4
--- /dev/null
+++ b/rust/ingestion-framework/tests/review_regressions.rs
@@ -0,0 +1,419 @@
+mod common;
+
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use common::{items, test_message, MockOutputs, MsgCtx};
+use futures::FutureExt;
+use ingestion_framework::warning_handling::serialize_ingestion_warning;
+use ingestion_framework::{
+ chunk_step, create_ok_context, dlq, new_batching_pipeline, new_chunk_pipeline, new_pipeline,
+ ok, redirect_with, step, BatchingPipelineOptions, ChunkPipeline, MemoryOutputs, PipelineConfig,
+ PipelineError, PipelineWarning, PromiseScheduler, RetryOptions, SideEffect,
+ SideEffectHandlingConfig, SideEffectScheduler, DLQ_OUTPUT,
+};
+
+const AWAIT: SideEffectHandlingConfig = SideEffectHandlingConfig {
+ await_side_effects: true,
+};
+
+// The expected prefix is Node's `JSON.stringify({team_id, type, source, details, timestamp})`.
+#[test]
+fn warning_message_bytes_match_node() {
+ let mut w = PipelineWarning::new("client_ingestion_warning", serde_json::json!({}))
+ .with_detail("zeta", 1)
+ .with_detail("category", "caller")
+ .with_detail("alpha", 1.0)
+ .with_detail("big", 1e21);
+ w.pipeline_step = Some("s".into());
+ let s = serialize_ingestion_warning(42, &w).unwrap();
+ let expected = r#"{"team_id":42,"type":"client_ingestion_warning","source":"plugin-server","details":"{\"zeta\":1,\"category\":\"event\",\"alpha\":1,\"big\":1e+21,\"severity\":\"info\",\"pipelineStep\":\"s\"}","timestamp":""#;
+ assert!(s.starts_with(expected), "{s}");
+ let timestamp = &s[expected.len()..s.len() - 2];
+ assert!(
+ chrono::NaiveDateTime::parse_from_str(timestamp, "%Y-%m-%d %H:%M:%S%.3f").is_ok()
+ && timestamp.len() == 23,
+ "{timestamp}"
+ );
+}
+
+#[test]
+fn warning_type_outside_the_registry_fails_like_the_node_lookup() {
+ let w = PipelineWarning::new("not_a_registered_type", serde_json::json!({}));
+ assert!(serialize_ingestion_warning(1, &w).is_err());
+}
+
+// Node's DLQ error is `AggregateError(errors, reason)`, and `dlq_reason` is its message.
+#[tokio::test]
+async fn fan_out_dlq_reason_header_is_the_aggregate_reason_only() {
+ let outputs = MockOutputs::default();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()
+ .fan_out("split", |n: &i64| vec![*n])
+ .via(|b| {
+ b.pipe_chunk(chunk_step("check", |v: Vec<i64>| async move {
+ Ok(v.into_iter()
+ .map(|_| dlq::<i64>("bad sub", Some(anyhow::anyhow!("boom"))))
+ .collect())
+ }))
+ })
+ .fan_in("merge", |o: i64, _s: Vec<i64>| o)
+ .message_aware(|b| b)
+ .handle_results(PipelineConfig {
+ outputs: Arc::new(outputs.clone()),
+ scheduler: Arc::new(scheduler.clone()),
+ reject_on_dlq_failure: false,
+ })
+ .handle_side_effects(Arc::new(scheduler), AWAIT)
+ .build();
+ pipeline.feed(vec![create_ok_context(
+ 1,
+ MsgCtx {
+ message: test_message(None),
+ },
+ )]);
+ pipeline.next().await.unwrap().unwrap();
+ let calls = outputs.calls();
+ assert_eq!(
+ calls[0].1.headers.get("dlq_reason"),
+ Some("1/1 fan-out sub-elements dlq'd: bad sub")
+ );
+}
+
+// Node calls `outputs.produce` synchronously, in item order, inside `next()`.
+#[test]
+fn dlq_produces_follow_item_order_on_a_multi_thread_runtime() {
+ let rt = tokio::runtime::Builder::new_multi_thread()
+ .worker_threads(4)
+ .enable_all()
+ .build()
+ .unwrap();
+ for _ in 0..50 {
+ let order = rt
+ .block_on(async {
+ tokio::spawn(async {
+ let outputs = MemoryOutputs::new();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()
+ .pipe_chunk(chunk_step("v", |v: Vec<i64>| async move {
+ Ok(v.into_iter().map(|_| dlq::<i64>("x", None)).collect())
+ }))
+ .message_aware(|b| b)
+ .handle_results(PipelineConfig {
+ outputs: Arc::new(outputs.clone()),
+ scheduler: Arc::new(scheduler.clone()),
+ reject_on_dlq_failure: false,
+ })
+ .handle_side_effects(Arc::new(scheduler), AWAIT)
+ .build();
+ pipeline.feed(
+ (0..8)
+ .map(|i| {
+ let mut m = test_message(Some(b"k"));
+ m.offset = i;
+ create_ok_context(i, MsgCtx { message: m })
+ })
+ .collect(),
+ );
+ pipeline.next().await.unwrap();
+ outputs
+ .produced_to(DLQ_OUTPUT)
+ .iter()
+ .map(|m| m.headers.get("dlq_offset").unwrap().parse::<i64>().unwrap())
+ .collect::<Vec<_>>()
+ })
+ .await
+ })
+ .unwrap();
+ assert_eq!(order, (0..8).collect::<Vec<_>>());
+ }
+}
+
+// Node starts the redirect produce synchronously, so it is called when next() resolves.
+#[tokio::test]
+async fn redirect_without_await_ack_has_called_produce_when_next_returns() {
+ let outputs = MockOutputs::default();
+ outputs.close_gate();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()
+ .pipe_chunk(chunk_step("r", |v: Vec<i64>| async move {
+ Ok(v.into_iter()
+ .map(|_| redirect_with::<i64>("o", "overflow", true, false))
+ .collect())
+ }))
+ .message_aware(|b| b)
+ .handle_results(PipelineConfig {
+ outputs: Arc::new(outputs.clone()),
+ scheduler: Arc::new(scheduler.clone()),
+ reject_on_dlq_failure: false,
+ })
+ .handle_side_effects(Arc::new(scheduler), AWAIT)
+ .build();
+ pipeline.feed(vec![create_ok_context(
+ 1,
+ MsgCtx {
+ message: test_message(None),
+ },
+ )]);
+ pipeline.next().await.unwrap();
+ assert_eq!(outputs.calls().len(), 1);
+}
+
+// Node has no cancellation, so a batch always comes back, even after a dropped `next()`.
+#[tokio::test(start_paused = true)]
+async fn dropped_batching_next_does_not_lose_the_batch() {
+ let after_runs = Arc::new(AtomicU32::new(0));
+ let runs = after_runs.clone();
+ let pipeline = new_batching_pipeline::<i64, _, (), (), (), ()>(
+ |b| b.pipe(ingestion_framework::before_batch_step("before", |_| ())),
+ |b| b,
+ move |b| {
+ let runs = runs.clone();
+ b.pipe(step(
+ "slowAfter",
+ move |input: ingestion_framework::AfterBatchInput<_, (), ()>| {
+ runs.fetch_add(1, Ordering::SeqCst);
+ async move {
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ Ok(ok(input))
+ }
+ },
+ ))
+ },
+ BatchingPipelineOptions {
+ concurrent_batches: 1,
+ },
+ );
+ pipeline.feed(items([1, 2]), (), None).await.unwrap();
+ let _ = tokio::time::timeout(Duration::from_millis(10), pipeline.next()).await;
+ let batch = pipeline.next().await.unwrap().expect("batch was lost");
+ assert_eq!(batch.elements.len(), 2);
+ assert_eq!(after_runs.load(Ordering::SeqCst), 1, "afterBatch ran twice");
+ assert!(pipeline.next().await.unwrap().is_none());
+}
+
+#[tokio::test]
+async fn scheduler_releases_settled_side_effects() {
+ let scheduler = PromiseScheduler::new();
+ for _ in 0..1000 {
+ scheduler.schedule(SideEffect::spawn(async {
+ tokio::task::yield_now().await;
+ Ok(())
+ }));
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ assert_eq!(scheduler.pending_count(), 0);
+}
+
+// Node's retryIfRetriable with tries: 0 never calls fn and throws.
+#[tokio::test]
+async fn retry_with_zero_tries_never_calls_the_step() {
+ let calls = Arc::new(AtomicU32::new(0));
+ let c = calls.clone();
+ let p = new_pipeline::<i64, ()>()
+ .pipe_with_retry(
+ step("s", move |v: i64| {
+ c.fetch_add(1, Ordering::SeqCst);
+ async move { Ok(ok(v)) }
+ }),
+ RetryOptions {
+ tries: Some(0),
+ ..Default::default()
+ },
+ )
+ .build();
+ let r = p.process(create_ok_context(1, ())).await;
+ assert_eq!(calls.load(Ordering::SeqCst), 0);
+ assert!(r.is_err());
+}
+
+// Node reads only error.isRetriable; a wrapper error without the flag is retried.
+#[tokio::test(start_paused = true)]
+async fn wrapped_non_retriable_error_is_retried_like_node() {
+ let calls = Arc::new(AtomicU32::new(0));
+ let c = calls.clone();
+ let p = new_pipeline::<i64, ()>()
+ .pipe_with_retry(
+ step("s", move |_v: i64| {
+ c.fetch_add(1, Ordering::SeqCst);
+ async move {
+ Err::<ingestion_framework::PipelineResult<i64>, _>(
+ PipelineError::non_retriable("inner").context("outer"),
+ )
+ }
+ }),
+ RetryOptions::default(),
+ )
+ .build();
+ let _ = p.process(create_ok_context(1, ())).await;
+ assert_eq!(calls.load(Ordering::SeqCst), 3);
+}
+
+// A Node fan-in throw rejects next() and poisons the stage without crashing the process.
+#[tokio::test]
+async fn fan_in_panic_rejects_next_and_keeps_rejecting() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .fan_out("split", |n: &i64| vec![*n])
+ .via(|b| b)
+ .fan_in("explode", |_o: i64, _s: Vec<i64>| -> i64 {
+ panic!("fan-in bug")
+ })
+ .build();
+ pipeline.feed(items([1]));
+ for _ in 0..2 {
+ let outcome = std::panic::AssertUnwindSafe(pipeline.next())
+ .catch_unwind()
+ .await;
+ let error = outcome.expect("next() must not panic").unwrap_err();
+ assert!(error.to_string().contains("fan-in bug"), "{error}");
+ }
+}
+
+// `pLimit(0)` throws when the Node stage is built.
+#[test]
+#[should_panic(expected = "from 1 and up")]
+fn zero_max_concurrency_fails_like_p_limit() {
+ let _ = new_chunk_pipeline::<i64, ()>().concurrently(
+ |b| b.pipe(step("s", |v: i64| async move { Ok(ok(v)) })),
+ Some(0),
+ );
+}
+
+// A quadratic start makes 4x the distinct keys cost 16x the time.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn grouping_start_scales_linearly_in_distinct_keys() {
+ async fn run(n: i64) -> Duration {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently_per_group(
+ |v: &i64| *v,
+ |g| g.sequentially(|b| b.pipe(step("s", |v: i64| async move { Ok(ok(v)) }))),
+ None,
+ )
+ .build();
+ pipeline.feed(items(0..n));
+ let t = Instant::now();
+ common::consume_all(pipeline.as_ref()).await;
+ t.elapsed()
+ }
+ // Warm the runtime and allocator first, so the small run is not inflated.
+ run(5_000).await;
+ let small = run(10_000).await;
+ let big = run(40_000).await;
+ assert!(
+ big < small * 8,
+ "4x keys took {:.1}x ({small:?} vs {big:?})",
+ big.as_secs_f64() / small.as_secs_f64()
+ );
+}
+
+// Node keeps a promise in the scheduler until it settles.
+#[tokio::test]
+async fn scheduler_keeps_in_flight_side_effects_for_every_waiter() {
+ let scheduler = PromiseScheduler::new();
+ let (release, gate) = tokio::sync::oneshot::channel::<()>();
+ scheduler.schedule(SideEffect::spawn(async move {
+ let _ = gate.await;
+ Err(anyhow::anyhow!("produce failed"))
+ }));
+ let mut cancelled = Box::pin(scheduler.wait_for_all());
+ assert!(futures::poll!(cancelled.as_mut()).is_pending());
+ let mut overlapping = Box::pin(scheduler.wait_for_all());
+ assert!(futures::poll!(overlapping.as_mut()).is_pending());
+ drop(cancelled);
+ release.send(()).unwrap();
+ let error = overlapping.await.unwrap_err();
+ assert!(error.to_string().contains("produce failed"), "{error}");
+ assert!(scheduler.wait_for_all().await.is_ok());
+}
+
+// A Node fan-in throw leaves the parents completed before it in readyResults.
+#[tokio::test]
+async fn fan_in_panic_keeps_the_parents_completed_before_it() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .fan_out("split", |n: &i64| vec![*n])
+ .via(|b| b)
+ .fan_in("explode_on_2", |o: i64, _s: Vec<i64>| -> i64 {
+ assert!(o != 2, "fan-in bug");
+ o
+ })
+ .build();
+ pipeline.feed(items([1, 2]));
+ let first = std::panic::AssertUnwindSafe(pipeline.next())
+ .catch_unwind()
+ .await
+ .expect("next() must not panic");
+ assert!(first.is_err());
+ let kept = pipeline.next().await.unwrap().expect("completed parent");
+ let values: Vec<_> = kept
+ .into_iter()
+ .filter_map(|r| r.result.into_value())
+ .collect();
+ assert_eq!(values, vec![1]);
+ assert!(pipeline.next().await.is_err());
+}
+
+// A Node group processor that throws before it returns its promise rejects that group.
+#[tokio::test]
+async fn group_processor_panic_before_its_future_fails_next() {
+ let processor: ingestion_framework::grouping::GroupChunkProcessor<i64, i64, ()> =
+ Arc::new(|_items| panic!("processor bug"));
+ let pipeline = ingestion_framework::grouping::ConcurrentlyGroupingChunkPipeline::new(
+ Arc::new(|v: &i64| *v),
+ processor,
+ new_chunk_pipeline::<i64, ()>().build(),
+ None,
+ );
+ pipeline.feed(items([1]));
+ let outcome = tokio::time::timeout(Duration::from_secs(5), pipeline.next())
+ .await
+ .expect("next() must not wait forever");
+ let error = outcome.unwrap_err();
+ assert!(error.to_string().contains("processor bug"), "{error}");
+}
+
+// p-limit does not hold a slot for a caller that is gone.
+#[tokio::test]
+async fn limiter_releases_the_slot_of_a_dropped_acquire() {
+ let limiter = ingestion_framework::sync::FifoLimiter::new(1);
+ drop(limiter.acquire());
+ let held = tokio::time::timeout(Duration::from_secs(5), limiter.acquire())
+ .await
+ .expect("the slot of the dropped acquire was released");
+ // This waiter gets the slot when `held` drops, and is gone before it reads it.
+ let queued = limiter.acquire();
+ drop(held);
+ drop(queued);
+ tokio::time::timeout(Duration::from_secs(5), limiter.acquire())
+ .await
+ .expect("the slot was released");
+}
+
+// Node's setTimeout uses 1 ms for a negative, NaN or too-large delay.
+#[tokio::test(start_paused = true)]
+async fn retry_with_out_of_range_sleep_options_does_not_panic() {
+ let cases = [
+ (Some(-1.0), Some(0.0)),
+ (Some(f64::NAN), None),
+ (Some(f64::INFINITY), Some(0.0)),
+ (None, Some(3.0)),
+ (Some(f64::MAX), Some(-1.0)),
+ ];
+ for (backoff_factor, jitter) in cases {
+ let options = RetryOptions {
+ tries: Some(4),
+ sleep_ms: Some(100),
+ max_sleep_ms: Some(u64::MAX),
+ backoff_factor,
+ jitter,
+ ..RetryOptions::default()
+ };
+ let (result, attempts) = ingestion_framework::retry::retry_if_retriable(
+ || async { Err::<(), _>(anyhow::anyhow!("retriable")) },
+ &options,
+ )
+ .await;
+ assert!(result.is_err());
+ assert_eq!(attempts, 4, "{backoff_factor:?} {jitter:?}");
+ }
+}
diff --git a/rust/ingestion-framework/tests/unit_semantics.rs b/rust/ingestion-framework/tests/unit_semantics.rs
new file mode 100644
index 000000000000..674fd08d20e3
--- /dev/null
+++ b/rust/ingestion-framework/tests/unit_semantics.rs
@@ -0,0 +1,471 @@
+mod common;
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use common::{items, shared, sleep_ms, test_message, MsgCtx};
+use ingestion_framework::{
+ chunk_step, create_ok_context, dlq, new_chunk_pipeline, ok, step, timeout, MemoryOutputs,
+ PipelineConfig, PipelineResultType, PipelineWarning, PromiseScheduler,
+ SideEffectHandlingConfig, DLQ_OUTPUT,
+};
+use tokio::time::Instant;
+
+fn failing_on(n: i64) -> ingestion_framework::ProcessingStep<i64, i64> {
+ step("maybeFails", move |v: i64| async move {
+ sleep_ms(10 * v as u64).await;
+ if v == n {
+ anyhow::bail!("boom {v}");
+ }
+ Ok(ok(v))
+ })
+}
+
+#[tokio::test(start_paused = true)]
+async fn concurrent_poison_drains_in_flight_items_in_order_then_fails_for_good() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently(|b| b.pipe(failing_on(2)), None)
+ .build();
+ pipeline.feed(items([1, 2, 3]));
+ assert_eq!(pipeline.next().await.unwrap().unwrap()[0].value(), Some(&1));
+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");
+ assert_eq!(pipeline.next().await.unwrap().unwrap()[0].value(), Some(&3));
+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");
+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");
+}
+
+#[tokio::test(start_paused = true)]
+async fn concurrent_upstream_failure_poisons_the_stage() {
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .pipe_chunk(chunk_step("explodes", |_v: Vec<i64>| async move {
+ Err::<Vec<ingestion_framework::PipelineResult<i64>>, _>(anyhow::anyhow!("upstream"))
+ }))
+ .concurrently(
+ |b| b.pipe(step("id", |v: i64| async move { Ok(ok(v)) })),
+ None,
+ )
+ .build();
+ pipeline.feed(items([1]));
+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "upstream");
+ pipeline.feed(items([2]));
+ // Poisoned: no new source pull, so the later feed never runs.
+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "upstream");
+}
+
+#[tokio::test(start_paused = true)]
+async fn concurrent_feed_while_parked_starts_new_input_without_waiting_for_the_head() {
+ let starts = shared(Vec::<(i64, u128)>::new());
+ let s = starts.clone();
+ let origin = Instant::now();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently(
+ move |b| {
+ b.pipe(step("slowHead", move |v: i64| {
+ s.lock().unwrap().push((v, origin.elapsed().as_millis()));
+ async move {
+ sleep_ms(if v == 1 { 100 } else { 1 }).await;
+ Ok(ok(v))
+ }
+ }))
+ },
+ None,
+ )
+ .build();
+ pipeline.feed(items([1]));
+ let p = pipeline.clone();
+ let consumer = tokio::spawn(async move { common::consume_all(p.as_ref()).await });
+ sleep_ms(10).await;
+ pipeline.feed(items([2]));
+ let values = consumer.await.unwrap();
+ assert_eq!(values, [1, 2]);
+ let starts = starts.lock().unwrap().clone();
+ assert_eq!(starts[1].0, 2);
+ assert!(
+ starts[1].1 < 100,
+ "item 2 started at {}ms, behind the slow head",
+ starts[1].1
+ );
+}
+
+#[tokio::test(start_paused = true)]
+async fn max_concurrency_admits_items_in_input_order() {
+ let started = shared(Vec::<i64>::new());
+ let s = started.clone();
+ let pipeline = new_chunk_pipeline::<i64, ()>()
+ .concurrently(
+ move |b| {
+ b.pipe(step("record", move |v: i64| {
+ s.lock().unwrap().push(v);
+ async move {
+ sleep_ms(5).await;
+ Ok(ok(v))
+ }
+ }))
+ },
+ Some(1),
+ )
+ .build();
+ pipeline.feed(items(1..=20));
+ let values = common::consume_all(pipeline.as_ref()).await;
+ assert_eq!(values, (1..=20).collect::<Vec<_>>());
+ assert_eq!(*started.lock().unwrap(), (1..=20).collect::<Vec<_>>());
+}
+
+#[tokio::test(start_paused = true)]
+async fn grouping_keeps_per_key_order_across_next_calls_and_feeds() {
+ let seen = shared(Vec::<(&'static str, i64)>::new());
+ let s = seen.clone();
+ let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()
+ .concurrently_per_group(
+ |e: &(&'static str, i64)| e.0,
+ move |g| {
+ g.sequentially(move |b| {
+ b.pipe(step("record", move |e: (&'static str, i64)| {
+ let s = s.clone();
+ async move {
+ sleep_ms(if e.1 % 2 == 0 { 3 } else { 7 }).await;
+ s.lock().unwrap().push(e);
+ Ok(ok(e))
+ }
+ }))
+ })
+ },
+ Some(2),
+ )
+ .build();
+ pipeline.feed(items([("a", 1), ("b", 1), ("a", 2)]));
+ let first = pipeline.next().await.unwrap();
+ pipeline.feed(items([("a", 3), ("c", 1), ("b", 2), ("a", 4)]));
+ let mut rest = common::consume_all(pipeline.as_ref()).await;
+ rest.extend(
+ first
+ .unwrap()
+ .into_iter()
+ .filter_map(|r| r.result.into_value()),
+ );
+ assert_eq!(rest.len(), 7);
+ let order_of = |key| {
+ seen.lock()
+ .unwrap()
+ .iter()
+ .filter(|e| e.0 == key)
+ .map(|e| e.1)
+ .collect::<Vec<_>>()
+ };
+ assert_eq!(order_of("a"), [1, 2, 3, 4]);
+ assert_eq!(order_of("b"), [1, 2]);
+}
+
+#[tokio::test(start_paused = true)]
+async fn grouping_failure_drains_completed_groups_then_fails_for_good() {
+ let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()
+ .concurrently_per_group(
+ |e: &(&'static str, i64)| e.0,
+ |g| {
+ g.sequentially(|b| {
+ b.pipe(step("maybeFails", |e: (&'static str, i64)| async move {
+ if e.0 == "bad" {
+ sleep_ms(10).await;
+ anyhow::bail!("group failed");
+ }
+ sleep_ms(5).await;
+ Ok(ok(e))
+ }))
+ })
+ },
+ None,
+ )
+ .build();
+ // Only groups that completed before the failure drain ahead of it, as in Node.
+ pipeline.feed(items([("bad", 1), ("good", 1)]));
+ let good = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(good[0].value(), Some(&("good", 1)));
+ assert_eq!(
+ pipeline.next().await.unwrap_err().to_string(),
+ "group failed"
+ );
+ assert_eq!(
+ pipeline.next().await.unwrap_err().to_string(),
+ "group failed"
+ );
+}
+
+#[tokio::test]
+async fn filter_map_emits_non_ok_passthroughs_before_sub_pipeline_results() {
+ #[derive(Debug)]
+ struct Ctx;
+ let pipeline = new_chunk_pipeline::<i64, Ctx>()
+ .pipe_chunk(chunk_step("split", |v: Vec<i64>| async move {
+ Ok(v.into_iter()
+ .map(|n| if n % 2 == 0 { dlq("even", None) } else { ok(n) })
+ .collect())
+ }))
+ .filter_map(
+ |el| el,
+ |b| {
+ b.pipe_chunk(chunk_step("double", |v: Vec<i64>| async move {
+ Ok(v.into_iter().map(|n| ok(n * 2)).collect())
+ }))
+ },
+ )
+ .build();
+ pipeline.feed(vec![
+ create_ok_context(1, Ctx),
+ create_ok_context(2, Ctx),
+ create_ok_context(3, Ctx),
+ ]);
+ let first = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(first.len(), 1);
+ assert_eq!(first[0].result.result_type(), PipelineResultType::Dlq);
+ let second = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(
+ second
+ .iter()
+ .map(|r| *r.value().unwrap())
+ .collect::<Vec<_>>(),
+ [2, 6]
+ );
+ assert!(pipeline.next().await.unwrap().is_none());
+}
+
+#[tokio::test]
+async fn failed_dlq_produce_rejects_the_side_effect_only_when_configured() {
+ for reject in [false, true] {
+ let outputs = MemoryOutputs::new();
+ outputs.fail_output(DLQ_OUTPUT);
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()
+ .pipe_chunk(chunk_step("fail", |v: Vec<i64>| async move {
+ Ok(v.into_iter().map(|_| dlq::<i64>("bad", None)).collect())
+ }))
+ .message_aware(|b| b)
+ .handle_results(PipelineConfig {
+ outputs: Arc::new(outputs.clone()),
+ scheduler: Arc::new(scheduler.clone()),
+ reject_on_dlq_failure: reject,
+ })
+ .handle_side_effects(
+ Arc::new(scheduler.clone()),
+ SideEffectHandlingConfig {
+ await_side_effects: false,
+ },
+ )
+ .build();
+ pipeline.feed(vec![create_ok_context(
+ 1,
+ MsgCtx {
+ message: test_message(None),
+ },
+ )]);
+ pipeline.next().await.unwrap();
+ assert_eq!(scheduler.wait_for_all().await.is_err(), reject);
+ }
+}
+
+#[tokio::test]
+async fn dlq_without_error_uses_the_reason_and_copies_source_headers() {
+ let outputs = MemoryOutputs::new();
+ let scheduler = PromiseScheduler::new();
+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()
+ .pipe_chunk(chunk_step("rejectStep", |v: Vec<i64>| async move {
+ Ok(v.into_iter()
+ .map(|_| dlq::<i64>("plain reason", None))
+ .collect())
+ }))
+ .message_aware(|b| b)
+ .handle_results(PipelineConfig {
+ outputs: Arc::new(outputs.clone()),
+ scheduler: Arc::new(scheduler.clone()),
+ reject_on_dlq_failure: false,
+ })
+ .handle_side_effects(
+ Arc::new(scheduler.clone()),
+ SideEffectHandlingConfig {
+ await_side_effects: true,
+ },
+ )
+ .build();
+ let mut message = test_message(Some(b"k"));
+ message.topic = "events".into();
+ message.partition = 3;
+ message.offset = 42;
+ message.headers = vec![
+ ("token".into(), Some(b"phc_x".to_vec())),
+ ("dlq_step".into(), Some(b"stale".to_vec())),
+ ];
+ pipeline.feed(vec![create_ok_context(1, MsgCtx { message })]);
+ pipeline.next().await.unwrap();
+ let produced = outputs.produced_to(DLQ_OUTPUT);
+ let h = &produced[0].headers;
+ assert_eq!(h.get("token"), Some("phc_x"));
+ assert_eq!(h.get("dlq_reason"), Some("plain reason"));
+ // A source header with the same name keeps its position and takes the new value, as a JS object spread does.
+ assert_eq!(h.0[1], ("dlq_step".to_string(), "rejectStep".to_string()));
+ assert_eq!(h.get("dlq_topic"), Some("events"));
+ assert_eq!(h.get("dlq_partition"), Some("3"));
+ assert_eq!(h.get("dlq_offset"), Some("42"));
+}
+
+#[tokio::test]
+async fn warnings_on_a_timed_out_result_are_dropped_not_emitted() {
+ let outputs = MemoryOutputs::new();
+ let pipeline = new_chunk_pipeline::<i64, common::TeamCtx>()
+ .pipe_chunk(chunk_step("warnThenTimeout", |v: Vec<i64>| async move {
+ Ok(v.into_iter()
+ .map(|n| {
+ let r = if n == 1 { timeout::<i64>("cut") } else { ok(n) };
+ r.with_warning(PipelineWarning::new(
+ "client_ingestion_warning",
+ serde_json::json!({}),
+ ))
+ })
+ .collect())
+ }))
+ .team_aware(|b| b)
+ .handle_ingestion_warnings(Arc::new(
+ ingestion_framework::warning_handling::OutputsWarningEmitter {
+ outputs: Arc::new(outputs.clone()),
+ },
+ ))
+ .handle_side_effects(
+ Arc::new(PromiseScheduler::new()),
+ SideEffectHandlingConfig {
+ await_side_effects: true,
+ },
+ )
+ .build();
+ pipeline.feed(vec![
+ create_ok_context(1, common::TeamCtx { team_id: 1 }),
+ create_ok_context(2, common::TeamCtx { team_id: 1 }),
+ ]);
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert!(results.iter().all(|r| r.context.warnings.is_empty()));
+ assert_eq!(outputs.produced().len(), 1);
+}
+
+#[tokio::test]
+async fn fan_out_dlq_sub_result_fails_the_parent_with_an_aggregated_reason() {
+ let pipeline = new_chunk_pipeline::<Vec<i64>, ()>()
+ .fan_out("split", |v: &Vec<i64>| v.clone())
+ .via(|sub| {
+ sub.sequentially(|b| {
+ b.pipe(step("check", |n: i64| async move {
+ Ok(if n < 0 { dlq("negative", None) } else { ok(n) })
+ }))
+ })
+ })
+ .fan_in("sum", |_v: Vec<i64>, parts: Vec<i64>| {
+ parts.iter().sum::<i64>()
+ })
+ .build();
+ pipeline.feed(items([vec![1, -1, -2]]));
+ let results = pipeline.next().await.unwrap().unwrap();
+ assert_eq!(
+ results[0].result.reason(),
+ Some("2/3 fan-out sub-elements dlq'd: negative")
+ );
+ assert_eq!(results[0].context.last_step(), Some("check"));
+}
+
+#[tokio::test]
+async fn budget_flag_stays_set_after_expiry() {
+ let budget =
+ ingestion_framework::BatchBudget::soft_deadline(Instant::now() - Duration::from_millis(1));
+ assert!(budget.exhausted());
+ assert!(ingestion_framework::BatchBudget::unlimited().is_unlimited());
+ assert!(!ingestion_framework::BatchBudget::unlimited().exhausted());
+ assert_eq!(
+ ingestion_framework::budget::budget_deadline(Instant::now(), 0),
+ None
+ );
+ assert_eq!(
+ ingestion_framework::budget::budget_deadline(Instant::now(), 1 << 31),
+ None
+ );
+}
+
+#[test]
+fn compact_offset_ranges_matches_node() {
+ use ingestion_framework::helpers::compact_offset_ranges;
+ assert_eq!(compact_offset_ranges(&[1, 2, 3, 7, 10, 9]), "1-3,7,9-10");
+ assert_eq!(compact_offset_ranges(&[5, 5, 5]), "5");
+ assert_eq!(compact_offset_ranges(&[]), "");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn batching_with_concurrent_feeds_on_a_multi_thread_runtime_completes_every_batch() {
+ use ingestion_framework::{
+ before_batch_step, new_batching_pipeline, passthrough_after_batch_step,
+ BatchingPipelineOptions, WithBatch,
+ };
+ let pipeline = Arc::new(new_batching_pipeline::<
+ i64,
+ WithBatch<i64, u64>,
+ (),
+ (),
+ u64,
+ u64,
+ >(
+ |b| b.pipe(before_batch_step("before", |id| id)),
+ |b| {
+ b.concurrently_per_group(
+ |v: &WithBatch<i64, u64>| v.value % 7,
+ |g| {
+ g.sequentially(|s| {
+ s.pipe(step("work", |v: WithBatch<i64, u64>| async move {
+ tokio::task::yield_now().await;
+ Ok(ok(v))
+ }))
+ })
+ },
+ Some(3),
+ )
+ },
+ |b| b.pipe(passthrough_after_batch_step("after")),
+ BatchingPipelineOptions {
+ concurrent_batches: 8,
+ },
+ ));
+ let mut feeders = Vec::new();
+ for batch in 0..8u64 {
+ let p = pipeline.clone();
+ feeders.push(tokio::spawn(async move {
+ loop {
+ let r = p
+ .feed(items((0..50).map(|i| i + batch as i64 * 1000)), batch, None)
+ .await
+ .unwrap();
+ if r.is_ok() {
+ break;
+ }
+ tokio::task::yield_now().await;
+ }
+ }));
+ }
+ for f in feeders {
+ f.await.unwrap();
+ }
+ let mut seen = Vec::new();
+ while let Some(done) = pipeline.next().await.unwrap() {
+ assert_eq!(done.elements.len(), 50);
+ assert_eq!(
+ *done.batch_context, done.batch_id,
+ "the batch context is the one built for this batch"
+ );
+ let values: Vec<i64> = done
+ .elements
+ .iter()
+ .map(|e| e.value().unwrap().value)
+ .collect();
+ assert_eq!(
+ values,
+ (0..50)
+ .map(|i| i + done.feed_context as i64 * 1000)
+ .collect::<Vec<_>>(),
+ "feed order kept in a batch"
+ );
+ seen.push(done.feed_context);
+ }
+ seen.sort();
+ assert_eq!(seen, (0..8).collect::<Vec<_>>());
+}