:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion): port the ingestion framework to rust (#127607)
Frank Hamand185bac0d5a2b
diff --git a/nodejs/src/ingestion/framework/benchmarks/framework-bench.ts b/nodejs/src/ingestion/framework/benchmarks/framework-bench.tsnew file mode 100644index 000000000000..f84d29cab24c--- /dev/null+++ b/nodejs/src/ingestion/framework/benchmarks/framework-bench.ts@@ -0,0 +1,214 @@+// The workloads and sizes match `rust/ingestion-framework/benches/framework.rs`, so the two can be compared.+import {+ newBatchingPipeline,+ newChunkPipelineBuilder,+ newPipelineBuilder,+ perElement,+} from '~/ingestion/framework/builders'+import { createOkContext } from '~/ingestion/framework/helpers'+import { isOkResult, ok } from '~/ingestion/framework/results'+import { ProcessingStep } from '~/ingestion/framework/steps'++const SIZES = [1, 100, 10_000]+const MIN_BENCH_MS = Number(process.env.BENCH_MS ?? 2000)++type Ctx = Record<string, never>++function inc(name: string): ProcessingStep<number, number> {+ const step: ProcessingStep<number, number> = (v) => Promise.resolve(ok(v + 1))+ Object.defineProperty(step, 'name', { value: name })+ return step+}++function incChunk(name: string): (values: number[]) => Promise<ReturnType<typeof ok<number>>[]> {+ const step = (values: number[]): Promise<ReturnType<typeof ok<number>>[]> =>+ Promise.resolve(values.map((v) => ok(v + 1)))+ Object.defineProperty(step, 'name', { value: name })+ return step+}++function inputs(n: number): ReturnType<typeof createOkContext<number, Ctx>>[] {+ const out = new Array(n)+ for (let i = 0; i < n; i++) {+ out[i] = createOkContext<number, Ctx>(i, {})+ }+ return out+}++function objectInputs(n: number): ReturnType<typeof createOkContext<{ value: number }, Ctx>>[] {+ const out = new Array(n)+ for (let i = 0; i < n; i++) {+ out[i] = createOkContext<{ value: number }, Ctx>({ value: i }, {})+ }+ return out+}++async function drain(pipeline: { next(): Promise<unknown[] | null> }): Promise<number> {+ let count = 0+ let chunk = await pipeline.next()+ while (chunk !== null) {+ count += chunk.length+ chunk = await pipeline.next()+ }+ return count+}++const stepPipeline = newPipelineBuilder<number, Ctx>()+ .pipe(inc('s1'))+ .pipe(inc('s2'))+ .pipe(inc('s3'))+ .pipe(inc('s4'))+ .pipe(inc('s5'))+ .build()++const WORKLOADS: Record<string, (n: number) => Promise<void>> = {+ step_pipeline_5_steps: async (n) => {+ for (const input of inputs(n)) {+ await stepPipeline.process(input)+ }+ },+ pipe_chunk_5_steps: async (n) => {+ const pipeline = newChunkPipelineBuilder<number, Ctx>()+ .pipeChunk(incChunk('c1'))+ .pipeChunk(incChunk('c2'))+ .pipeChunk(incChunk('c3'))+ .pipeChunk(incChunk('c4'))+ .pipeChunk(incChunk('c5'))+ .build()+ pipeline.feed(inputs(n))+ assertCount(await drain(pipeline), n)+ },+ concurrently_max50: async (n) => {+ const pipeline = newChunkPipelineBuilder<number, Ctx>()+ .concurrently((b) => b.pipe(inc('s1')).pipe(inc('s2')), { maxConcurrency: 50 })+ .build()+ pipeline.feed(inputs(n))+ assertCount(await drain(pipeline), n)+ },+ per_element_max50: async (n) => {+ const pipeline = newChunkPipelineBuilder<number, Ctx>()+ .pipeChunk(perElement(inc('s1'), { maxConcurrency: 50 }))+ .pipeChunk(perElement(inc('s2'), { maxConcurrency: 50 }))+ .build()+ pipeline.feed(inputs(n))+ assertCount(await drain(pipeline), n)+ },+ concurrently_per_group_16: async (n) => {+ const pipeline = newChunkPipelineBuilder<number, Ctx>()+ .concurrentlyPerGroup(+ (v) => v % 16,+ (g) => g.sequentially((b) => b.pipe(inc('s1')).pipe(inc('s2')))+ )+ .build()+ pipeline.feed(inputs(n))+ assertCount(await drain(pipeline), n)+ },+ batching_full: async (n) => {+ // feed() spreads `batchId` into each value, so the values are objects with a numeric field.+ type V = { value: number; batchId?: number }+ const pipeline = newBatchingPipeline<V, V, Ctx>(+ (b) =>+ b.pipe(function before(input) {+ return Promise.resolve(ok({ elements: input.elements, batchContext: input.batchContext }))+ }),+ (b) =>+ b+ .pipeChunk(function prepare(values: V[]) {+ return Promise.resolve(+ values.map((v) => {+ v.value += 1+ return ok(v)+ })+ )+ })+ .concurrentlyPerGroup(+ (v: V) => v.value % 16,+ (g) =>+ g.sequentially((s) =>+ s.pipe(function work(v: V) {+ v.value += 1+ return Promise.resolve(ok(v))+ })+ ),+ { maxConcurrency: 50 }+ )+ .gather(),+ (b) =>+ b.pipe(function after(input) {+ return Promise.resolve(ok(input))+ }),+ { concurrentBatches: 1 }+ )+ const fed = await pipeline.feed(objectInputs(n), {})+ if (!fed.ok) {+ throw new Error(`feed rejected: ${fed.reason}`)+ }+ const done = await pipeline.next()+ assertCount(done!.elements.length, n)+ let sum = 0+ for (const element of done!.elements) {+ if (!isOkResult(element.result)) {+ throw new Error('batching_full produced a non-ok result')+ }+ sum += element.result.value.value+ }+ assertCount(sum, batchingSum(n))+ },+}++// Each input `i` passes two `+ 1` steps, so the outputs sum to `0 + 1 + ... + (n - 1) + 2n`.+function batchingSum(n: number): number {+ return (n * (n - 1)) / 2 + 2 * n+}++function assertCount(actual: number, expected: number): void {+ if (actual !== expected) {+ throw new Error(`expected ${expected} results, got ${actual}`)+ }+}++async function bench(name: string, n: number, run: (n: number) => Promise<void>): Promise<void> {+ const warmupUntil = performance.now() + Math.min(500, MIN_BENCH_MS / 4)+ while (performance.now() < warmupUntil) {+ await run(n)+ }+ let iterations = 0+ const started = process.hrtime.bigint()+ const deadline = performance.now() + MIN_BENCH_MS+ while (performance.now() < deadline) {+ await run(n)+ iterations+++ }+ const elapsedNs = Number(process.hrtime.bigint() - started)+ const nsPerElement = elapsedNs / (iterations * n)+ console.log(+ JSON.stringify({+ impl: 'node',+ workload: name,+ size: n,+ iterations,+ ns_per_element: Math.round(nsPerElement),+ elements_per_sec: Math.round(1e9 / nsPerElement),+ })+ )+}++async function main(): Promise<void> {+ const only = process.argv[2]+ for (const [name, run] of Object.entries(WORKLOADS)) {+ if (only && !name.includes(only)) {+ continue+ }+ for (const n of SIZES) {+ await bench(name, n, run)+ }+ }+}++void main().then(+ () => process.exit(0),+ (error) => {+ console.error(error)+ process.exit(1)+ }+)diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 2e9c7b852ea8..4c45c73a6630 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -6667,6 +6667,28 @@ dependencies = ["uuid",]+[[package]]+name = "ingestion-framework"+version = "0.1.0"+dependencies = [+ "anyhow",+ "async-trait",+ "bytes",+ "chrono",+ "criterion",+ "futures",+ "indexmap 2.13.0",+ "metrics",+ "metrics-exporter-prometheus",+ "rand 0.8.5",+ "serde",+ "serde_json",+ "thiserror 2.0.18",+ "tokio",+ "tracing",+ "tracing-subscriber",+]+[[package]]name = "ingestion-worker-proto"version = "0.1.0"diff --git a/rust/Cargo.toml b/rust/Cargo.tomlindex a06d02b8d485..8576a51fb387 100644--- a/rust/Cargo.toml+++ b/rust/Cargo.toml@@ -60,6 +60,7 @@ members = ["kafka-assigner","kafka-assigner-proto","ingestion-consumer",+ "ingestion-framework","ingestion-control-plane","ingestion-worker-proto","kafka-deduplicator",diff --git a/rust/ingestion-framework/Cargo.toml b/rust/ingestion-framework/Cargo.tomlnew file mode 100644index 000000000000..ffd01a213444--- /dev/null+++ b/rust/ingestion-framework/Cargo.toml@@ -0,0 +1,33 @@+[package]+name = "ingestion-framework"+version = "0.1.0"+edition = "2021"+description = "Rust port of the Node ingestion pipeline framework (nodejs/src/ingestion/framework)"++[lib]+path = "src/lib.rs"++[dependencies]+anyhow = { workspace = true }+async-trait = { workspace = true }+bytes = { workspace = true }+chrono = { workspace = true }+futures = { workspace = true }+indexmap = { version = "2", features = ["serde"] }+metrics = { workspace = true }+rand = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+thiserror = { workspace = true }+tokio = { workspace = true }+tracing = { workspace = true }++[dev-dependencies]+criterion = { version = "0.5", features = ["async_tokio"] }+metrics-exporter-prometheus = { workspace = true }+tokio = { workspace = true, features = ["test-util"] }+tracing-subscriber = { workspace = true }++[[bench]]+name = "framework"+harness = falsediff --git a/rust/ingestion-framework/PARITY.md b/rust/ingestion-framework/PARITY.mdnew file mode 100644index 000000000000..e40b1ce46e5c--- /dev/null+++ b/rust/ingestion-framework/PARITY.md@@ -0,0 +1,136 @@+# Parity with the Node ingestion framework++This crate is a port of `nodejs/src/ingestion/framework`.+The two implementations are kept in step: a change to one lands with the same change to the other, and the mirrored tests below change together.++## File map++| Node (`nodejs/src/ingestion/framework/`) | Rust (`rust/ingestion-framework/src/`) |+| --------------------------------------------- | ------------------------------------------------------------------------------------------ |+| `results.ts` | `results.rs` |+| `pipeline.interface.ts` | `context.rs` (types), `pipeline.rs` (`Pipeline` trait) |+| `chunk-pipeline.interface.ts` | `chunk_pipeline.rs` (`ChunkPipeline` trait), `context.rs` (`ChunkResults`) |+| `steps.ts` | `steps.rs` |+| `start-pipeline.ts`, `step-pipeline.ts` | `pipeline.rs` |+| `branching-pipeline.ts` | `branching.rs` |+| `timeout-barrier-pipeline.ts` | `timeout_barrier.rs` |+| `batch-budget.ts` | `budget.rs` |+| `base-chunk-pipeline.ts` | `chunk_pipeline.rs` (`BaseChunkPipeline`, `apply_chunk_step_to_results`) |+| `buffering-chunk-pipeline.ts` | `chunk_pipeline.rs` (`BufferingChunkPipeline`) |+| `sequential-chunk-pipeline.ts` | `chunk_pipeline.rs` (`SequentialChunkPipeline`) |+| `gathering-chunk-pipeline.ts` | `chunk_pipeline.rs` (`GatheringChunkPipeline`) |+| `chunk-pipeline-unwrapper.ts` | `chunk_pipeline.rs` (`ChunkPipelineUnwrapper`) |+| `concurrent-chunk-pipeline.ts` | `concurrent.rs` |+| `concurrently-grouping-chunk-pipeline.ts` | `grouping.rs` |+| `filter-map-chunk-pipeline.ts` | `filter_map.rs` |+| `fan-out-fan-in-chunk-pipeline.ts` | `fan_out_fan_in.rs` |+| `interleaving-chunk-pipeline.ts` | `interleaving.rs` |+| `resettable-signal.ts` | `sync.rs` (`ResettableSignal`); `sync.rs` (`FifoLimiter`) replaces `p-limit` |+| `result-handling-pipeline.ts` | `result_handling.rs` |+| `result-handling-helpers.ts` | `result_handling.rs` (`produce_message_to_dlq`, `redirect_message_to_output`, headers) |+| `side-effect-handling-pipeline.ts` | `side_effect_handling.rs` |+| `ingestion-warning-handling-chunk-pipeline.ts`| `warning_handling.rs` |+| `per-element-step.ts` | `per_element.rs` (`per_element`, also exported from `builders.rs`) |+| `retry.ts` (+ `retryIfRetriable`) | `retry.rs` |+| `batching-pipeline.ts` | `batching.rs` |+| `metrics.ts` | `metrics.rs` (same metric names and labels) |+| `helpers.ts` | `helpers.rs` |+| `builders/pipeline-builders.ts` | `builders.rs` (`new_pipeline`, `PipelineBuilder`, `BranchingPipelineBuilder`) |+| `builders/chunk-pipeline-builders.ts` | `builders.rs` (`new_chunk_pipeline`, `ChunkPipelineBuilder`, group/fan-out/result builders)|+| `builders/helpers.ts` | `builders.rs` (`new_batching_pipeline`) |+| `~/common/outputs` (`IngestionOutputs`) | `outputs.rs` (`IngestionOutputs` trait, `KafkaMessage`, `Headers`, `MemoryOutputs`) |+| `~/common/utils/promise-scheduler` | `side_effects.rs` (`PromiseScheduler`, `SideEffectScheduler`) |+| `~/ingestion/common/ingestion-warnings.ts` (`serializeIngestionWarning`, `emitIngestionWarning`) | `warning_handling.rs` (`serialize_ingestion_warning`, `OutputsWarningEmitter`), `js_json.rs` (`JSON.stringify` output) |+| `~/ingestion/common/ingestion-warning-types.ts` (`INGESTION_WARNING_TYPES`) | `warning_handling.rs` (`warning_classification`), read from `rust/common/ingestion_warnings/warning_types.generated.json`, which the Node registry exports |+| `span-attributes.ts`, `tophog/`, `extensions/`| not ported (see below) |++## Mirrored tests++Each living-docs chapter `docs/NN-*.test.ts` has a Rust twin `tests/docs_NN_*.rs`, with one `#[tokio::test]` per Node `it(...)`.+The Rust test name is the snake_case form of the Node test name.+Jest fake timers map to `#[tokio::test(start_paused = true)]`: tokio advances the paused clock by itself when every task waits, and `tokio::time::advance` replaces a synchronous `jest.advanceTimersByTime`.++`per-element-step.test.ts` has the twin `tests/per_element_step.rs`.++`tests/unit_semantics.rs` pins behavior the chapters leave implicit: poison-then-drain for the concurrent and grouping stages, feed-while-parked, FIFO admission under `max_concurrency`, filter-map passthrough order, DLQ header layout, DLQ rethrow, warnings on timeouts, fan-out DLQ aggregation, and concurrent batch feeds on a multi-thread runtime.++`tests/review_regressions.rs` pins divergences that a review found and that are now fixed: the ingestion warning bytes, the fan-out `dlq_reason`, produce order on a multi-thread runtime, the redirect produce before `next()` returns, a dropped batching `next()`, scheduler cleanup, `tries: 0`, top-level retriability, a fan-in panic, `max_concurrency: Some(0)`, the cost of a group start, overlapping and cancelled scheduler waits, the parents a fan-in panic leaves ready, a group processor that panics before it returns its future, a dropped limiter `acquire()`, and retry options that give an out-of-range sleep.++`consumeAll(pipeline, ms)` and `jest.advanceTimersByTimeAsync(ms)` before an `await` map to `consume_all_within` and `within_ms` in `tests/common`, which fail the test when the work needs more virtual time than Node gives it.++### Skipped cases++| Node test | Why |+| ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| `docs/16-tophog-metrics.test.ts` (all four cases) | TopHog (`framework/tophog/`) is a separate metrics aggregation subsystem, not part of the pipeline composition this crate ports. Port it as its own module if a Rust pipeline needs it. |++Everything else is mirrored. Where a Node assertion relies on a JS runtime detail, the Rust test checks the same observable behavior another way:++- `13-conventions` "pipelines are created via factory functions" uses `Arc::ptr_eq` for `not.toBe`.+- `07-result-handling` "REDIRECT with awaitAck=true" checks that `next()` does not finish within 50 ms while the ack is held, instead of `setImmediate`.+- `08-side-effects` "steps can add side effects" checks that the side effect ran before the step result is awaited, because `SideEffect::spawn` runs the future up to its first pending point, as a Node promise executor runs.++## Known differences++These are deliberate. Each one follows from a language difference. Items 1 to 15 do not change what a pipeline emits; items 16 to 19 change only timing or a failure path.++1. **Step names are explicit.** Node reads `fn.name`. Rust closures have no name, so `step("name", f)` and `chunk_step("name", f)` take one. Use the Node function name.+2. **A step error is `Err`, not a throw.** `PipelineError::retriable` / `non_retriable` stand in for `error.isRetriable`. An error with neither flag behaves like a Node error with no `isRetriable`: retried, then propagated. `is_retriable` reads the outermost error only, as Node reads the property of the thrown object: `PipelineError::non_retriable(..).context(..)` has no flag, so it retries. A `SharedError` reads through, because it is the same error object shared by more than one caller.+3. **Side effects start on creation.** A Node promise runs its executor (or an async function its body up to the first `await`) when it is created; a Rust future runs only when polled. `SideEffect::spawn` polls the future once on the caller's thread and moves the unfinished rest to a tokio task, so DLQ, redirect and warning produces are issued in item order inside `next()`, as in Node. A panic in that first poll becomes a failed side effect, as a throw before the first `await` rejects a promise. A `SideEffect` cannot be cloned, so a step's side effects and warnings move from the result into the context (Node keeps a copy on both).+4. **`ResultWithContext.result` is an `Outcome`.** Side effects and warnings live only on the context once a step finishes.+5. **The user context is a field.** Node spreads `C` into the context object; Rust stores it in `PipelineContext.data` with `Deref`. `message_id` (Node's `BatchingContext`) is a base context field, so batching does not change the context type. `traceContext` is not ported.+6. **Structural context access is a trait.** `message_aware`/`handle_results` require `HasMessage`; `team_aware`/`handle_ingestion_warnings` require `HasTeamId`.+7. **`filter_map` output context.** Node types it as the union `CSubOutput | CIntermediate`. Rust requires `CSubOutput: From<CIntermediate>` and converts non-OK passthroughs with it.+8. **Batch context is not spread into values.** Node's sub-pipeline values are `TInput & CBatch`. Rust's are `WithBatch<T, CB>` (value plus `Arc<CB>`, with `Deref` to the value). The caller's feed context returns as `BatchResult.feed_context` rather than merged into `batch_context`.+9. **Retries clone the input.** `pipe_with_retry` needs `T: Clone + Sync`, because each attempt takes the value by value. Every attempt but the last possible one gets a clone, so `tries: 1` clones nothing; the first of several attempts cannot take the original, because a retry needs it again. Node passes the same reference.+10. **Compile-time checks that become run-time.** Node checks branch exhaustiveness, redirect output names and warning types in the type system. Rust uses string names, so an unknown branch is the run-time DLQ that Node also has, an unknown output fails in the `IngestionOutputs` implementation, and an unregistered warning type fails its side effect, as Node's registry lookup throws at run time.+11. **Ingestion warnings.** The Node `IngestionWarningLimiter` is an app concern: `IngestionWarningEmitter` is the seam, and `OutputsWarningEmitter` emits with no limiter. `serialize_ingestion_warning` writes the Node bytes with `js_json` (key order, `JSON.stringify` numbers and escapes, array-index keys first), with no dependence on serde_json's `preserve_order` feature. `category` and `severity` come from the registry JSON that the Node registry exports. `PipelineWarning.details` is an `IndexMap`, so its keys keep insertion order; build ordered details with `with_detail`, because a `json!` object sorts its keys unless the build enables `preserve_order`. A nested object inside a detail value is a `serde_json::Map` and is written in that map's order, which is sorted in a build without `preserve_order`; Node keeps insertion order there too. Keep details flat where the bytes must match.+12. **Fan-out DLQ error.** Node builds `AggregateError(errors, reason)`; Rust builds `errors::AggregateError` with the same message and every sub error. The `dlq_reason` header is identical.+13. **Concurrency.** Concurrent items and groups run as tokio tasks, so on a multi-thread runtime they run in parallel; Node interleaves them on one thread. Ordering guarantees (FIFO emission, per-key order, FIFO admission under a cap) are the same. When a group completes, the queues still waiting all belong to running groups, so their relative order no longer matters and the remove is O(1). The interleaving stage keeps the pending sub-pipeline pull as a stored boxed future so it survives across `next()` calls, which is how Node keeps its memoized promise. That future is polled only while a `next()` is awaited. Item and group work runs in its own tasks and never pauses, but sub-pipeline stages that the pull itself drives (for example a `sequentially` inside `filter_map`) advance only while the caller waits on `next()`. A Node promise advances in the background. This changes timing, not results or order; every consumer loops on `next()` anyway. A spawned task per `next()` was measured first and cost a cross-thread wake per emitted chunk.+14. **Clock.** `BatchBudget` reads `tokio::time::Instant`, so it follows a paused test clock as Node's follows `Date.now()` under fake timers.+15. **Not ported:** `span-attributes.ts` and OTel span parenting (Rust uses `tracing`), `tophog/`, and `extensions/`.+16. **Cancel safety.** Node has no cancellation. `BatchingPipeline::next()` is cancel-safe: it keeps its sub-pipeline pull, the results it has not routed, and a running afterBatch hook under its pump lock, and the next call resumes them, so a dropped call (for example from an HTTP handler whose client went away) loses no batch and runs no hook twice. A `ChunkPipeline::next()` is not cancel-safe: the interleaving stages keep their pull, but a stage such as `pipe_chunk`, result handling or side-effect handling holds the chunk in the dropped future. Do not race a chunk pipeline's `next()` against a timeout. A dropped batching `next()` also leaves the hook paused until the next call polls it, where Node's would run on.+17. **`PromiseScheduler` keeps the first failure.** Node removes a promise when it settles, so a rejection that settles before `waitForAll()` is not seen there. The Rust scheduler keeps it, as the `FailureLatchingPromiseScheduler` of the Node metrics consumer does, because a consumer that misses it stores the offsets of a batch whose side effect failed. A settled side effect leaves the set at the next `schedule()` that finds the set grown to twice its size after the last cleanup, at `pending_count()`, or when a `wait_for_all()` sees it settle. As in Node, a side effect stays in the set until it settles, so an overlapping `wait_for_all()` also waits for it, and a cancelled one leaves it, and its failure, for the next call. `wait_for_all()` waits for every side effect pending at the call and then returns the first failure; Node's `Promise.all` rejects at the first failure without waiting for the rest.+18. **User callbacks that panic.** A panic in a grouping function, fan-out function or fan-in function becomes an error from `next()`, as a Node throw rejects `next()` and poisons the stage. No lock is held while user code runs. `max_concurrency: Some(0)` panics when the stage is built, as `pLimit(0)` throws.+19. **`per_element` rejects late.** Node's `Promise.all` rejects at the first failed step, and the other steps run on. Rust waits for the other steps and then returns the first error to occur.++## Benchmarks++`benches/framework.rs` (criterion) and `nodejs/src/ingestion/framework/benchmarks/framework-bench.ts` run the same six workloads at sizes 1, 100 and 10 000, with trivial steps, so the numbers are framework overhead per element. The Rust benchmark installs a Prometheus recorder, so both sides pay for their metrics.++```bash+cd rust && cargo bench -p ingestion-framework --bench framework+```++```bash+cd nodejs && DATABASE_URL=postgres://localhost/bench pnpm exec tsx src/ingestion/framework/benchmarks/framework-bench.ts+```++### Measured overhead++One 4-vCPU cloud container, the two runs one after the other. Mean ns per element; lower is better. Rust numbers are on a current-thread runtime, which matches the single Node event loop.++| Workload | Size | Node (ns/elem) | Rust (ns/elem) | Node ÷ Rust |+| -------------------------------- | ------ | -------------- | -------------- | ----------- |+| `step_pipeline_5_steps` | 1 | 30 383 | 2 379 | 12.8× |+| `step_pipeline_5_steps` | 100 | 17 827 | 2 349 | 7.6× |+| `step_pipeline_5_steps` | 10 000 | 19 693 | 2 307 | 8.5× |+| `pipe_chunk_5_steps` | 1 | 25 940 | 3 483 | 7.4× |+| `pipe_chunk_5_steps` | 100 | 722 | 431 | 1.7× |+| `pipe_chunk_5_steps` | 10 000 | 740 | 541 | 1.4× |+| `concurrently_max50` | 1 | 24 758 | 3 613 | 6.9× |+| `concurrently_max50` | 100 | 8 488 | 1 873 | 4.5× |+| `concurrently_max50` | 10 000 | 10 007 | 2 187 | 4.6× |+| `per_element_max50` | 1 | 20 834 | 2 399 | 8.7× |+| `per_element_max50` | 100 | 1 912 | 1 118 | 1.7× |+| `per_element_max50` | 10 000 | 4 108 | 2 248 | 1.8× |+| `concurrently_per_group_16` | 1 | 14 492 | 5 165 | 2.8× |+| `concurrently_per_group_16` | 100 | 7 470 | 1 302 | 5.7× |+| `concurrently_per_group_16` | 10 000 | 8 586 | 1 091 | 7.9× |+| `batching_full` | 1 | 53 331 | 8 726 | 6.1× |+| `batching_full` | 100 | 7 091 | 1 454 | 4.9× |+| `batching_full` | 10 000 | 6 868 | 1 400 | 4.9× |++On a 4-worker multi-thread runtime, the 10 000-element workloads run at 1 885 (`concurrently_max50`), 947 (`concurrently_per_group_16`) and 1 798 (`batching_full`) ns per element. Single-element batches cost about 50 µs there, which is worker wake-up latency, so a current-thread runtime suits a consumer that feeds tiny batches.++The Node per-step cost is dominated by the per-step instrumentation (`instrumentFn`, the prom-client step histogram); the Rust port records the same histogram through the `metrics` crate.diff --git a/rust/ingestion-framework/benches/framework.rs b/rust/ingestion-framework/benches/framework.rsnew file mode 100644index 000000000000..7e82cdaacb79--- /dev/null+++ b/rust/ingestion-framework/benches/framework.rs@@ -0,0 +1,257 @@+//! Workloads and sizes match `framework-bench.ts`. A current-thread runtime matches the single Node event loop.++use std::sync::Arc;++use criterion::{criterion_group, BenchmarkId, Criterion, Throughput};+use ingestion_framework::{+ before_batch_step, chunk_step, create_ok_context, new_batching_pipeline, new_chunk_pipeline,+ new_pipeline, ok, passthrough_after_batch_step, per_element, step, BatchingPipelineOptions,+ ChunkProcessingStep, DynChunkPipeline, OkResultWithContext, Outcome, ProcessingStep, WithBatch,+};++const SIZES: [usize; 3] = [1, 100, 10_000];++fn inc(name: &str) -> ProcessingStep<i64, i64> {+ ProcessingStep::sync(name, |v: i64| Ok(ok(v + 1)))+}++fn inc_chunk(name: &str) -> ChunkProcessingStep<i64, i64> {+ ChunkProcessingStep::sync(name, |v: Vec<i64>| {+ Ok(v.into_iter().map(|x| ok(x + 1)).collect())+ })+}++fn inputs(n: usize) -> Vec<OkResultWithContext<i64, ()>> {+ (0..n as i64).map(|v| create_ok_context(v, ())).collect()+}++fn current_thread() -> tokio::runtime::Runtime {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+}++fn multi_thread() -> tokio::runtime::Runtime {+ tokio::runtime::Builder::new_multi_thread()+ .worker_threads(4)+ .enable_all()+ .build()+ .unwrap()+}++async fn drain(pipeline: &DynChunkPipeline<i64, i64, (), ()>) -> usize {+ let mut n = 0;+ while let Some(chunk) = pipeline.next().await.unwrap() {+ n += chunk.len();+ }+ n+}++fn step_pipeline_5_steps(c: &mut Criterion) {+ let rt = current_thread();+ let pipeline = new_pipeline::<i64, ()>()+ .pipe(inc("s1"))+ .pipe(inc("s2"))+ .pipe(inc("s3"))+ .pipe(inc("s4"))+ .pipe(inc("s5"))+ .build();+ let mut group = c.benchmark_group("step_pipeline_5_steps");+ for n in SIZES {+ group.throughput(Throughput::Elements(n as u64));+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {+ b.to_async(&rt).iter(|| {+ let pipeline = pipeline.clone();+ async move {+ for input in inputs(n) {+ pipeline.process(input).await.unwrap();+ }+ }+ })+ });+ }+ group.finish();+}++fn pipe_chunk_chain(c: &mut Criterion) {+ let rt = current_thread();+ let mut group = c.benchmark_group("pipe_chunk_5_steps");+ for n in SIZES {+ group.throughput(Throughput::Elements(n as u64));+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {+ b.to_async(&rt).iter(|| async move {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(inc_chunk("c1"))+ .pipe_chunk(inc_chunk("c2"))+ .pipe_chunk(inc_chunk("c3"))+ .pipe_chunk(inc_chunk("c4"))+ .pipe_chunk(inc_chunk("c5"))+ .build();+ pipeline.feed(inputs(n));+ assert_eq!(drain(&pipeline).await, n);+ })+ });+ }+ group.finish();+}++fn concurrently(c: &mut Criterion) {+ for (label, rt) in [+ ("concurrently_max50", current_thread()),+ ("concurrently_max50_multi_thread", multi_thread()),+ ] {+ let mut group = c.benchmark_group(label);+ for n in SIZES {+ group.throughput(Throughput::Elements(n as u64));+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {+ b.to_async(&rt).iter(|| async move {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently(|b| b.pipe(inc("s1")).pipe(inc("s2")), Some(50))+ .build();+ pipeline.feed(inputs(n));+ assert_eq!(drain(&pipeline).await, n);+ })+ });+ }+ group.finish();+ }+}++fn per_element_max50(c: &mut Criterion) {+ let rt = current_thread();+ let mut group = c.benchmark_group("per_element_max50");+ for n in SIZES {+ group.throughput(Throughput::Elements(n as u64));+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {+ b.to_async(&rt).iter(|| async move {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(per_element(inc("s1"), Some(50)))+ .pipe_chunk(per_element(inc("s2"), Some(50)))+ .build();+ pipeline.feed(inputs(n));+ assert_eq!(drain(&pipeline).await, n);+ })+ });+ }+ group.finish();+}++fn concurrently_per_group(c: &mut Criterion) {+ for (label, rt) in [+ ("concurrently_per_group_16", current_thread()),+ ("concurrently_per_group_16_multi_thread", multi_thread()),+ ] {+ let mut group = c.benchmark_group(label);+ for n in SIZES {+ group.throughput(Throughput::Elements(n as u64));+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {+ b.to_async(&rt).iter(|| async move {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently_per_group(+ |v: &i64| v % 16,+ |g| g.sequentially(|b| b.pipe(inc("s1")).pipe(inc("s2"))),+ None,+ )+ .build();+ pipeline.feed(inputs(n));+ assert_eq!(drain(&pipeline).await, n);+ })+ });+ }+ group.finish();+ }+}++type V = WithBatch<i64, u64>;++fn batching(c: &mut Criterion) {+ for (label, rt) in [+ ("batching_full", current_thread()),+ ("batching_full_multi_thread", multi_thread()),+ ] {+ let mut group = c.benchmark_group(label);+ for n in SIZES {+ group.throughput(Throughput::Elements(n as u64));+ group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, &n| {+ b.to_async(&rt).iter(|| async move {+ let pipeline = Arc::new(new_batching_pipeline::<i64, V, (), (), u64, ()>(+ |b| b.pipe(before_batch_step("before", |id| id)),+ |b| {+ b.pipe_chunk(chunk_step("prepare", |v: Vec<V>| async move {+ Ok(v.into_iter()+ .map(|mut x| {+ x.value += 1;+ ok(x)+ })+ .collect())+ }))+ .concurrently_per_group(+ |v: &V| v.value % 16,+ |g| {+ g.sequentially(|s| {+ s.pipe(step("work", |mut v: V| async move {+ v.value += 1;+ Ok(ok(v))+ }))+ })+ },+ Some(50),+ )+ .gather()+ },+ |b| b.pipe(passthrough_after_batch_step("after")),+ BatchingPipelineOptions::default(),+ ));+ assert!(pipeline.feed(inputs(n), (), None).await.unwrap().is_ok());+ let done = pipeline.next().await.unwrap().unwrap();+ assert_eq!(done.elements.len(), n);+ // Each input `i` passes two `+ 1` steps, as in the Node workload.+ let sum: i64 = done+ .elements+ .iter()+ .map(|e| match &e.result {+ Outcome::Ok(v) => v.value,+ Outcome::NonOk(_) => panic!("batching_full produced a non-ok result"),+ })+ .sum();+ let n = n as i64;+ assert_eq!(sum, n * (n - 1) / 2 + 2 * n);+ })+ });+ }+ group.finish();+ }+}++criterion_group!(+ benches,+ step_pipeline_5_steps,+ pipe_chunk_chain,+ concurrently,+ per_element_max50,+ concurrently_per_group,+ batching+);+/// Without a recorder every metric call is a no-op, while Node updates real `prom-client` metrics.+fn install_metrics_recorder() {+ const BUCKETS: &[f64] = &[+ 1.0, 5.0, 10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0,+ ];+ let handle = metrics_exporter_prometheus::PrometheusBuilder::new()+ .set_buckets(BUCKETS)+ .expect("buckets are not empty")+ .install_recorder()+ .expect("no other recorder is installed");+ // Histogram samples stay buffered until upkeep drains them, as the production exporter does.+ std::thread::spawn(move || loop {+ std::thread::sleep(std::time::Duration::from_secs(1));+ handle.run_upkeep();+ });+}++fn main() {+ install_metrics_recorder();+ benches();+ Criterion::default().configure_from_args().final_summary();+}diff --git a/rust/ingestion-framework/src/batching.rs b/rust/ingestion-framework/src/batching.rsnew file mode 100644index 000000000000..c2219ccbb961--- /dev/null+++ b/rust/ingestion-framework/src/batching.rs@@ -0,0 +1,458 @@+//! Node spreads the batch context into each value. Here a value is `WithBatch<T, CB>`, and the feed context is a `BatchResult` field.++use std::collections::{HashMap, HashSet, VecDeque};+use std::ops::{Deref, DerefMut};+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{Arc, Mutex};++use futures::future::BoxFuture;+use futures::FutureExt;++use crate::budget::BatchBudget;+use crate::chunk_pipeline::DynChunkPipeline;+use crate::context::{create_ok_context, ChunkResults, OkResultWithContext, ResultWithContext};+use crate::metrics::{BATCH_BUDGET_EXHAUSTED_COUNTER, BATCH_BUDGET_OVERRUN_HISTOGRAM};+use crate::pipeline::DynPipeline;+use crate::results::Outcome;+use crate::side_effects::SideEffect;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum FeedRejectionKind {+ AtCapacity,+ BeforeBatchFailed,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub enum FeedResult {+ Ok,+ Rejected {+ kind: FeedRejectionKind,+ reason: String,+ },+}++impl FeedResult {+ pub fn is_ok(&self) -> bool {+ matches!(self, FeedResult::Ok)+ }+}++#[derive(Debug)]+pub struct WithBatch<T, CB> {+ pub value: T,+ pub batch: Arc<CB>,+}++impl<T, CB> Deref for WithBatch<T, CB> {+ type Target = T;+ fn deref(&self) -> &T {+ &self.value+ }+}++impl<T, CB> DerefMut for WithBatch<T, CB> {+ fn deref_mut(&mut self) -> &mut T {+ &mut self.value+ }+}++impl<T: Clone, CB> Clone for WithBatch<T, CB> {+ fn clone(&self) -> Self {+ Self {+ value: self.value.clone(),+ batch: self.batch.clone(),+ }+ }+}++pub struct BeforeBatchInput<T, CI> {+ pub elements: Vec<OkResultWithContext<T, CI>>,+ pub batch_id: u64,+}++/// beforeBatch must return exactly as many elements as it received, or `feed()` fails.+pub struct BeforeBatchOutput<T, CI, CB> {+ pub elements: Vec<OkResultWithContext<T, CI>>,+ pub batch_context: CB,+ pub batch_id: u64,+}++pub struct AfterBatchInput<O, CO, CB> {+ pub elements: ChunkResults<O, CO>,+ pub batch_context: Arc<CB>,+ pub batch_id: u64,+}++pub type AfterBatchOutput<O, CO, CB> = AfterBatchInput<O, CO, CB>;++pub struct BatchResult<O, CO, CB, CF> {+ pub elements: ChunkResults<O, CO>,+ pub batch_context: Arc<CB>,+ pub feed_context: CF,+ pub batch_id: u64,+ pub side_effects: Vec<SideEffect>,+}++#[derive(Debug, Clone, Copy)]+pub struct BatchingPipelineOptions {+ pub concurrent_batches: usize,+}++impl Default for BatchingPipelineOptions {+ fn default() -> Self {+ Self {+ concurrent_batches: 1,+ }+ }+}++// The overrun includes afterBatch, because the checkpoints only stop work from starting.+fn record_budget_overrun(budget: &BatchBudget) {+ if !budget.exhausted() {+ return;+ }+ metrics::counter!(BATCH_BUDGET_EXHAUSTED_COUNTER).increment(1);+ if let Some(at) = budget.soft_at() {+ let overrun = tokio::time::Instant::now().saturating_duration_since(at);+ metrics::histogram!(BATCH_BUDGET_OVERRUN_HISTOGRAM).record(overrun.as_secs_f64());+ }+}++struct TrackedBatch<O, CO, CB, CF> {+ batch_context: Arc<CB>,+ feed_context: Option<CF>,+ budget: BatchBudget,+ message_ids: Vec<u64>,+ inflight: HashSet<u64>,+ results: HashMap<u64, ResultWithContext<O, CO>>,+ before_side_effects: Vec<SideEffect>,+}++struct BatchingState<O, CO, CB, CF> {+ next_batch_id: u64,+ next_message_id: u64,+ batches: HashMap<u64, TrackedBatch<O, CO, CB, CF>>,+ message_to_batch: HashMap<u64, u64>,+ completed: VecDeque<BatchResult<O, CO, CB, CF>>,+}++pub struct BatchingPipeline<T, O, CI, CO, CB, CF> {+ sub: DynChunkPipeline<WithBatch<T, CB>, O, CI, CO>,+ before: DynPipeline<BeforeBatchInput<T, CI>, BeforeBatchOutput<T, CI, CB>, ()>,+ after: DynPipeline<AfterBatchInput<O, CO, CB>, AfterBatchOutput<O, CO, CB>, ()>,+ options: BatchingPipelineOptions,+ state: Mutex<BatchingState<O, CO, CB, CF>>,+ // Bumped where a batch is fed, so the pump can tell a feed that raced its pull from lost messages.+ feed_epoch: AtomicU64,+ // The stages assume one caller. tokio's Mutex is FIFO-fair, like `pLimit(1)`.+ feed_lock: tokio::sync::Mutex<()>,+ // Pump progress lives here, not in the `next()` future, so a dropped `next()` loses nothing.+ pump: tokio::sync::Mutex<PumpState<O, CO, CB, CF>>,+}++type SubPull<O, CO> = BoxFuture<'static, anyhow::Result<Option<ChunkResults<O, CO>>>>;+type AfterRun<O, CO, CB> =+ BoxFuture<'static, anyhow::Result<ResultWithContext<AfterBatchOutput<O, CO, CB>, ()>>>;++struct PendingPull<O, CO> {+ epoch: u64,+ future: SubPull<O, CO>,+}++struct RunningAfter<O, CO, CB, CF> {+ batch_id: u64,+ batch: TrackedBatch<O, CO, CB, CF>,+ future: AfterRun<O, CO, CB>,+}++struct PumpState<O, CO, CB, CF> {+ pull: Option<PendingPull<O, CO>>,+ unrouted: VecDeque<ResultWithContext<O, CO>>,+ after: Option<RunningAfter<O, CO, CB, CF>>,+}++impl<T, O, CI, CO, CB, CF> BatchingPipeline<T, O, CI, CO, CB, CF>+where+ T: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+ CB: Send + Sync + 'static,+ CF: Send + 'static,+{+ pub fn new(+ sub: DynChunkPipeline<WithBatch<T, CB>, O, CI, CO>,+ before: DynPipeline<BeforeBatchInput<T, CI>, BeforeBatchOutput<T, CI, CB>, ()>,+ after: DynPipeline<AfterBatchInput<O, CO, CB>, AfterBatchOutput<O, CO, CB>, ()>,+ options: BatchingPipelineOptions,+ ) -> Self {+ Self {+ sub,+ before,+ after,+ options,+ state: Mutex::new(BatchingState {+ next_batch_id: 0,+ next_message_id: 0,+ batches: HashMap::new(),+ message_to_batch: HashMap::new(),+ completed: VecDeque::new(),+ }),+ feed_epoch: AtomicU64::new(0),+ feed_lock: tokio::sync::Mutex::new(()),+ pump: tokio::sync::Mutex::new(PumpState {+ pull: None,+ unrouted: VecDeque::new(),+ after: None,+ }),+ }+ }++ pub async fn feed(+ &self,+ elements: Vec<OkResultWithContext<T, CI>>,+ feed_context: CF,+ budget: Option<BatchBudget>,+ ) -> anyhow::Result<FeedResult> {+ let _guard = self.feed_lock.lock().await;+ let budget = budget.unwrap_or_else(BatchBudget::unlimited);++ // An empty feed could never complete, so it registers nothing and runs no hooks.+ if elements.is_empty() {+ return Ok(FeedResult::Ok);+ }++ let batch_id = {+ let mut state = self.state.lock().unwrap();+ if state.batches.len() >= self.options.concurrent_batches {+ return Ok(FeedResult::Rejected {+ kind: FeedRejectionKind::AtCapacity,+ reason: format!(+ "at concurrent batch capacity ({})",+ self.options.concurrent_batches+ ),+ });+ }+ let id = state.next_batch_id;+ state.next_batch_id += 1;+ id+ };++ let fed_count = elements.len();+ let before = self+ .before+ .process(create_ok_context(+ BeforeBatchInput { elements, batch_id },+ (),+ ))+ .await?;+ let before_side_effects;+ let output = match before.result {+ Outcome::Ok(output) => {+ before_side_effects = before.context.side_effects;+ output+ }+ Outcome::NonOk(_) => {+ return Ok(FeedResult::Rejected {+ kind: FeedRejectionKind::BeforeBatchFailed,+ reason: format!("beforeBatch hook returned non-ok result for batch {batch_id}"),+ })+ }+ };++ // A changed count leaves a batch that never completes, so fail before registering it.+ if output.elements.len() != fed_count {+ anyhow::bail!(+ "batching_pipeline beforeBatch changed element count ({} -> {}) for batch {}",+ fed_count,+ output.elements.len(),+ batch_id+ );+ }++ let batch_context = Arc::new(output.batch_context);+ let mut state = self.state.lock().unwrap();+ let mut message_ids = Vec::with_capacity(fed_count);+ let mut inflight = HashSet::with_capacity(fed_count);+ let tagged: Vec<_> = output+ .elements+ .into_iter()+ .map(|element| {+ let message_id = state.next_message_id;+ state.next_message_id += 1;+ message_ids.push(message_id);+ inflight.insert(message_id);+ state.message_to_batch.insert(message_id, batch_id);+ let mut context = element.context;+ context.message_id = Some(message_id);+ context.budget = Some(budget.clone());+ OkResultWithContext {+ value: WithBatch {+ value: element.value,+ batch: batch_context.clone(),+ },+ context,+ }+ })+ .collect();+ state.batches.insert(+ batch_id,+ TrackedBatch {+ batch_context,+ feed_context: Some(feed_context),+ budget,+ message_ids,+ inflight,+ results: HashMap::new(),+ before_side_effects,+ },+ );+ self.sub.feed(tagged);+ // Bump after buffering, so a pump that reads the new epoch also finds the elements.+ self.feed_epoch.fetch_add(1, Ordering::SeqCst);+ Ok(FeedResult::Ok)+ }++ /// Cancel-safe: the next call resumes a dropped call's pull, results and afterBatch run.+ pub async fn next(&self) -> anyhow::Result<Option<BatchResult<O, CO, CB, CF>>> {+ let mut pump = self.pump.lock().await;+ self.pump(&mut pump).await+ }++ async fn pump(+ &self,+ pump: &mut PumpState<O, CO, CB, CF>,+ ) -> anyhow::Result<Option<BatchResult<O, CO, CB, CF>>> {+ loop {+ // Finish an afterBatch run first. A dropped call may have left one.+ if let Some(running) = pump.after.as_mut() {+ let after = running.future.as_mut().await;+ let running = pump.after.take().expect("present");+ self.complete_batch(running, after)?;+ }++ while let Some(result) = pump.unrouted.pop_front() {+ if let Some(running) = self.route(result)? {+ pump.after = Some(running);+ break;+ }+ }+ if pump.after.is_some() {+ continue;+ }++ // Node returns a completed batch only after it routed the whole chunk.+ if let Some(done) = self.state.lock().unwrap().completed.pop_front() {+ return Ok(Some(done));+ }++ if pump.pull.is_none() {+ let sub = self.sub.clone();+ pump.pull = Some(PendingPull {+ epoch: self.feed_epoch.load(Ordering::SeqCst),+ future: async move { sub.next().await }.boxed(),+ });+ }+ let pulled = pump.pull.as_mut().expect("present").future.as_mut().await;+ let epoch_at_pull = pump.pull.take().expect("present").epoch;+ let Some(results) = pulled? else {+ let (batches, messages) = {+ let state = self.state.lock().unwrap();+ (state.batches.len(), state.message_to_batch.len())+ };+ if batches > 0 {+ // A feed that landed during an empty pull is picked up by a retry.+ if self.feed_epoch.load(Ordering::SeqCst) != epoch_at_pull {+ continue;+ }+ anyhow::bail!(+ "batching_pipeline sub-pipeline returned null with {batches} in-flight batches and {messages} in-flight messages"+ );+ }+ return Ok(None);+ };+ pump.unrouted.extend(results);+ }+ }++ fn route(+ &self,+ result: ResultWithContext<O, CO>,+ ) -> anyhow::Result<Option<RunningAfter<O, CO, CB, CF>>> {+ let mut state = self.state.lock().unwrap();+ let message_id = result.context.message_id.ok_or_else(|| {+ anyhow::anyhow!("batching_pipeline received result without messageId")+ })?;+ let Some(batch_id) = state.message_to_batch.get(&message_id).copied() else {+ anyhow::bail!("batching_pipeline received result with unknown messageId {message_id}");+ };+ let Some(batch) = state.batches.get_mut(&batch_id) else {+ anyhow::bail!(+ "batching_pipeline has batchId mapping but batch {batch_id} is missing for messageId {message_id}"+ );+ };+ batch.inflight.remove(&message_id);+ batch.results.insert(message_id, result);+ if !batch.inflight.is_empty() {+ return Ok(None);+ }+ let mut batch = state.batches.remove(&batch_id).expect("present");+ for id in &batch.message_ids {+ state.message_to_batch.remove(id);+ }+ drop(state);+ let ordered: Vec<_> = batch+ .message_ids+ .iter()+ .map(|id| {+ batch+ .results+ .remove(id)+ .expect("every message has a result")+ })+ .collect();+ let after = self.after.clone();+ let input = create_ok_context(+ AfterBatchInput {+ elements: ordered,+ batch_context: batch.batch_context.clone(),+ batch_id,+ },+ (),+ );+ Ok(Some(RunningAfter {+ batch_id,+ batch,+ future: async move { after.process(input).await }.boxed(),+ }))+ }++ fn complete_batch(+ &self,+ running: RunningAfter<O, CO, CB, CF>,+ after: anyhow::Result<ResultWithContext<AfterBatchOutput<O, CO, CB>, ()>>,+ ) -> anyhow::Result<()> {+ let RunningAfter {+ batch_id,+ mut batch,+ ..+ } = running;+ let after = after?;+ record_budget_overrun(&batch.budget);+ let Outcome::Ok(after_output) = after.result else {+ anyhow::bail!(+ "batching_pipeline afterBatch hook returned non-ok result for batch {batch_id}"+ );+ };+ let mut side_effects = std::mem::take(&mut batch.before_side_effects);+ side_effects.extend(after.context.side_effects);+ self.state.lock().unwrap().completed.push_back(BatchResult {+ elements: after_output.elements,+ batch_context: batch.batch_context,+ feed_context: batch.feed_context.take().expect("set at feed"),+ batch_id,+ side_effects,+ });+ Ok(())+ }+}diff --git a/rust/ingestion-framework/src/branching.rs b/rust/ingestion-framework/src/branching.rsnew file mode 100644index 000000000000..02613c0679ce--- /dev/null+++ b/rust/ingestion-framework/src/branching.rs@@ -0,0 +1,66 @@+use std::collections::HashMap;+use std::sync::Arc;++use async_trait::async_trait;++use crate::context::{OkResultWithContext, ResultWithContext};+use crate::pipeline::{DynPipeline, Pipeline};+use crate::results::{NonOk, Outcome};++pub type BranchDecisionFn<T> = Arc<dyn Fn(&T) -> String + Send + Sync>;++pub struct BranchingPipeline<I, M, O, C> {+ decision: BranchDecisionFn<M>,+ branches: HashMap<String, DynPipeline<M, O, C>>,+ previous: DynPipeline<I, M, C>,+}++impl<I, M, O, C> BranchingPipeline<I, M, O, C> {+ pub fn new(+ decision: BranchDecisionFn<M>,+ branches: HashMap<String, DynPipeline<M, O, C>>,+ previous: DynPipeline<I, M, C>,+ ) -> Self {+ Self {+ decision,+ branches,+ previous,+ }+ }+}++#[async_trait]+impl<I, M, O, C> Pipeline<I, O, C> for BranchingPipeline<I, M, O, C>+where+ I: Send + 'static,+ M: Send + 'static,+ O: Send + 'static,+ C: Send + 'static,+{+ async fn process(+ &self,+ input: OkResultWithContext<I, C>,+ ) -> anyhow::Result<ResultWithContext<O, C>> {+ let previous = self.previous.process(input).await?;+ let ok = match previous.into_ok() {+ Ok(ok) => ok,+ Err((non_ok, context)) => {+ return Ok(ResultWithContext::<M, C>::non_ok(non_ok, context))+ }+ };++ let branch_name = (self.decision)(&ok.value);+ match self.branches.get(&branch_name) {+ Some(branch) => branch.process(ok).await,+ None => Ok(ResultWithContext {+ result: Outcome::NonOk(NonOk::Dlq {+ reason: format!("Unknown branch: {branch_name}"),+ error: Some(Arc::new(anyhow::anyhow!(+ "Branch '{branch_name}' not found in branching pipeline"+ ))),+ }),+ context: ok.context,+ }),+ }+ }+}diff --git a/rust/ingestion-framework/src/budget.rs b/rust/ingestion-framework/src/budget.rsnew file mode 100644index 000000000000..890fafce6f5b--- /dev/null+++ b/rust/ingestion-framework/src/budget.rs@@ -0,0 +1,77 @@+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::Arc;+use std::time::Duration;++use tokio::time::Instant;++/// Uses `tokio::time::Instant`, so a paused test clock controls it, as Jest fake timers do in Node.+#[derive(Clone, Debug)]+pub struct BatchBudget {+ inner: Arc<BudgetInner>,+}++#[derive(Debug)]+struct BudgetInner {+ soft_at: Option<Instant>,+ expired: AtomicBool,+}++impl BatchBudget {+ pub fn soft_deadline(soft_at: Instant) -> Self {+ Self {+ inner: Arc::new(BudgetInner {+ soft_at: Some(soft_at),+ expired: AtomicBool::new(false),+ }),+ }+ }++ pub fn from_now(allowance: Duration) -> Self {+ Self::soft_deadline(Instant::now() + allowance)+ }++ pub fn unlimited() -> Self {+ Self {+ inner: Arc::new(BudgetInner {+ soft_at: None,+ expired: AtomicBool::new(false),+ }),+ }+ }++ pub fn is_unlimited(&self) -> bool {+ self.inner.soft_at.is_none()+ }++ pub fn soft_at(&self) -> Option<Instant> {+ self.inner.soft_at+ }++ /// Reads the clock, not a timer flag, so a step that holds the thread past the deadline sees it spent.+ pub fn exhausted(&self) -> bool {+ if self.inner.expired.load(Ordering::Relaxed) {+ return true;+ }+ match self.inner.soft_at {+ None => false,+ Some(at) => {+ let expired = Instant::now() >= at;+ if expired {+ self.inner.expired.store(true, Ordering::Relaxed);+ }+ expired+ }+ }+ }+}++const MAX_TIMER_DELAY_MS: u64 = (1 << 31) - 1;++/// Zero, or an allowance above the Node timer limit, means no deadline.+pub fn budget_deadline(armed_at: Instant, budget_ms: u64) -> Option<Instant> {+ if budget_ms == 0 || budget_ms > MAX_TIMER_DELAY_MS {+ None+ } else {+ Some(armed_at + Duration::from_millis(budget_ms))+ }+}diff --git a/rust/ingestion-framework/src/builders.rs b/rust/ingestion-framework/src/builders.rsnew file mode 100644index 000000000000..2afb2703b3ad--- /dev/null+++ b/rust/ingestion-framework/src/builders.rs@@ -0,0 +1,588 @@+//! Branch and redirect names are strings, so Node's compile-time coverage checks run at run time, and an unknown branch is a DLQ.++use std::collections::HashMap;+use std::hash::Hash;+use std::sync::Arc;++use futures::FutureExt;++pub use crate::per_element::per_element;++use crate::batching::{+ AfterBatchInput, AfterBatchOutput, BatchingPipeline, BatchingPipelineOptions, BeforeBatchInput,+ BeforeBatchOutput, WithBatch,+};+use crate::branching::BranchingPipeline;+use crate::chunk_pipeline::{+ apply_chunk_step_to_results, BaseChunkPipeline, BufferingChunkPipeline, DynChunkPipeline,+ GatheringChunkPipeline, SequentialChunkPipeline,+};+use crate::concurrent::ConcurrentChunkProcessingPipeline;+use crate::context::ChunkResults;+use crate::fan_out_fan_in::{+ FanInFunction, FanOutFanInChunkPipeline, FanOutFunction, FanOutSubContext,+};+use crate::filter_map::{FilterMapChunkPipeline, FilterMapMappingFunction};+use crate::grouping::{+ process_group_chunk_sequentially, ConcurrentlyGroupingChunkPipeline, GroupChunkProcessor,+};+use crate::outputs::{HasMessage, HasTeamId};+use crate::pipeline::{DynPipeline, StartPipeline, StepPipeline};+use crate::result_handling::{PipelineConfig, ResultHandlingPipeline};+use crate::retry::{with_chunk_retry, with_step_retry, RetryOptions};+use crate::side_effect_handling::{+ SideEffectHandlingConfig, SideEffectHandlingPipeline, SideEffectHandlingProcessor,+};+use crate::side_effects::SideEffectScheduler;+use crate::steps::{ChunkProcessingStep, ProcessingStep};+use crate::timeout_barrier::TimeoutBarrierPipeline;+use crate::warning_handling::{IngestionWarningEmitter, IngestionWarningHandlingChunkPipeline};++pub struct StartPipelineBuilder<T, C> {+ _marker: std::marker::PhantomData<fn(T, C)>,+}++impl<T, C> Default for StartPipelineBuilder<T, C> {+ fn default() -> Self {+ Self {+ _marker: std::marker::PhantomData,+ }+ }+}++pub fn new_pipeline<T, C>() -> StartPipelineBuilder<T, C> {+ StartPipelineBuilder::default()+}++impl<T: Send + 'static, C: Send + 'static> StartPipelineBuilder<T, C> {+ fn start(&self) -> DynPipeline<T, T, C> {+ Arc::new(StartPipeline)+ }++ pub fn pipe<U: Send + 'static>(self, step: ProcessingStep<T, U>) -> PipelineBuilder<T, U, C> {+ PipelineBuilder::new(Arc::new(StepPipeline::new(step, self.start())))+ }++ /// Retrying passes the value again, so it must be `Clone`.+ pub fn pipe_with_retry<U: Send + 'static>(+ self,+ step: ProcessingStep<T, U>,+ retry: RetryOptions,+ ) -> PipelineBuilder<T, U, C>+ where+ T: Clone + Sync,+ {+ self.pipe(with_step_retry(step, retry))+ }++ pub fn branching<U: Send + 'static>(+ self,+ decision: impl Fn(&T) -> String + Send + Sync + 'static,+ callback: impl FnOnce(+ BranchingPipelineBuilder<T, T, U, C>,+ ) -> BranchingPipelineBuilder<T, T, U, C>,+ ) -> PipelineBuilder<T, U, C> {+ let builder = BranchingPipelineBuilder {+ decision: Arc::new(decision),+ previous: self.start(),+ branches: HashMap::new(),+ };+ PipelineBuilder::new(callback(builder).build())+ }+}++pub struct PipelineBuilder<I, O, C> {+ pipeline: DynPipeline<I, O, C>,+}++impl<I: Send + 'static, O: Send + 'static, C: Send + 'static> PipelineBuilder<I, O, C> {+ pub fn new(pipeline: DynPipeline<I, O, C>) -> Self {+ Self { pipeline }+ }++ pub fn pipe<U: Send + 'static>(self, step: ProcessingStep<O, U>) -> PipelineBuilder<I, U, C> {+ PipelineBuilder::new(Arc::new(StepPipeline::new(step, self.pipeline)))+ }++ pub fn pipe_with_retry<U: Send + 'static>(+ self,+ step: ProcessingStep<O, U>,+ retry: RetryOptions,+ ) -> PipelineBuilder<I, U, C>+ where+ O: Clone + Sync,+ {+ self.pipe(with_step_retry(step, retry))+ }++ pub fn branching<U: Send + 'static>(+ self,+ decision: impl Fn(&O) -> String + Send + Sync + 'static,+ callback: impl FnOnce(+ BranchingPipelineBuilder<I, O, U, C>,+ ) -> BranchingPipelineBuilder<I, O, U, C>,+ ) -> PipelineBuilder<I, U, C> {+ let builder = BranchingPipelineBuilder {+ decision: Arc::new(decision),+ previous: self.pipeline,+ branches: HashMap::new(),+ };+ PipelineBuilder::new(callback(builder).build())+ }++ pub fn timeout_barrier(self) -> Self {+ PipelineBuilder::new(Arc::new(TimeoutBarrierPipeline::new(self.pipeline)))+ }++ pub fn handle_side_effects(+ self,+ scheduler: Arc<dyn SideEffectScheduler>,+ config: SideEffectHandlingConfig,+ ) -> Self {+ PipelineBuilder::new(Arc::new(SideEffectHandlingProcessor::new(+ self.pipeline,+ scheduler,+ config,+ )))+ }++ pub fn build(self) -> DynPipeline<I, O, C> {+ self.pipeline+ }+}++pub struct BranchingPipelineBuilder<I, M, O, C> {+ decision: crate::branching::BranchDecisionFn<M>,+ previous: DynPipeline<I, M, C>,+ branches: HashMap<String, DynPipeline<M, O, C>>,+}++impl<I: Send + 'static, M: Send + 'static, O: Send + 'static, C: Send + 'static>+ BranchingPipelineBuilder<I, M, O, C>+{+ pub fn branch(+ mut self,+ name: &str,+ callback: impl FnOnce(StartPipelineBuilder<M, C>) -> PipelineBuilder<M, O, C>,+ ) -> Self {+ let pipeline = callback(StartPipelineBuilder::default()).build();+ self.branches.insert(name.to_string(), pipeline);+ self+ }++ pub fn build(self) -> DynPipeline<I, O, C> {+ Arc::new(BranchingPipeline::new(+ self.decision,+ self.branches,+ self.previous,+ ))+ }+}++pub fn new_chunk_pipeline<T: Send + 'static, C: Send + 'static>() -> ChunkPipelineBuilder<T, T, C, C>+{+ ChunkPipelineBuilder::new(Arc::new(BufferingChunkPipeline::<T, C>::new()))+}++pub struct ChunkPipelineBuilder<I, O, CI, CO> {+ pipeline: DynChunkPipeline<I, O, CI, CO>,+}++impl<I, O, CI, CO> ChunkPipelineBuilder<I, O, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ pub fn new(pipeline: DynChunkPipeline<I, O, CI, CO>) -> Self {+ Self { pipeline }+ }++ pub fn pipe_chunk<U: Send + 'static>(+ self,+ step: ChunkProcessingStep<O, U>,+ ) -> ChunkPipelineBuilder<I, U, CI, CO> {+ ChunkPipelineBuilder::new(Arc::new(BaseChunkPipeline::new(step, self.pipeline)))+ }++ pub fn pipe_chunk_with_retry<U: Send + 'static>(+ self,+ step: ChunkProcessingStep<O, U>,+ retry: RetryOptions,+ ) -> ChunkPipelineBuilder<I, U, CI, CO>+ where+ O: Clone + Sync,+ {+ self.pipe_chunk(with_chunk_retry(step, retry))+ }++ /// Panics on `Some(0)`, as Node's `pLimit(0)` throws.+ pub fn concurrently<U: Send + 'static>(+ self,+ callback: impl FnOnce(StartPipelineBuilder<O, CO>) -> PipelineBuilder<O, U, CO>,+ max_concurrency: Option<usize>,+ ) -> ChunkPipelineBuilder<I, U, CI, CO> {+ let processor = callback(StartPipelineBuilder::default()).build();+ ChunkPipelineBuilder::new(Arc::new(ConcurrentChunkProcessingPipeline::new(+ processor,+ self.pipeline,+ max_concurrency,+ )))+ }++ pub fn sequentially<U: Send + 'static>(+ self,+ callback: impl FnOnce(StartPipelineBuilder<O, CO>) -> PipelineBuilder<O, U, CO>,+ ) -> ChunkPipelineBuilder<I, U, CI, CO> {+ let processor = callback(StartPipelineBuilder::default()).build();+ ChunkPipelineBuilder::new(Arc::new(SequentialChunkPipeline::new(+ processor,+ self.pipeline,+ )))+ }++ pub fn gather(self) -> Self {+ ChunkPipelineBuilder::new(Arc::new(GatheringChunkPipeline::new(self.pipeline)))+ }++ pub fn filter_map<MM, U, CMM, CSO>(+ self,+ mapping: impl Fn(+ crate::context::OkResultWithContext<O, CO>,+ ) -> crate::context::OkResultWithContext<MM, CMM>+ + Send+ + Sync+ + 'static,+ sub_callback: impl FnOnce(+ ChunkPipelineBuilder<MM, MM, CMM, CMM>,+ ) -> ChunkPipelineBuilder<MM, U, CMM, CSO>,+ ) -> ChunkPipelineBuilder<I, U, CI, CSO>+ where+ MM: Send + 'static,+ U: Send + 'static,+ CMM: Send + 'static,+ CSO: From<CO> + Send + 'static,+ {+ let sub = sub_callback(new_chunk_pipeline::<MM, CMM>()).build();+ let mapping: FilterMapMappingFunction<O, MM, CO, CMM> = Arc::new(mapping);+ ChunkPipelineBuilder::new(Arc::new(FilterMapChunkPipeline::new(+ self.pipeline,+ mapping,+ sub,+ )))+ }++ pub fn fan_out<S: Send + 'static>(+ self,+ name: &str,+ fan_out: impl Fn(&O) -> Vec<S> + Send + Sync + 'static,+ ) -> FanOutBuilder<I, O, S, CI, CO> {+ FanOutBuilder {+ previous: self.pipeline,+ name: name.to_string(),+ fan_out: Arc::new(fan_out),+ }+ }++ /// Panics on `Some(0)`, as Node's `pLimit(0)` throws.+ pub fn concurrently_per_group<K, U>(+ self,+ grouping: impl Fn(&O) -> K + Send + Sync + 'static,+ callback: impl FnOnce(GroupProcessingBuilder<O, O, CO>) -> GroupProcessingBuilder<O, U, CO>,+ max_concurrency: Option<usize>,+ ) -> ChunkPipelineBuilder<I, U, CI, CO>+ where+ K: Hash + Eq + Clone + Send + Sync + 'static,+ U: Send + 'static,+ {+ let group = callback(GroupProcessingBuilder::identity());+ ChunkPipelineBuilder::new(Arc::new(ConcurrentlyGroupingChunkPipeline::new(+ Arc::new(grouping),+ group.processor,+ self.pipeline,+ max_concurrency,+ )))+ }++ pub fn handle_side_effects(+ self,+ scheduler: Arc<dyn SideEffectScheduler>,+ config: SideEffectHandlingConfig,+ ) -> Self {+ ChunkPipelineBuilder::new(Arc::new(SideEffectHandlingPipeline::new(+ self.pipeline,+ scheduler,+ config,+ )))+ }++ pub fn message_aware<U, CU>(+ self,+ callback: impl FnOnce(Self) -> ChunkPipelineBuilder<I, U, CI, CU>,+ ) -> MessageAwareChunkPipelineBuilder<I, U, CI, CU>+ where+ U: Send + 'static,+ CU: HasMessage + Send + 'static,+ {+ MessageAwareChunkPipelineBuilder {+ pipeline: callback(self).build(),+ }+ }++ pub fn team_aware<U, CU>(+ self,+ callback: impl FnOnce(Self) -> ChunkPipelineBuilder<I, U, CI, CU>,+ ) -> TeamAwareChunkPipelineBuilder<I, U, CI, CU>+ where+ U: Send + 'static,+ CU: HasTeamId + Send + 'static,+ {+ TeamAwareChunkPipelineBuilder {+ pipeline: callback(self).build(),+ }+ }++ pub fn build(self) -> DynChunkPipeline<I, O, CI, CO> {+ self.pipeline+ }+}++pub struct GroupProcessingBuilder<T, U, C> {+ processor: GroupChunkProcessor<T, U, C>,+}++impl<T: Send + 'static, C: Send + 'static> GroupProcessingBuilder<T, T, C> {+ fn identity() -> Self {+ Self {+ processor: Arc::new(|items| futures::future::ready(Ok(items)).boxed()),+ }+ }+}++impl<T: Send + 'static, U: Send + 'static, C: Send + 'static> GroupProcessingBuilder<T, U, C> {+ fn stage<V: Send + 'static>(+ self,+ next: GroupChunkProcessor<U, V, C>,+ ) -> GroupProcessingBuilder<T, V, C> {+ let first = self.processor;+ GroupProcessingBuilder {+ processor: Arc::new(move |items: ChunkResults<T, C>| {+ let first = first.clone();+ let next = next.clone();+ async move { next(first(items).await?).await }.boxed()+ }),+ }+ }++ pub fn pipe_chunk<V: Send + 'static>(+ self,+ step: ChunkProcessingStep<U, V>,+ ) -> GroupProcessingBuilder<T, V, C> {+ self.stage(Arc::new(move |items| {+ let step = step.clone();+ async move { apply_chunk_step_to_results(&step, items).await }.boxed()+ }))+ }++ pub fn sequentially<V: Send + 'static>(+ self,+ callback: impl FnOnce(StartPipelineBuilder<U, C>) -> PipelineBuilder<U, V, C>,+ ) -> GroupProcessingBuilder<T, V, C> {+ let processor = callback(StartPipelineBuilder::default()).build();+ self.stage(process_group_chunk_sequentially(processor))+ }+}++pub struct FanOutBuilder<I, O, S, CI, CO> {+ previous: DynChunkPipeline<I, O, CI, CO>,+ name: String,+ fan_out: FanOutFunction<O, S>,+}++impl<I, O, S, CI, CO> FanOutBuilder<I, O, S, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ S: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ pub fn via<SO: Send + 'static>(+ self,+ callback: impl FnOnce(+ ChunkPipelineBuilder<S, S, FanOutSubContext, FanOutSubContext>,+ )+ -> ChunkPipelineBuilder<S, SO, FanOutSubContext, FanOutSubContext>,+ ) -> FanInBuilder<I, O, S, SO, CI, CO> {+ FanInBuilder {+ previous: self.previous,+ fan_out_name: self.name,+ fan_out: self.fan_out,+ sub: callback(new_chunk_pipeline()).build(),+ }+ }+}++pub struct FanInBuilder<I, O, S, SO, CI, CO> {+ previous: DynChunkPipeline<I, O, CI, CO>,+ fan_out_name: String,+ fan_out: FanOutFunction<O, S>,+ sub: DynChunkPipeline<S, SO, FanOutSubContext, FanOutSubContext>,+}++impl<I, O, S, SO, CI, CO> FanInBuilder<I, O, S, SO, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ S: Send + 'static,+ SO: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ pub fn fan_in<U: Send + 'static>(+ self,+ name: &str,+ fan_in: impl Fn(O, Vec<SO>) -> U + Send + Sync + 'static,+ ) -> ChunkPipelineBuilder<I, U, CI, CO> {+ let fan_in: FanInFunction<O, SO, U> = Arc::new(fan_in);+ ChunkPipelineBuilder::new(Arc::new(FanOutFanInChunkPipeline::new(+ self.previous,+ &self.fan_out_name,+ self.fan_out,+ self.sub,+ name,+ fan_in,+ )))+ }+}++pub struct MessageAwareChunkPipelineBuilder<I, O, CI, CO> {+ pipeline: DynChunkPipeline<I, O, CI, CO>,+}++impl<I, O, CI, CO> MessageAwareChunkPipelineBuilder<I, O, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: HasMessage + Send + 'static,+{+ pub fn handle_results(+ self,+ config: PipelineConfig,+ ) -> ResultHandledChunkPipelineBuilder<I, O, CI, CO> {+ ResultHandledChunkPipelineBuilder {+ pipeline: Arc::new(ResultHandlingPipeline::new(self.pipeline, config)),+ }+ }+}++/// The only next step is `handle_side_effects`, so side effects are always handled before build.+pub struct ResultHandledChunkPipelineBuilder<I, O, CI, CO> {+ pipeline: DynChunkPipeline<I, O, CI, CO>,+}++impl<I, O, CI, CO> ResultHandledChunkPipelineBuilder<I, O, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ pub fn handle_side_effects(+ self,+ scheduler: Arc<dyn SideEffectScheduler>,+ config: SideEffectHandlingConfig,+ ) -> ChunkPipelineBuilder<I, O, CI, CO> {+ ChunkPipelineBuilder::new(Arc::new(SideEffectHandlingPipeline::new(+ self.pipeline,+ scheduler,+ config,+ )))+ }+}++pub struct TeamAwareChunkPipelineBuilder<I, O, CI, CO> {+ pipeline: DynChunkPipeline<I, O, CI, CO>,+}++impl<I, O, CI, CO> TeamAwareChunkPipelineBuilder<I, O, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: HasTeamId + Send + 'static,+{+ pub fn handle_ingestion_warnings(+ self,+ emitter: Arc<dyn IngestionWarningEmitter>,+ ) -> ChunkPipelineBuilder<I, O, CI, CO> {+ ChunkPipelineBuilder::new(Arc::new(IngestionWarningHandlingChunkPipeline::new(+ emitter,+ self.pipeline,+ )))+ }++ pub fn into_builder(self) -> ChunkPipelineBuilder<I, O, CI, CO> {+ ChunkPipelineBuilder::new(self.pipeline)+ }+}++#[allow(clippy::type_complexity)]+pub fn new_batching_pipeline<T, O, CI, CO, CB, CF>(+ before: impl FnOnce(+ StartPipelineBuilder<BeforeBatchInput<T, CI>, ()>,+ )+ -> PipelineBuilder<BeforeBatchInput<T, CI>, BeforeBatchOutput<T, CI, CB>, ()>,+ batch: impl FnOnce(+ ChunkPipelineBuilder<WithBatch<T, CB>, WithBatch<T, CB>, CI, CI>,+ ) -> ChunkPipelineBuilder<WithBatch<T, CB>, O, CI, CO>,+ after: impl FnOnce(+ StartPipelineBuilder<AfterBatchInput<O, CO, CB>, ()>,+ )+ -> PipelineBuilder<AfterBatchInput<O, CO, CB>, AfterBatchOutput<O, CO, CB>, ()>,+ options: BatchingPipelineOptions,+) -> BatchingPipeline<T, O, CI, CO, CB, CF>+where+ T: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+ CB: Send + Sync + 'static,+ CF: Send + 'static,+{+ let sub = batch(new_chunk_pipeline()).build();+ let before = before(StartPipelineBuilder::default()).build();+ let after = after(StartPipelineBuilder::default()).build();+ BatchingPipeline::new(sub, before, after, options)+}++pub fn before_batch_step<T, CI, CB>(+ name: &str,+ make_context: impl Fn(u64) -> CB + Send + Sync + 'static,+) -> ProcessingStep<BeforeBatchInput<T, CI>, BeforeBatchOutput<T, CI, CB>>+where+ T: Send + 'static,+ CI: Send + 'static,+ CB: Send + 'static,+{+ ProcessingStep::sync(name, move |input: BeforeBatchInput<T, CI>| {+ Ok(crate::results::ok(BeforeBatchOutput {+ batch_context: make_context(input.batch_id),+ elements: input.elements,+ batch_id: input.batch_id,+ }))+ })+}++pub fn passthrough_after_batch_step<O, CO, CB>(+ name: &str,+) -> ProcessingStep<AfterBatchInput<O, CO, CB>, AfterBatchOutput<O, CO, CB>>+where+ O: Send + 'static,+ CO: Send + 'static,+ CB: Send + Sync + 'static,+{+ ProcessingStep::sync(name, |input| Ok(crate::results::ok(input)))+}diff --git a/rust/ingestion-framework/src/chunk_pipeline.rs b/rust/ingestion-framework/src/chunk_pipeline.rsnew file mode 100644index 000000000000..c7ea524beefe--- /dev/null+++ b/rust/ingestion-framework/src/chunk_pipeline.rs@@ -0,0 +1,321 @@+use std::sync::{Arc, Mutex};+use std::time::Instant;++use async_trait::async_trait;++use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};+use crate::metrics::{record_budget_checkpoint, record_step_duration, BudgetCheckpoint};+use crate::pipeline::DynPipeline;+use crate::results::{NonOk, Outcome};+use crate::steps::ChunkProcessingStep;++/// `&self`, because a stage takes a `feed()` while a `next()` is parked, as in Node. Never overlap two `next()` calls.+#[async_trait]+pub trait ChunkPipeline<I, O, CI, CO>: Send + Sync {+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>);+ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>>;+}++pub type DynChunkPipeline<I, O, CI, CO> = Arc<dyn ChunkPipeline<I, O, CI, CO>>;++pub struct BufferingChunkPipeline<T, C> {+ buffer: Mutex<Vec<ResultWithContext<T, C>>>,+}++impl<T, C> Default for BufferingChunkPipeline<T, C> {+ fn default() -> Self {+ Self {+ buffer: Mutex::new(Vec::new()),+ }+ }+}++impl<T, C> BufferingChunkPipeline<T, C> {+ pub fn new() -> Self {+ Self::default()+ }+}++#[async_trait]+impl<T: Send + 'static, C: Send + 'static> ChunkPipeline<T, T, C, C>+ for BufferingChunkPipeline<T, C>+{+ fn feed(&self, elements: Vec<OkResultWithContext<T, C>>) {+ self.buffer+ .lock()+ .unwrap()+ .extend(elements.into_iter().map(OkResultWithContext::into_result));+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<T, C>>> {+ let results = std::mem::take(&mut *self.buffer.lock().unwrap());+ Ok(if results.is_empty() {+ None+ } else {+ Some(results)+ })+ }+}++/// The budget check runs per element, because one chunk can hold elements of batches with different budgets.+pub async fn apply_chunk_step_to_results<T, U, C>(+ step: &ChunkProcessingStep<T, U>,+ items: ChunkResults<T, C>,+) -> anyhow::Result<ChunkResults<U, C>>+where+ T: Send + 'static,+ U: Send + 'static,+{+ let step_name = step.name();+ enum Slot<U, C> {+ Done(ResultWithContext<U, C>),+ CutOff(crate::context::PipelineContext<C>),+ Pending(crate::context::PipelineContext<C>),+ }++ let mut slots: Vec<Slot<U, C>> = Vec::with_capacity(items.len());+ let mut values: Vec<T> = Vec::with_capacity(items.len());+ for item in items {+ match item.result {+ Outcome::NonOk(non_ok) => slots.push(Slot::Done(ResultWithContext {+ result: Outcome::NonOk(non_ok),+ context: item.context,+ })),+ Outcome::Ok(value) => {+ if item.context.budget.as_ref().is_some_and(|b| b.exhausted()) {+ record_budget_checkpoint(BudgetCheckpoint::Chunk, step_name.clone());+ slots.push(Slot::CutOff(item.context));+ } else {+ values.push(value);+ slots.push(Slot::Pending(item.context));+ }+ }+ }+ }++ let mut step_results = Vec::new();+ if !values.is_empty() {+ let value_count = values.len();+ let started = Instant::now();+ match step.call(values).await {+ Ok(results) => {+ record_step_duration(+ step_name.clone(),+ "chunk",+ "chunk",+ started.elapsed().as_secs_f64(),+ );+ step_results = results;+ }+ Err(error) => {+ record_step_duration(+ step_name.clone(),+ "chunk",+ "exception",+ started.elapsed().as_secs_f64(),+ );+ // Node logs the origin of every OK item, also of the items the budget cut off.+ let debug_contexts: Vec<_> = slots+ .iter()+ .filter_map(|s| match s {+ Slot::Pending(c) | Slot::CutOff(c) => c.debug_context.clone(),+ Slot::Done(_) => None,+ })+ .collect();+ tracing::error!(+ step = %step_name,+ error = %error,+ chunk_size = value_count,+ debug_context = ?debug_contexts,+ "Chunk step threw"+ );+ return Err(error);+ }+ }+ if step_results.len() != value_count {+ anyhow::bail!(+ "Chunk pipeline step {} returned different number of results than input values: {} !== {}",+ step_name,+ step_results.len(),+ value_count+ );+ }+ }++ let mut step_results = step_results.into_iter();+ Ok(slots+ .into_iter()+ .map(|slot| match slot {+ Slot::Done(done) => done,+ Slot::CutOff(context) => ResultWithContext {+ result: Outcome::NonOk(NonOk::Timeout {+ reason: format!("budget exceeded before {step_name}"),+ }),+ context,+ },+ Slot::Pending(mut context) => {+ let result = step_results.next().expect("length checked above");+ let outcome = context.absorb_step(step_name, result);+ ResultWithContext {+ result: outcome,+ context,+ }+ }+ })+ .collect())+}++pub struct BaseChunkPipeline<I, M, O, CI, CO> {+ step: ChunkProcessingStep<M, O>,+ previous: DynChunkPipeline<I, M, CI, CO>,+}++impl<I, M, O, CI, CO> BaseChunkPipeline<I, M, O, CI, CO> {+ pub fn new(step: ChunkProcessingStep<M, O>, previous: DynChunkPipeline<I, M, CI, CO>) -> Self {+ Self { step, previous }+ }+}++#[async_trait]+impl<I, M, O, CI, CO> ChunkPipeline<I, O, CI, CO> for BaseChunkPipeline<I, M, O, CI, CO>+where+ I: Send + 'static,+ M: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.previous.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ match self.previous.next().await? {+ None => Ok(None),+ Some(results) => Ok(Some(+ apply_chunk_step_to_results(&self.step, results).await?,+ )),+ }+ }+}++pub struct SequentialChunkPipeline<I, M, O, CI, CO> {+ processor: DynPipeline<M, O, CO>,+ previous: DynChunkPipeline<I, M, CI, CO>,+}++impl<I, M, O, CI, CO> SequentialChunkPipeline<I, M, O, CI, CO> {+ pub fn new(processor: DynPipeline<M, O, CO>, previous: DynChunkPipeline<I, M, CI, CO>) -> Self {+ Self {+ processor,+ previous,+ }+ }+}++pub(crate) async fn process_sequentially<M, O, C>(+ processor: &DynPipeline<M, O, C>,+ items: ChunkResults<M, C>,+) -> anyhow::Result<ChunkResults<O, C>>+where+ M: Send + 'static,+ O: Send + 'static,+ C: Send + 'static,+{+ let mut results = Vec::with_capacity(items.len());+ for item in items {+ match item.into_ok() {+ Ok(ok) => results.push(processor.process(ok).await?),+ Err((non_ok, context)) => {+ results.push(ResultWithContext::<M, C>::non_ok(non_ok, context))+ }+ }+ }+ Ok(results)+}++#[async_trait]+impl<I, M, O, CI, CO> ChunkPipeline<I, O, CI, CO> for SequentialChunkPipeline<I, M, O, CI, CO>+where+ I: Send + 'static,+ M: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.previous.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ match self.previous.next().await? {+ None => Ok(None),+ Some(results) => Ok(Some(process_sequentially(&self.processor, results).await?)),+ }+ }+}++pub struct GatheringChunkPipeline<I, O, CI, CO> {+ sub: DynChunkPipeline<I, O, CI, CO>,+}++impl<I, O, CI, CO> GatheringChunkPipeline<I, O, CI, CO> {+ pub fn new(sub: DynChunkPipeline<I, O, CI, CO>) -> Self {+ Self { sub }+ }+}++#[async_trait]+impl<I, O, CI, CO> ChunkPipeline<I, O, CI, CO> for GatheringChunkPipeline<I, O, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.sub.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ let mut all = Vec::new();+ while let Some(chunk) = self.sub.next().await? {+ all.extend(chunk);+ }+ Ok(if all.is_empty() { None } else { Some(all) })+ }+}++pub struct ChunkPipelineUnwrapper<I, O, C> {+ pipeline: DynChunkPipeline<I, O, C, C>,+}++impl<I: Send + 'static, O: Send + 'static, C: Send + 'static> ChunkPipelineUnwrapper<I, O, C> {+ pub fn new(pipeline: DynChunkPipeline<I, O, C, C>) -> Self {+ Self { pipeline }+ }++ pub fn feed(&self, elements: Vec<OkResultWithContext<I, C>>) {+ self.pipeline.feed(elements)+ }++ pub async fn next(&self) -> anyhow::Result<Option<Vec<O>>> {+ let Some(results) = self.pipeline.next().await? else {+ return Ok(None);+ };+ let mut values = Vec::with_capacity(results.len());+ let mut side_effects = 0;+ for result in results {+ side_effects += result.context.side_effects.len();+ if let Outcome::Ok(value) = result.result {+ values.push(value);+ }+ }+ if side_effects > 0 {+ tracing::warn!(+ "ChunkPipelineUnwrapper found {side_effects} remaining side effects that were not handled"+ );+ }+ Ok(Some(values))+ }+}diff --git a/rust/ingestion-framework/src/concurrent.rs b/rust/ingestion-framework/src/concurrent.rsnew file mode 100644index 000000000000..66873abc0089--- /dev/null+++ b/rust/ingestion-framework/src/concurrent.rs@@ -0,0 +1,124 @@+use std::collections::VecDeque;+use std::sync::{Arc, Mutex};++use async_trait::async_trait;+use tokio::task::JoinHandle;++use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};+use crate::errors::join_error;+use crate::interleaving::{InterleavingCallbacks, InterleavingChunkPipeline, PullOutcome};+use crate::pipeline::DynPipeline;+use crate::sync::FifoLimiter;++enum Queued<O, C> {+ Ready(ResultWithContext<O, C>),+ Running(JoinHandle<anyhow::Result<ResultWithContext<O, C>>>),+}++struct ConcurrentCallbacks<I, M, O, CI, CO> {+ processor: DynPipeline<M, O, CO>,+ previous: DynChunkPipeline<I, M, CI, CO>,+ limit: Option<Arc<FifoLimiter>>,+ queue: Mutex<VecDeque<Queued<O, CO>>>,+}++#[async_trait]+impl<I, M, O, CI, CO> InterleavingCallbacks<I, O, CI, CO> for ConcurrentCallbacks<I, M, O, CI, CO>+where+ I: Send + 'static,+ M: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.previous.feed(elements);+ }++ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<O, CO>> {+ let Some(previous) = self.previous.next().await? else {+ return Ok(PullOutcome::Drained);+ };+ let mut queue = self.queue.lock().unwrap();+ for item in previous {+ match item.into_ok() {+ Ok(ok) => {+ let processor = self.processor.clone();+ // Admission is reserved in pull order, so the head of the queue gets a slot first and emission order holds.+ let permit = self.limit.as_ref().map(|limit| limit.acquire());+ queue.push_back(Queued::Running(tokio::spawn(async move {+ let _permit = match permit {+ Some(permit) => Some(permit.await),+ None => None,+ };+ processor.process(ok).await+ })));+ }+ Err((non_ok, context)) => queue.push_back(Queued::Ready(+ ResultWithContext::<M, CO>::non_ok(non_ok, context),+ )),+ }+ }+ Ok(PullOutcome::Drain)+ }++ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ let head = self.queue.lock().unwrap().pop_front();+ match head {+ None => Ok(None),+ Some(Queued::Ready(result)) => Ok(Some(vec![result])),+ Some(Queued::Running(handle)) => Ok(Some(vec![handle.await.map_err(join_error)??])),+ }+ }+}++pub struct ConcurrentChunkProcessingPipeline<I, M, O, CI, CO> {+ inner: Inner<I, M, O, CI, CO>,+}++type Inner<I, M, O, CI, CO> =+ InterleavingChunkPipeline<I, O, CI, CO, ConcurrentCallbacks<I, M, O, CI, CO>>;++impl<I, M, O, CI, CO> ConcurrentChunkProcessingPipeline<I, M, O, CI, CO>+where+ I: Send + 'static,+ M: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ pub fn new(+ processor: DynPipeline<M, O, CO>,+ previous: DynChunkPipeline<I, M, CI, CO>,+ max_concurrency: Option<usize>,+ ) -> Self {+ Self {+ inner: InterleavingChunkPipeline::new(Arc::new(ConcurrentCallbacks {+ processor,+ previous,+ limit: max_concurrency.map(FifoLimiter::new),+ queue: Mutex::new(VecDeque::new()),+ })),+ }+ }+}++#[async_trait]+impl<I, M, O, CI, CO> ChunkPipeline<I, O, CI, CO>+ for ConcurrentChunkProcessingPipeline<I, M, O, CI, CO>+where+ I: Send + 'static,+ M: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.inner.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ self.inner.next().await+ }+}diff --git a/rust/ingestion-framework/src/context.rs b/rust/ingestion-framework/src/context.rsnew file mode 100644index 000000000000..bb8c94568dc7--- /dev/null+++ b/rust/ingestion-framework/src/context.rs@@ -0,0 +1,195 @@+use std::ops::{Deref, DerefMut};+use std::sync::Arc;++use indexmap::IndexMap;+use serde::{Deserialize, Serialize};++use crate::budget::BatchBudget;+use crate::results::{NonOk, Outcome, PipelineResult};+use crate::side_effects::SideEffect;++/// Keys keep insertion order, as in a JavaScript object, and re-inserting a key keeps its place.+pub type WarningDetails = IndexMap<String, serde_json::Value>;++#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct PipelineWarning {+ #[serde(rename = "type")]+ pub warning_type: String,+ pub details: WarningDetails,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub pipeline_step: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub key: Option<String>,+ #[serde(default)]+ pub always_send: bool,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub source: Option<String>,+}++impl PipelineWarning {+ /// A `json!` object has sorted keys without serde_json's `preserve_order`, so use `with_detail` where key order matters.+ pub fn new(warning_type: impl Into<String>, details: serde_json::Value) -> Self {+ Self {+ warning_type: warning_type.into(),+ details: match details {+ serde_json::Value::Object(map) => map.into_iter().collect(),+ _ => WarningDetails::new(),+ },+ pipeline_step: None,+ key: None,+ always_send: false,+ source: None,+ }+ }++ pub fn with_detail(+ mut self,+ key: impl Into<String>,+ value: impl Into<serde_json::Value>,+ ) -> Self {+ self.details.insert(key.into(), value.into());+ self+ }+}++/// Node spreads `C` into the context. Here `C` lives in `data` behind `Deref`, and `message_id` lives here so batching keeps the type.+pub struct PipelineContext<C> {+ pub data: C,+ pub last_step: Option<Arc<str>>,+ pub debug_context: Option<serde_json::Value>,+ pub budget: Option<BatchBudget>,+ pub message_id: Option<u64>,+ pub side_effects: Vec<SideEffect>,+ pub warnings: Vec<PipelineWarning>,+}++impl<C> PipelineContext<C> {+ pub fn new(data: C) -> Self {+ Self {+ data,+ last_step: None,+ debug_context: None,+ budget: None,+ message_id: None,+ side_effects: Vec::new(),+ warnings: Vec::new(),+ }+ }++ pub fn last_step(&self) -> Option<&str> {+ self.last_step.as_deref()+ }++ pub fn map_data<D>(self, f: impl FnOnce(C) -> D) -> PipelineContext<D> {+ PipelineContext {+ data: f(self.data),+ last_step: self.last_step,+ debug_context: self.debug_context,+ budget: self.budget,+ message_id: self.message_id,+ side_effects: self.side_effects,+ warnings: self.warnings,+ }+ }++ pub(crate) fn absorb_step<T>(+ &mut self,+ step_name: &Arc<str>,+ result: PipelineResult<T>,+ ) -> Outcome<T> {+ self.last_step = Some(step_name.clone());+ self.side_effects.extend(result.side_effects);+ self.warnings.extend(result.warnings);+ result.outcome+ }+}++impl<C: Default> Default for PipelineContext<C> {+ fn default() -> Self {+ Self::new(C::default())+ }+}++impl<C> Deref for PipelineContext<C> {+ type Target = C;+ fn deref(&self) -> &C {+ &self.data+ }+}++impl<C> DerefMut for PipelineContext<C> {+ fn deref_mut(&mut self) -> &mut C {+ &mut self.data+ }+}++impl<C: std::fmt::Debug> std::fmt::Debug for PipelineContext<C> {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("PipelineContext")+ .field("data", &self.data)+ .field("last_step", &self.last_step)+ .field("message_id", &self.message_id)+ .field("side_effects", &self.side_effects.len())+ .field("warnings", &self.warnings)+ .finish()+ }+}++#[derive(Debug)]+pub struct ResultWithContext<T, C> {+ pub result: Outcome<T>,+ pub context: PipelineContext<C>,+}++impl<T, C> ResultWithContext<T, C> {+ pub fn is_ok(&self) -> bool {+ self.result.is_ok()+ }++ pub fn value(&self) -> Option<&T> {+ self.result.value()+ }++ // Boxing the error side would allocate for each non-OK result on every step.+ #[allow(clippy::result_large_err)]+ pub fn into_ok(self) -> Result<OkResultWithContext<T, C>, (NonOk, PipelineContext<C>)> {+ match self.result {+ Outcome::Ok(value) => Ok(OkResultWithContext {+ value,+ context: self.context,+ }),+ Outcome::NonOk(non_ok) => Err((non_ok, self.context)),+ }+ }++ pub(crate) fn non_ok<U>(non_ok: NonOk, context: PipelineContext<C>) -> ResultWithContext<U, C> {+ ResultWithContext {+ result: Outcome::NonOk(non_ok),+ context,+ }+ }+}++#[derive(Debug)]+pub struct OkResultWithContext<T, C> {+ pub value: T,+ pub context: PipelineContext<C>,+}++impl<T, C> OkResultWithContext<T, C> {+ pub fn into_result(self) -> ResultWithContext<T, C> {+ ResultWithContext {+ result: Outcome::Ok(self.value),+ context: self.context,+ }+ }+}++pub fn create_ok_context<T, C>(value: T, data: C) -> OkResultWithContext<T, C> {+ OkResultWithContext {+ value,+ context: PipelineContext::new(data),+ }+}++pub type ChunkResults<T, C> = Vec<ResultWithContext<T, C>>;diff --git a/rust/ingestion-framework/src/errors.rs b/rust/ingestion-framework/src/errors.rsnew file mode 100644index 000000000000..ce31d7d21bb6--- /dev/null+++ b/rust/ingestion-framework/src/errors.rs@@ -0,0 +1,121 @@+//! `is_retriable` reads only the top-level error, as Node reads `isRetriable` on the thrown object and not on its `cause`.++use std::fmt;+use std::sync::Arc;++#[derive(Debug, thiserror::Error)]+#[error("{message}")]+pub struct PipelineError {+ pub message: String,+ /// `Some(true)` retries, `Some(false)` goes to the DLQ, `None` retries and then crashes.+ pub retriable: Option<bool>,+}++impl PipelineError {+ pub fn retriable(message: impl Into<String>) -> anyhow::Error {+ anyhow::Error::new(Self {+ message: message.into(),+ retriable: Some(true),+ })+ }++ pub fn non_retriable(message: impl Into<String>) -> anyhow::Error {+ anyhow::Error::new(Self {+ message: message.into(),+ retriable: Some(false),+ })+ }+}++/// A `context()` wrapper reads as `None`, as in Node. A `SharedError` is one error object that several callers share, so it reads through.+pub fn is_retriable(error: &anyhow::Error) -> Option<bool> {+ // `anyhow`'s `downcast_ref` also finds a context wrapper's inner error, so check the outermost.+ let outermost: &(dyn std::error::Error + 'static) = error.chain().next()?;+ if let Some(pipeline_error) = outermost.downcast_ref::<PipelineError>() {+ return pipeline_error.retriable;+ }+ if let Some(shared) = outermost.downcast_ref::<SharedError>() {+ return is_retriable(&shared.0);+ }+ None+}++/// A poisoned pipeline rejects every later `next()` with the same error, and `anyhow::Error` is not `Clone`.+#[derive(Clone)]+pub struct SharedError(pub Arc<anyhow::Error>);++impl fmt::Debug for SharedError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ fmt::Debug::fmt(&*self.0, f)+ }+}++impl fmt::Display for SharedError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ fmt::Display::fmt(&*self.0, f)+ }+}++impl std::error::Error for SharedError {+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {+ self.0.source()+ }+}++pub fn shared_error(error: Arc<anyhow::Error>) -> anyhow::Error {+ anyhow::Error::new(SharedError(error))+}++pub fn join_error(error: tokio::task::JoinError) -> anyhow::Error {+ anyhow::anyhow!("pipeline task failed: {error}")+}++/// As in JavaScript's `AggregateError`, the message is `message` only, and a `None` sub error is Node's `undefined`.+pub struct AggregateError {+ pub message: String,+ pub errors: Vec<Option<Arc<anyhow::Error>>>,+}++impl fmt::Display for AggregateError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str(&self.message)+ }+}++impl fmt::Debug for AggregateError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("AggregateError")+ .field("message", &self.message)+ .field("errors", &self.errors)+ .finish()+ }+}++impl std::error::Error for AggregateError {}++pub fn panic_message(payload: &(dyn std::any::Any + Send)) -> String {+ if let Some(message) = payload.downcast_ref::<&str>() {+ (*message).to_string()+ } else if let Some(message) = payload.downcast_ref::<String>() {+ message.clone()+ } else {+ "unknown panic".to_string()+ }+}++/// A panic becomes an error logged with the element's origin, as a Node callback throw rejects `next()`.+pub fn call_user_fn<R>(+ kind: &str,+ name: &str,+ debug_context: Option<&serde_json::Value>,+ f: impl FnOnce() -> R,+) -> anyhow::Result<R> {+ match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {+ Ok(value) => Ok(value),+ Err(payload) => {+ let message = panic_message(payload.as_ref());+ tracing::error!(error = %message, debug_context = ?debug_context, "{kind} {name} threw");+ Err(anyhow::anyhow!("{kind} {name} threw: {message}"))+ }+ }+}diff --git a/rust/ingestion-framework/src/fan_out_fan_in.rs b/rust/ingestion-framework/src/fan_out_fan_in.rsnew file mode 100644index 000000000000..a9f4c8d93048--- /dev/null+++ b/rust/ingestion-framework/src/fan_out_fan_in.rs@@ -0,0 +1,372 @@+use std::collections::HashMap;+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{Arc, Mutex};++use async_trait::async_trait;++use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};+use crate::context::{ChunkResults, OkResultWithContext, PipelineContext, ResultWithContext};+use crate::errors::{call_user_fn, AggregateError};+use crate::interleaving::{InterleavingCallbacks, InterleavingChunkPipeline, PullOutcome};+use crate::results::{NonOk, Outcome};++/// Must be cheap, because heavy work belongs in the sub-pipeline.+pub type FanOutFunction<T, S> = Arc<dyn Fn(&T) -> Vec<S> + Send + Sync>;++pub type FanInFunction<T, SO, U> = Arc<dyn Fn(T, Vec<SO>) -> U + Send + Sync>;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct FanOutParentRef(u64);++#[derive(Debug, Clone)]+pub struct FanOutSubContext {+ parent: FanOutParentRef,+}++impl FanOutSubContext {+ pub fn parent(&self) -> FanOutParentRef {+ self.parent+ }+}++struct PendingParent<T, SO, C> {+ original: T,+ context: PipelineContext<C>,+ outstanding: usize,+ total: usize,+ collected: Vec<SO>,+ dlq_failures: Vec<(String, Option<Arc<anyhow::Error>>)>,+ dlq_last_step: Option<Arc<str>>,+ timed_out: Option<(String, Option<Arc<str>>)>,+}++enum Settled<T, SO, U, C> {+ Done(ResultWithContext<U, C>),+ FanIn {+ original: T,+ context: PipelineContext<C>,+ collected: Vec<SO>,+ },+}++struct FanState<T, SO, U, C> {+ pending: HashMap<FanOutParentRef, PendingParent<T, SO, C>>,+ ready: Vec<ResultWithContext<U, C>>,+}++struct FanCallbacks<I, T, S, SO, U, CI, C> {+ previous: DynChunkPipeline<I, T, CI, C>,+ fan_out: FanOutFunction<T, S>,+ sub: DynChunkPipeline<S, SO, FanOutSubContext, FanOutSubContext>,+ fan_in: FanInFunction<T, SO, U>,+ fan_out_name: Arc<str>,+ fan_in_name: Arc<str>,+ next_ref: AtomicU64,+ state: Mutex<FanState<T, SO, U, C>>,+}++impl<I, T, S, SO, U, CI, C> FanCallbacks<I, T, S, SO, U, CI, C> {+ /// Call it with no lock held, so a panic in user code cannot poison the stage state.+ fn complete_parent(+ &self,+ original: T,+ mut context: PipelineContext<C>,+ collected: Vec<SO>,+ ) -> anyhow::Result<ResultWithContext<U, C>> {+ context.last_step = Some(self.fan_in_name.clone());+ let merged = call_user_fn(+ "Fan-in",+ &self.fan_in_name,+ context.debug_context.as_ref(),+ || (self.fan_in)(original, collected),+ )?;+ Ok(ResultWithContext {+ result: Outcome::Ok(merged),+ context,+ })+ }++ fn finish(&self, settled: Settled<T, SO, U, C>) -> anyhow::Result<ResultWithContext<U, C>> {+ match settled {+ Settled::Done(result) => Ok(result),+ Settled::FanIn {+ original,+ context,+ collected,+ } => self.complete_parent(original, context, collected),+ }+ }++ fn settle(+ &self,+ state: &mut FanState<T, SO, U, C>,+ sub: ResultWithContext<SO, FanOutSubContext>,+ out: &mut Vec<Settled<T, SO, U, C>>,+ ) -> anyhow::Result<()> {+ let parent_ref = sub.context.data.parent;+ let Some(parent) = state.pending.get_mut(&parent_ref) else {+ anyhow::bail!(+ "Fan-out/fan-in ({}/{}) received a sub-result without a known parent tag — the subpipeline must preserve context",+ self.fan_out_name,+ self.fan_in_name+ );+ };+ let mut sub_context = sub.context;+ parent+ .context+ .side_effects+ .append(&mut sub_context.side_effects);+ parent.context.warnings.append(&mut sub_context.warnings);+ match sub.result {+ Outcome::Ok(value) => parent.collected.push(value),+ Outcome::NonOk(NonOk::Timeout { reason }) => {+ if parent.timed_out.is_none() {+ parent.timed_out = Some((reason, sub_context.last_step.clone()));+ }+ }+ Outcome::NonOk(NonOk::Dlq { reason, error }) => {+ parent.dlq_failures.push((reason, error));+ parent.dlq_last_step = sub_context.last_step.clone();+ }+ Outcome::NonOk(NonOk::Drop { .. }) => {}+ Outcome::NonOk(redirect @ NonOk::Redirect { .. }) => {+ tracing::warn!(+ fan_out_step = %self.fan_out_name,+ fan_in_step = %self.fan_in_name,+ reason = redirect.reason(),+ "Fan-out subpipeline produced a redirect result; excluding it"+ );+ }+ }+ parent.outstanding -= 1;+ if parent.outstanding > 0 {+ return Ok(());+ }+ let mut parent = state.pending.remove(&parent_ref).expect("present");+ let result = if let Some((reason, last_step)) = parent.timed_out.take() {+ parent.context.last_step = last_step;+ Settled::Done(ResultWithContext {+ result: Outcome::NonOk(NonOk::Timeout {+ reason: format!("fan-out sub-element timed out: {reason}"),+ }),+ context: parent.context,+ })+ } else if !parent.dlq_failures.is_empty() {+ let mut reasons: Vec<&str> = Vec::new();+ for (reason, _) in &parent.dlq_failures {+ if !reasons.contains(&reason.as_str()) {+ reasons.push(reason);+ }+ }+ let reason = format!(+ "{}/{} fan-out sub-elements dlq'd: {}",+ parent.dlq_failures.len(),+ parent.total,+ reasons.join("; ")+ );+ parent.context.last_step = parent.dlq_last_step.take();+ let error = AggregateError {+ message: reason.clone(),+ errors: parent.dlq_failures.into_iter().map(|(_, e)| e).collect(),+ };+ Settled::Done(ResultWithContext {+ result: Outcome::NonOk(NonOk::Dlq {+ error: Some(Arc::new(anyhow::Error::new(error))),+ reason,+ }),+ context: parent.context,+ })+ } else {+ Settled::FanIn {+ original: parent.original,+ context: parent.context,+ collected: parent.collected,+ }+ };+ out.push(result);+ Ok(())+ }+}++#[async_trait]+impl<I, T, S, SO, U, CI, C> InterleavingCallbacks<I, U, CI, C>+ for FanCallbacks<I, T, S, SO, U, CI, C>+where+ I: Send + 'static,+ T: Send + 'static,+ S: Send + 'static,+ SO: Send + 'static,+ U: Send + 'static,+ CI: Send + 'static,+ C: Send + 'static,+{+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.previous.feed(elements);+ }++ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<U, C>> {+ let Some(previous) = self.previous.next().await? else {+ return Ok(PullOutcome::Drained);+ };+ let mut settled = Vec::new();+ let mut subs = Vec::new();+ let mut parents = Vec::new();+ // User callbacks run before the lock is taken, so a panic cannot poison it.+ for element in previous {+ let ok = match element.into_ok() {+ Ok(ok) => ok,+ Err((n, context)) => {+ settled.push(ResultWithContext::<T, C>::non_ok(n, context));+ continue;+ }+ };+ let sub_values = call_user_fn(+ "Fan-out",+ &self.fan_out_name,+ ok.context.debug_context.as_ref(),+ || (self.fan_out)(&ok.value),+ )?;+ if sub_values.is_empty() {+ settled.push(self.complete_parent(ok.value, ok.context, Vec::new())?);+ continue;+ }+ let parent_ref = FanOutParentRef(self.next_ref.fetch_add(1, Ordering::Relaxed));+ let total = sub_values.len();+ for sub in sub_values {+ let mut context = PipelineContext::new(FanOutSubContext { parent: parent_ref });+ context.debug_context = ok.context.debug_context.clone();+ context.budget = ok.context.budget.clone();+ subs.push(OkResultWithContext {+ value: sub,+ context,+ });+ }+ parents.push((+ parent_ref,+ PendingParent {+ original: ok.value,+ context: ok.context,+ outstanding: total,+ total,+ collected: Vec::new(),+ dlq_failures: Vec::new(),+ dlq_last_step: None,+ timed_out: None,+ },+ ));+ }+ if !parents.is_empty() {+ self.state.lock().unwrap().pending.extend(parents);+ }+ let fed = !subs.is_empty();+ if fed {+ self.sub.feed(subs);+ }+ if !settled.is_empty() {+ return Ok(PullOutcome::Emit(settled));+ }+ if !fed {+ return Ok(PullOutcome::Emit(Vec::new()));+ }+ Ok(PullOutcome::Drain)+ }++ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<U, C>>> {+ loop {+ {+ let mut state = self.state.lock().unwrap();+ if !state.ready.is_empty() {+ return Ok(Some(std::mem::take(&mut state.ready)));+ }+ }+ let Some(sub_results) = self.sub.next().await? else {+ let pending = self.state.lock().unwrap().pending.len();+ if pending > 0 {+ anyhow::bail!(+ "Fan-out/fan-in ({}/{}) subpipeline drained with {} parent(s) still pending — sub-results were lost",+ self.fan_out_name,+ self.fan_in_name,+ pending+ );+ }+ return Ok(None);+ };+ let mut done = Vec::new();+ {+ let mut state = self.state.lock().unwrap();+ for sub_result in sub_results {+ self.settle(&mut state, sub_result, &mut done)?;+ }+ }+ // A parent is ready before the next fan-in runs, so the poisoned drain returns it, as Node's readyResults does.+ for settled in done {+ let finished = self.finish(settled)?;+ self.state.lock().unwrap().ready.push(finished);+ }+ }+ }+}++pub struct FanOutFanInChunkPipeline<I, T, S, SO, U, CI, C> {+ inner: Inner<I, T, S, SO, U, CI, C>,+}++type Inner<I, T, S, SO, U, CI, C> =+ InterleavingChunkPipeline<I, U, CI, C, FanCallbacks<I, T, S, SO, U, CI, C>>;++impl<I, T, S, SO, U, CI, C> FanOutFanInChunkPipeline<I, T, S, SO, U, CI, C>+where+ I: Send + 'static,+ T: Send + 'static,+ S: Send + 'static,+ SO: Send + 'static,+ U: Send + 'static,+ CI: Send + 'static,+ C: Send + 'static,+{+ pub fn new(+ previous: DynChunkPipeline<I, T, CI, C>,+ fan_out_name: &str,+ fan_out: FanOutFunction<T, S>,+ sub: DynChunkPipeline<S, SO, FanOutSubContext, FanOutSubContext>,+ fan_in_name: &str,+ fan_in: FanInFunction<T, SO, U>,+ ) -> Self {+ Self {+ inner: InterleavingChunkPipeline::new(Arc::new(FanCallbacks {+ previous,+ fan_out,+ sub,+ fan_in,+ fan_out_name: Arc::from(fan_out_name),+ fan_in_name: Arc::from(fan_in_name),+ next_ref: AtomicU64::new(0),+ state: Mutex::new(FanState {+ pending: HashMap::new(),+ ready: Vec::new(),+ }),+ })),+ }+ }+}++#[async_trait]+impl<I, T, S, SO, U, CI, C> ChunkPipeline<I, U, CI, C>+ for FanOutFanInChunkPipeline<I, T, S, SO, U, CI, C>+where+ I: Send + 'static,+ T: Send + 'static,+ S: Send + 'static,+ SO: Send + 'static,+ U: Send + 'static,+ CI: Send + 'static,+ C: Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.inner.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<U, C>>> {+ self.inner.next().await+ }+}diff --git a/rust/ingestion-framework/src/filter_map.rs b/rust/ingestion-framework/src/filter_map.rsnew file mode 100644index 000000000000..70eaf47ec071--- /dev/null+++ b/rust/ingestion-framework/src/filter_map.rs@@ -0,0 +1,124 @@+//! Rust has no union types, so the output context implements `From` the earlier one for passthroughs.++use std::sync::Arc;++use async_trait::async_trait;++use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};+use crate::interleaving::{InterleavingCallbacks, InterleavingChunkPipeline, PullOutcome};++pub type FilterMapMappingFunction<T, U, C, CM> =+ Arc<dyn Fn(OkResultWithContext<T, C>) -> OkResultWithContext<U, CM> + Send + Sync>;++struct FilterMapCallbacks<I, M, MM, O, CI, CM, CMM, CO> {+ previous: DynChunkPipeline<I, M, CI, CM>,+ mapping: FilterMapMappingFunction<M, MM, CM, CMM>,+ sub: DynChunkPipeline<MM, O, CMM, CO>,+}++#[async_trait]+impl<I, M, MM, O, CI, CM, CMM, CO> InterleavingCallbacks<I, O, CI, CO>+ for FilterMapCallbacks<I, M, MM, O, CI, CM, CMM, CO>+where+ I: Send + 'static,+ M: Send + 'static,+ MM: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CM: Send + 'static,+ CMM: Send + 'static,+ CO: From<CM> + Send + 'static,+{+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.previous.feed(elements);+ }++ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<O, CO>> {+ let Some(previous) = self.previous.next().await? else {+ return Ok(PullOutcome::Drained);+ };+ let mut ok = Vec::new();+ let mut non_ok = Vec::new();+ for element in previous {+ match element.into_ok() {+ Ok(element) => ok.push((self.mapping)(element)),+ Err((n, context)) => non_ok.push(ResultWithContext::<M, CO>::non_ok(+ n,+ context.map_data(CO::from),+ )),+ }+ }+ let had_ok = !ok.is_empty();+ if had_ok {+ self.sub.feed(ok);+ }+ if !non_ok.is_empty() {+ return Ok(PullOutcome::Emit(non_ok));+ }+ // A non-empty upstream chunk with nothing in it surfaces as an empty chunk, 1:1.+ if !had_ok {+ return Ok(PullOutcome::Emit(Vec::new()));+ }+ Ok(PullOutcome::Drain)+ }++ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ self.sub.next().await+ }+}++pub struct FilterMapChunkPipeline<I, M, MM, O, CI, CM, CMM, CO> {+ inner: Inner<I, M, MM, O, CI, CM, CMM, CO>,+}++type Inner<I, M, MM, O, CI, CM, CMM, CO> =+ InterleavingChunkPipeline<I, O, CI, CO, FilterMapCallbacks<I, M, MM, O, CI, CM, CMM, CO>>;++impl<I, M, MM, O, CI, CM, CMM, CO> FilterMapChunkPipeline<I, M, MM, O, CI, CM, CMM, CO>+where+ I: Send + 'static,+ M: Send + 'static,+ MM: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CM: Send + 'static,+ CMM: Send + 'static,+ CO: From<CM> + Send + 'static,+{+ pub fn new(+ previous: DynChunkPipeline<I, M, CI, CM>,+ mapping: FilterMapMappingFunction<M, MM, CM, CMM>,+ sub: DynChunkPipeline<MM, O, CMM, CO>,+ ) -> Self {+ Self {+ inner: InterleavingChunkPipeline::new(Arc::new(FilterMapCallbacks {+ previous,+ mapping,+ sub,+ })),+ }+ }+}++#[async_trait]+impl<I, M, MM, O, CI, CM, CMM, CO> ChunkPipeline<I, O, CI, CO>+ for FilterMapChunkPipeline<I, M, MM, O, CI, CM, CMM, CO>+where+ I: Send + 'static,+ M: Send + 'static,+ MM: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CM: Send + 'static,+ CMM: Send + 'static,+ CO: From<CM> + Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.inner.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ self.inner.next().await+ }+}diff --git a/rust/ingestion-framework/src/grouping.rs b/rust/ingestion-framework/src/grouping.rsnew file mode 100644index 000000000000..ef05f697da08--- /dev/null+++ b/rust/ingestion-framework/src/grouping.rs@@ -0,0 +1,276 @@+use std::collections::{HashSet, VecDeque};+use std::hash::Hash;+use std::panic::AssertUnwindSafe;+use std::sync::{Arc, Mutex};++use async_trait::async_trait;+use futures::future::BoxFuture;+use futures::FutureExt;+use indexmap::IndexMap;++use crate::chunk_pipeline::{process_sequentially, ChunkPipeline, DynChunkPipeline};+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};+use crate::errors::call_user_fn;+use crate::interleaving::{InterleavingCallbacks, InterleavingChunkPipeline, PullOutcome};+use crate::pipeline::DynPipeline;+use crate::sync::{FifoLimiter, ResettableSignal};++pub type GroupingFunction<T, K> = Arc<dyn Fn(&T) -> K + Send + Sync>;++pub type GroupChunkProcessor<T, U, C> = Arc<+ dyn Fn(ChunkResults<T, C>) -> BoxFuture<'static, anyhow::Result<ChunkResults<U, C>>>+ + Send+ + Sync,+>;++pub fn process_group_chunk_sequentially<T, U, C>(+ processor: DynPipeline<T, U, C>,+) -> GroupChunkProcessor<T, U, C>+where+ T: Send + 'static,+ U: Send + 'static,+ C: Send + 'static,+{+ Arc::new(move |items| {+ let processor = processor.clone();+ async move { process_sequentially(&processor, items).await }.boxed()+ })+}++struct GroupState<K, M, O, C> {+ // Insertion order matters: groups start in the order their keys first arrived.+ queues: IndexMap<K, Vec<ResultWithContext<M, C>>>,+ active: HashSet<K>,+ completed: VecDeque<ChunkResults<O, C>>,+ failure: Option<Arc<anyhow::Error>>,+}++struct GroupingShared<K, M, O, C> {+ processor: GroupChunkProcessor<M, O, C>,+ limit: Option<Arc<FifoLimiter>>,+ state: Mutex<GroupState<K, M, O, C>>,+ // Resolves when any group finishes, so a parked drain also wakes for a group started later.+ group_completed: ResettableSignal,+}++impl<K, M, O, C> GroupingShared<K, M, O, C>+where+ K: Hash + Eq + Clone + Send + 'static,+ M: Send + 'static,+ O: Send + 'static,+ C: Send + 'static,+{+ /// Every key left in `queues` belongs to a running group, so its order no longer matters and `swap_remove` is safe.+ fn start_group_if_queued(self: &Arc<Self>, state: &mut GroupState<K, M, O, C>, key: &K) {+ if state.active.contains(key) {+ return;+ }+ let Some(queue) = state.queues.swap_remove(key) else {+ return;+ };+ if queue.is_empty() {+ return;+ }+ self.spawn_group(state, key.clone(), queue);+ }++ /// One pass in key-arrival order, so a pull costs O(keys) and not O(keys²).+ fn start_available(self: &Arc<Self>, state: &mut GroupState<K, M, O, C>) {+ let mut ready = Vec::new();+ let GroupState { queues, active, .. } = &mut *state;+ queues.retain(|key, queue| {+ if active.contains(key) || queue.is_empty() {+ return true;+ }+ ready.push((key.clone(), std::mem::take(queue)));+ false+ });+ for (key, queue) in ready {+ self.spawn_group(state, key, queue);+ }+ }++ fn spawn_group(+ self: &Arc<Self>,+ state: &mut GroupState<K, M, O, C>,+ key: K,+ queue: Vec<ResultWithContext<M, C>>,+ ) {+ // The key is claimed now, so per-key order holds even while the group waits for a slot.+ state.active.insert(key.clone());++ let shared = self.clone();+ let permit = self.limit.as_ref().map(|limit| limit.acquire());+ tokio::spawn(async move {+ let _permit = match permit {+ Some(permit) => Some(permit.await),+ None => None,+ };+ // The processor call is guarded, so a panic before it returns its future fails the group.+ let outcome = AssertUnwindSafe(async { (shared.processor)(queue).await })+ .catch_unwind()+ .await;+ let mut state = shared.state.lock().unwrap();+ match outcome {+ Ok(Ok(results)) => {+ state.completed.push_back(results);+ state.active.remove(&key);+ shared.start_group_if_queued(&mut state, &key);+ }+ Ok(Err(error)) => {+ // Poison. The key stays active, so later items for it never start.+ state.failure.get_or_insert_with(|| Arc::new(error));+ }+ Err(payload) => {+ let message = crate::errors::panic_message(payload.as_ref());+ state.failure.get_or_insert_with(|| {+ Arc::new(anyhow::anyhow!("group processor panicked: {message}"))+ });+ }+ }+ drop(state);+ shared.group_completed.resolve();+ });+ }+}++struct GroupingCallbacks<I, K, M, O, CI, CO> {+ grouping: GroupingFunction<M, K>,+ previous: DynChunkPipeline<I, M, CI, CO>,+ shared: Arc<GroupingShared<K, M, O, CO>>,+}++#[async_trait]+impl<I, K, M, O, CI, CO> InterleavingCallbacks<I, O, CI, CO>+ for GroupingCallbacks<I, K, M, O, CI, CO>+where+ I: Send + 'static,+ K: Hash + Eq + Clone + Send + Sync + 'static,+ M: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.previous.feed(elements);+ }++ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<O, CO>> {+ let Some(previous) = self.previous.next().await? else {+ return Ok(PullOutcome::Drained);+ };+ // The grouping function is user code, so it runs before the lock is taken.+ let mut routed = Vec::with_capacity(previous.len());+ let mut non_ok = Vec::new();+ for item in previous {+ match &item.result {+ crate::results::Outcome::Ok(value) => {+ let key = call_user_fn(+ "Grouping function",+ "groupingFn",+ item.context.debug_context.as_ref(),+ || (self.grouping)(value),+ )?;+ routed.push((key, item));+ }+ crate::results::Outcome::NonOk(_) => {+ let (n, context) = item.into_ok().err().expect("non-OK");+ non_ok.push(ResultWithContext::<M, CO>::non_ok(n, context));+ }+ }+ }+ let mut state = self.shared.state.lock().unwrap();+ for (key, item) in routed {+ state.queues.entry(key).or_default().push(item);+ }+ if !non_ok.is_empty() {+ state.completed.push_back(non_ok);+ }+ self.shared.start_available(&mut state);+ Ok(PullOutcome::Drain)+ }++ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ loop {+ let wait = {+ let mut state = self.shared.state.lock().unwrap();+ if let Some(completed) = state.completed.pop_front() {+ return Ok(Some(completed));+ }+ // Surface a failure only after everything that completed has drained.+ if let Some(failure) = &state.failure {+ return Err(crate::errors::shared_error(failure.clone()));+ }+ if state.active.is_empty() {+ return Ok(None);+ }+ self.shared.group_completed.reset();+ self.shared.group_completed.wait()+ };+ wait.await;+ }+ }+}++pub struct ConcurrentlyGroupingChunkPipeline<I, K, M, O, CI, CO> {+ inner: Inner<I, K, M, O, CI, CO>,+}++type Inner<I, K, M, O, CI, CO> =+ InterleavingChunkPipeline<I, O, CI, CO, GroupingCallbacks<I, K, M, O, CI, CO>>;++impl<I, K, M, O, CI, CO> ConcurrentlyGroupingChunkPipeline<I, K, M, O, CI, CO>+where+ I: Send + 'static,+ K: Hash + Eq + Clone + Send + Sync + 'static,+ M: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ pub fn new(+ grouping: GroupingFunction<M, K>,+ processor: GroupChunkProcessor<M, O, CO>,+ previous: DynChunkPipeline<I, M, CI, CO>,+ max_concurrency: Option<usize>,+ ) -> Self {+ let shared = Arc::new(GroupingShared {+ processor,+ limit: max_concurrency.map(FifoLimiter::new),+ state: Mutex::new(GroupState {+ queues: IndexMap::new(),+ active: HashSet::new(),+ completed: VecDeque::new(),+ failure: None,+ }),+ group_completed: ResettableSignal::new(),+ });+ Self {+ inner: InterleavingChunkPipeline::new(Arc::new(GroupingCallbacks {+ grouping,+ previous,+ shared,+ })),+ }+ }+}++#[async_trait]+impl<I, K, M, O, CI, CO> ChunkPipeline<I, O, CI, CO>+ for ConcurrentlyGroupingChunkPipeline<I, K, M, O, CI, CO>+where+ I: Send + 'static,+ K: Hash + Eq + Clone + Send + Sync + 'static,+ M: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.inner.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ self.inner.next().await+ }+}diff --git a/rust/ingestion-framework/src/helpers.rs b/rust/ingestion-framework/src/helpers.rsnew file mode 100644index 000000000000..5198c2a7290d--- /dev/null+++ b/rust/ingestion-framework/src/helpers.rs@@ -0,0 +1,115 @@+use std::collections::BTreeMap;++use serde_json::json;++use crate::context::{OkResultWithContext, PipelineContext};+use crate::outputs::{HasMessage, KafkaMessage};++// A responder needs these headers for an event restriction. distinct_id and session_id are customer-controlled and stay out.+const LOGGED_HEADERS: [&str; 3] = ["token", "event", "uuid"];++pub fn create_kafka_debug_context(message: &KafkaMessage) -> serde_json::Value {+ let headers: serde_json::Map<String, serde_json::Value> = message+ .headers+ .iter()+ .filter(|(key, _)| LOGGED_HEADERS.contains(&key.as_str()))+ .filter_map(|(key, value)| {+ value+ .as_ref()+ .map(|v| (key.clone(), json!(String::from_utf8_lossy(v))))+ })+ .collect();+ json!({+ "topic": message.topic,+ "partition": message.partition,+ "offset": message.offset,+ "headers": headers,+ })+}++pub fn compact_offset_ranges(offsets: &[i64]) -> String {+ let mut sorted: Vec<i64> = offsets.to_vec();+ sorted.sort_unstable();+ sorted.dedup();+ let mut ranges = Vec::new();+ let mut iter = sorted.into_iter();+ let Some(mut start) = iter.next() else {+ return String::new();+ };+ let mut end = start;+ let fmt = |s: i64, e: i64| {+ if s == e {+ s.to_string()+ } else {+ format!("{s}-{e}")+ }+ };+ for offset in iter {+ if offset == end + 1 {+ end = offset;+ } else {+ ranges.push(fmt(start, end));+ start = offset;+ end = offset;+ }+ }+ ranges.push(fmt(start, end));+ ranges.join(",")+}++pub fn aggregate_kafka_debug_contexts(debug_contexts: &[serde_json::Value]) -> serde_json::Value {+ let mut by_partition: BTreeMap<String, Vec<i64>> = BTreeMap::new();+ let mut order: Vec<String> = Vec::new();+ for context in debug_contexts {+ let key = format!(+ "{}[{}]",+ context["topic"].as_str().unwrap_or("undefined"),+ context["partition"].as_i64().unwrap_or_default()+ );+ if !by_partition.contains_key(&key) {+ order.push(key.clone());+ }+ by_partition+ .entry(key)+ .or_default()+ .push(context["offset"].as_i64().unwrap_or_default());+ }+ json!({+ "count": debug_contexts.len(),+ "offsets": order+ .iter()+ .map(|key| format!("{key}:{}", compact_offset_ranges(&by_partition[key])))+ .collect::<Vec<_>>(),+ })+}++#[derive(Debug, Clone)]+pub struct MessageContext {+ pub message: KafkaMessage,+}++impl HasMessage for MessageContext {+ fn message(&self) -> &KafkaMessage {+ &self.message+ }+}++/// The value is a copy of the context, as in Node, where it is `{ message }` too.+pub fn create_batch(+ messages: Vec<KafkaMessage>,+) -> Vec<OkResultWithContext<MessageContext, MessageContext>> {+ messages+ .into_iter()+ .map(|message| {+ let debug = create_kafka_debug_context(&message);+ let mut context = PipelineContext::new(MessageContext {+ message: message.clone(),+ });+ context.debug_context = Some(debug);+ OkResultWithContext {+ value: MessageContext { message },+ context,+ }+ })+ .collect()+}diff --git a/rust/ingestion-framework/src/interleaving.rs b/rust/ingestion-framework/src/interleaving.rsnew file mode 100644index 000000000000..216cdd351807--- /dev/null+++ b/rust/ingestion-framework/src/interleaving.rs@@ -0,0 +1,137 @@+//! The pull is a boxed future in a field, not a spawned task, because a spawn per `next()` costs a cross-thread wake.++use std::sync::{Arc, Mutex};++use async_trait::async_trait;+use futures::future::BoxFuture;+use futures::FutureExt;++use crate::context::{ChunkResults, OkResultWithContext};+use crate::errors::shared_error;+use crate::sync::ResettableSignal;++pub enum PullOutcome<O, C> {+ Emit(ChunkResults<O, C>),+ Drain,+ Drained,+}++#[async_trait]+pub trait InterleavingCallbacks<I, O, CI, CO>: Send + Sync + 'static {+ fn on_feed(&self, elements: Vec<OkResultWithContext<I, CI>>);+ async fn on_source_pull(&self) -> anyhow::Result<PullOutcome<O, CO>>;+ async fn on_process_pull(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>>;+}++type SubPull<O, CO> = BoxFuture<'static, anyhow::Result<Option<ChunkResults<O, CO>>>>;++pub struct InterleavingChunkPipeline<I, O, CI, CO, Cb> {+ callbacks: Arc<Cb>,+ new_input: ResettableSignal,+ sub_pending: Mutex<Option<SubPull<O, CO>>>,+ failure: Mutex<Option<Arc<anyhow::Error>>>,+ _marker: std::marker::PhantomData<fn(I, CI)>,+}++impl<I, O, CI, CO, Cb> InterleavingChunkPipeline<I, O, CI, CO, Cb>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+ Cb: InterleavingCallbacks<I, O, CI, CO>,+{+ pub fn new(callbacks: Arc<Cb>) -> Self {+ Self {+ callbacks,+ new_input: ResettableSignal::new(),+ sub_pending: Mutex::new(None),+ failure: Mutex::new(None),+ _marker: std::marker::PhantomData,+ }+ }++ pub fn callbacks(&self) -> &Arc<Cb> {+ &self.callbacks+ }++ pub fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.callbacks.on_feed(elements);+ self.new_input.resolve();+ }++ fn take_or_start_sub_pull(&self) -> SubPull<O, CO> {+ if let Some(pending) = self.sub_pending.lock().unwrap().take() {+ return pending;+ }+ let callbacks = self.callbacks.clone();+ async move { callbacks.on_process_pull().await }.boxed()+ }++ fn latch(&self, error: anyhow::Error) -> Arc<anyhow::Error> {+ let mut failure = self.failure.lock().unwrap();+ failure.get_or_insert_with(|| Arc::new(error)).clone()+ }++ pub async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ if self.failure.lock().unwrap().is_some() {+ return self.drain_then_reject().await;+ }++ loop {+ // Arm a fresh wake-up before the pull, so a stale feed signal does not pull the next chunk early.+ self.new_input.reset();++ let pulled = match self.callbacks.on_source_pull().await {+ Ok(pulled) => pulled,+ Err(error) => {+ self.latch(error);+ return self.drain_then_reject().await;+ }+ };+ let source_drained = match pulled {+ PullOutcome::Emit(chunk) => return Ok(Some(chunk)),+ PullOutcome::Drain => false,+ PullOutcome::Drained => true,+ };++ let mut pending = self.take_or_start_sub_pull();+ let wake = self.new_input.wait();+ let sub_result = tokio::select! {+ biased;+ result = &mut pending => result,+ _ = wake => {+ *self.sub_pending.lock().unwrap() = Some(pending);+ continue;+ }+ };++ match sub_result {+ Err(error) => {+ let shared = self.latch(error);+ return Err(shared_error(shared));+ }+ Ok(Some(chunk)) => return Ok(Some(chunk)),+ Ok(None) => {+ if source_drained {+ return Ok(None);+ }+ }+ }+ }+ }++ async fn drain_then_reject(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ let pending = self.take_or_start_sub_pull();+ if let Ok(Some(chunk)) = pending.await {+ return Ok(Some(chunk));+ }+ let failure = self+ .failure+ .lock()+ .unwrap()+ .clone()+ .expect("failure is latched");+ Err(shared_error(failure))+ }+}diff --git a/rust/ingestion-framework/src/js_json.rs b/rust/ingestion-framework/src/js_json.rsnew file mode 100644index 000000000000..12aa717832dd--- /dev/null+++ b/rust/ingestion-framework/src/js_json.rs@@ -0,0 +1,199 @@+//! Kafka messages that must equal the Node bytes use this writer, because `serde_json` prints `1.0` and `1e21`, not `1` and `1e+21`.++use std::fmt::Write;++use serde_json::Value;++pub fn write_value(out: &mut String, value: &Value) {+ match value {+ Value::Null => out.push_str("null"),+ Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }),+ Value::Number(n) => match n.as_f64() {+ Some(f) => write_number(out, f),+ None => out.push_str("null"),+ },+ Value::String(s) => write_string(out, s),+ Value::Array(items) => {+ out.push('[');+ for (i, item) in items.iter().enumerate() {+ if i > 0 {+ out.push(',');+ }+ write_value(out, item);+ }+ out.push(']');+ }+ Value::Object(map) => write_object(out, map.iter()),+ }+}++pub fn write_object<'a>(out: &mut String, entries: impl Iterator<Item = (&'a String, &'a Value)>) {+ let mut index_keys: Vec<(u32, &String, &Value)> = Vec::new();+ let mut other_keys: Vec<(&String, &Value)> = Vec::new();+ for (key, value) in entries {+ match array_index(key) {+ Some(index) => index_keys.push((index, key, value)),+ None => other_keys.push((key, value)),+ }+ }+ index_keys.sort_by_key(|(index, _, _)| *index);+ out.push('{');+ let ordered = index_keys+ .into_iter()+ .map(|(_, key, value)| (key, value))+ .chain(other_keys);+ for (i, (key, value)) in ordered.enumerate() {+ if i > 0 {+ out.push(',');+ }+ write_string(out, key);+ out.push(':');+ write_value(out, value);+ }+ out.push('}');+}++/// A canonical decimal below 2^32 - 1 is an array index, which a JavaScript object lists first.+fn array_index(key: &str) -> Option<u32> {+ if key.is_empty() || key.len() > 10 || !key.bytes().all(|b| b.is_ascii_digit()) {+ return None;+ }+ if key.len() > 1 && key.starts_with('0') {+ return None;+ }+ let value: u64 = key.parse().ok()?;+ (value < u32::MAX as u64).then_some(value as u32)+}++pub fn write_string(out: &mut String, s: &str) {+ out.push('"');+ for c in s.chars() {+ match c {+ '"' => out.push_str("\\\""),+ '\\' => out.push_str("\\\\"),+ '\u{08}' => out.push_str("\\b"),+ '\u{0c}' => out.push_str("\\f"),+ '\n' => out.push_str("\\n"),+ '\r' => out.push_str("\\r"),+ '\t' => out.push_str("\\t"),+ c if (c as u32) < 0x20 => {+ let _ = write!(out, "\\u{:04x}", c as u32);+ }+ c => out.push(c),+ }+ }+ out.push('"');+}++/// JavaScript numbers are f64, so an integer above 2^53 prints as in Node.+pub fn write_number(out: &mut String, f: f64) {+ if !f.is_finite() {+ out.push_str("null");+ return;+ }+ if f == 0.0 {+ out.push('0');+ return;+ }+ if f < 0.0 {+ out.push('-');+ }+ let sci = shortest_scientific(f.abs());+ let (mantissa, exponent) = sci.split_once('e').expect("LowerExp has an exponent");+ let digits: String = mantissa.chars().filter(|c| *c != '.').collect();+ let k = digits.len() as i32;+ let n = exponent.parse::<i32>().expect("integer exponent") + 1;+ if k <= n && n <= 21 {+ out.push_str(&digits);+ out.extend(std::iter::repeat_n('0', (n - k) as usize));+ } else if 0 < n && n <= 21 {+ out.push_str(&digits[..n as usize]);+ out.push('.');+ out.push_str(&digits[n as usize..]);+ } else if -6 < n && n <= 0 {+ out.push_str("0.");+ out.extend(std::iter::repeat_n('0', (-n) as usize));+ out.push_str(&digits);+ } else {+ out.push_str(&digits[..1]);+ if k > 1 {+ out.push('.');+ out.push_str(&digits[1..]);+ }+ let e = n - 1;+ let _ = write!(out, "e{}{}", if e >= 0 { "+" } else { "-" }, e.abs());+ }+}++/// ECMAScript breaks a tie between two shortest candidates toward even, so an odd last digit is checked against the exact rounding.+fn shortest_scientific(value: f64) -> String {+ let shortest = format!("{value:e}");+ let mantissa = shortest.split_once('e').map_or("", |(m, _)| m);+ let last_is_odd = mantissa.bytes().last().is_some_and(|d| (d - b'0') % 2 == 1);+ if !last_is_odd {+ return shortest;+ }+ let digits = mantissa.bytes().filter(u8::is_ascii_digit).count();+ let exact = format!("{:.*e}", digits - 1, value);+ // At a power of two the lower gap is narrower, so the nearest candidate can miss.+ if exact != shortest && exact.parse::<f64>() == Ok(value) {+ exact+ } else {+ shortest+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn number(f: f64) -> String {+ let mut out = String::new();+ write_number(&mut out, f);+ out+ }++ // Expected strings come from `String(x)` in Node 22.+ #[test]+ fn numbers_match_javascript() {+ let cases: &[(f64, &str)] = &[+ (1.0, "1"),+ (100.0, "100"),+ (-1.5, "-1.5"),+ (0.1, "0.1"),+ (-0.0, "0"),+ (1e21, "1e+21"),+ (1e20, "100000000000000000000"),+ (1.2345e-7, "1.2345e-7"),+ (0.000001, "0.000001"),+ (12345678901234567890.0, "12345678901234567000"),+ (5e-324, "5e-324"),+ (1.7976931348623157e308, "1.7976931348623157e+308"),+ (1e15 + 0.25, "1000000000000000.2"),+ (f64::from_bits(0x409a_ab1c_d000_0000), "1706.7781372070312"),+ (-1417389728594571.2, "-1417389728594571.2"),+ ];+ for (input, expected) in cases {+ assert_eq!(number(*input), *expected, "{input}");+ }+ }++ #[test]+ fn index_keys_come_first_like_a_javascript_object() {+ let mut map = indexmap::IndexMap::new();+ map.insert("b".to_string(), Value::from(1));+ map.insert("10".to_string(), Value::from(2));+ map.insert("2".to_string(), Value::from(3));+ map.insert("01".to_string(), Value::from(4));+ let mut out = String::new();+ write_object(&mut out, map.iter());+ assert_eq!(out, r#"{"2":3,"10":2,"b":1,"01":4}"#);+ }++ #[test]+ fn strings_escape_like_javascript() {+ let mut out = String::new();+ write_string(&mut out, "a\"\\/\u{7}\n\u{2028}é");+ assert_eq!(out, "\"a\\\"\\\\/\\u0007\\n\u{2028}é\"");+ }+}diff --git a/rust/ingestion-framework/src/lib.rs b/rust/ingestion-framework/src/lib.rsnew file mode 100644index 000000000000..fb3fa649e8f1--- /dev/null+++ b/rust/ingestion-framework/src/lib.rs@@ -0,0 +1,60 @@+//! Kept in step with `nodejs/src/ingestion/framework`. `PARITY.md` lists every known difference.++pub mod batching;+pub mod branching;+pub mod budget;+pub mod builders;+pub mod chunk_pipeline;+pub mod concurrent;+pub mod context;+pub mod errors;+pub mod fan_out_fan_in;+pub mod filter_map;+pub mod grouping;+pub mod helpers;+pub mod interleaving;+mod js_json;+pub mod metrics;+pub mod outputs;+pub mod per_element;+pub mod pipeline;+pub mod result_handling;+pub mod results;+pub mod retry;+pub mod side_effect_handling;+pub mod side_effects;+pub mod steps;+pub mod sync;+pub mod timeout_barrier;+pub mod warning_handling;++pub use batching::{+ AfterBatchInput, AfterBatchOutput, BatchResult, BatchingPipeline, BatchingPipelineOptions,+ BeforeBatchInput, BeforeBatchOutput, FeedRejectionKind, FeedResult, WithBatch,+};+pub use budget::BatchBudget;+pub use builders::{+ before_batch_step, new_batching_pipeline, new_chunk_pipeline, new_pipeline,+ passthrough_after_batch_step, ChunkPipelineBuilder, PipelineBuilder, StartPipelineBuilder,+};+pub use chunk_pipeline::{ChunkPipeline, ChunkPipelineUnwrapper, DynChunkPipeline};+pub use context::{+ create_ok_context, ChunkResults, OkResultWithContext, PipelineContext, PipelineWarning,+ ResultWithContext, WarningDetails,+};+pub use errors::{is_retriable, PipelineError};+pub use outputs::{+ HasMessage, HasTeamId, Headers, IngestionOutputs, KafkaMessage, MemoryOutputs, OutputMessage,+ DLQ_OUTPUT, INGESTION_WARNINGS_OUTPUT,+};+pub use per_element::per_element;+pub use pipeline::{DynPipeline, Pipeline};+pub use result_handling::PipelineConfig;+pub use results::{+ dlq, drop, ok, redirect, redirect_with, timeout, NonOk, Outcome, PipelineResult,+ PipelineResultType,+};+pub use retry::RetryOptions;+pub use side_effect_handling::SideEffectHandlingConfig;+pub use side_effects::{PromiseScheduler, SideEffect, SideEffectScheduler};+pub use steps::{chunk_step, step, ChunkProcessingStep, ProcessingStep};diff --git a/rust/ingestion-framework/src/metrics.rs b/rust/ingestion-framework/src/metrics.rsnew file mode 100644index 000000000000..22650470702e--- /dev/null+++ b/rust/ingestion-framework/src/metrics.rs@@ -0,0 +1,65 @@+//! Metric names and labels match the Node side, so one dashboard reads both.++use std::sync::Arc;++pub const SIDE_EFFECT_RESULT_COUNTER: &str = "pipelines_side_effects_total";+pub const PIPELINE_STEP_DURATION_HISTOGRAM: &str = "ingestion_pipeline_step_duration_seconds";+pub const PIPELINE_RETRY_ATTEMPTS_HISTOGRAM: &str = "ingestion_pipeline_retry_attempts";+pub const BATCH_BUDGET_EXHAUSTED_COUNTER: &str = "ingestion_batch_budget_exhausted_total";+pub const BATCH_BUDGET_OVERRUN_HISTOGRAM: &str = "ingestion_batch_budget_overrun_seconds";+pub const BATCH_BUDGET_CHECKPOINT_COUNTER: &str = "ingestion_batch_budget_checkpoint_total";+pub const INGESTION_PIPELINE_RESULTS_COUNTER: &str = "ingestion_pipeline_results";+pub const INGESTION_WARNING_COUNTER: &str = "ingestion_warnings_total";++#[derive(Debug, Clone, Copy)]+pub enum BudgetCheckpoint {+ Step,+ Chunk,+}++impl BudgetCheckpoint {+ fn label(self) -> &'static str {+ match self {+ BudgetCheckpoint::Step => "step",+ BudgetCheckpoint::Chunk => "chunk",+ }+ }+}++pub fn record_budget_checkpoint(checkpoint: BudgetCheckpoint, step_name: Arc<str>) {+ metrics::counter!(+ BATCH_BUDGET_CHECKPOINT_COUNTER,+ "checkpoint" => checkpoint.label(),+ "step_name" => step_name+ )+ .increment(1);+}++pub fn record_step_duration(+ step_name: Arc<str>,+ step_type: &'static str,+ result: &'static str,+ seconds: f64,+) {+ metrics::histogram!(+ PIPELINE_STEP_DURATION_HISTOGRAM,+ "step_name" => step_name,+ "step_type" => step_type,+ "result" => result+ )+ .record(seconds);+}++pub fn record_side_effect_result(ok: bool) {+ metrics::counter!(SIDE_EFFECT_RESULT_COUNTER, "result" => if ok { "ok" } else { "error" })+ .increment(1);+}++pub fn record_retry_attempts(name: Arc<str>, outcome: &'static str, attempts: u32) {+ metrics::histogram!(+ PIPELINE_RETRY_ATTEMPTS_HISTOGRAM,+ "name" => name,+ "outcome" => outcome+ )+ .record(attempts as f64);+}diff --git a/rust/ingestion-framework/src/outputs.rs b/rust/ingestion-framework/src/outputs.rsnew file mode 100644index 000000000000..bd55de251692--- /dev/null+++ b/rust/ingestion-framework/src/outputs.rs@@ -0,0 +1,179 @@+use std::sync::{Arc, Mutex};++use async_trait::async_trait;+pub use bytes::Bytes;++pub const DLQ_OUTPUT: &str = "dlq";+pub const INGESTION_WARNINGS_OUTPUT: &str = "ingestion_warnings";++#[derive(Debug, Clone, Default, PartialEq)]+pub struct KafkaMessage {+ pub topic: String,+ pub partition: i32,+ pub offset: i64,+ pub key: Option<Vec<u8>>,+ pub value: Option<Bytes>,+ /// In wire order. A header can repeat, as in Kafka.+ pub headers: Vec<(String, Option<Vec<u8>>)>,+ pub timestamp: Option<i64>,+}++impl KafkaMessage {+ /// The last value of a repeated key wins, as in `parseKafkaHeaders`.+ pub fn header(&self, name: &str) -> Option<String> {+ self.headers+ .iter()+ .rev()+ .find(|(key, _)| key == name)+ .and_then(|(_, value)| value.as_ref())+ .map(|value| String::from_utf8_lossy(value).into_owned())+ }+}++/// Headers in insertion order. `set` replaces a key in place, as a JS object does.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct Headers(pub Vec<(String, String)>);++impl Headers {+ pub fn new() -> Self {+ Self::default()+ }++ pub fn set(&mut self, key: impl Into<String>, value: impl Into<String>) {+ let key = key.into();+ let value = value.into();+ match self.0.iter_mut().find(|(k, _)| *k == key) {+ Some(entry) => entry.1 = value,+ None => self.0.push((key, value)),+ }+ }++ pub fn get(&self, key: &str) -> Option<&str> {+ self.0+ .iter()+ .find(|(k, _)| k == key)+ .map(|(_, v)| v.as_str())+ }++ pub fn remove(&mut self, key: &str) {+ self.0.retain(|(k, _)| k != key);+ }++ pub fn iter(&self) -> impl Iterator<Item = &(String, String)> {+ self.0.iter()+ }++ pub fn len(&self) -> usize {+ self.0.len()+ }++ pub fn is_empty(&self) -> bool {+ self.0.is_empty()+ }+}++impl<K: Into<String>, V: Into<String>> FromIterator<(K, V)> for Headers {+ fn from_iter<I: IntoIterator<Item = (K, V)>>(iter: I) -> Self {+ let mut headers = Headers::new();+ for (k, v) in iter {+ headers.set(k, v);+ }+ headers+ }+}++#[derive(Debug, Clone, Default, PartialEq)]+pub struct OutputMessage {+ pub value: Option<Bytes>,+ pub key: Option<Vec<u8>>,+ pub headers: Headers,+}++#[async_trait]+pub trait IngestionOutputs: Send + Sync {+ async fn produce(&self, output: &str, message: OutputMessage) -> anyhow::Result<()>;+ /// Resolves on the ack, as Node's `queueMessages` does.+ async fn queue_messages(+ &self,+ output: &str,+ messages: Vec<OutputMessage>,+ ) -> anyhow::Result<()>;+}++#[derive(Default, Clone)]+pub struct MemoryOutputs {+ inner: Arc<Mutex<MemoryOutputsState>>,+}++#[derive(Default)]+struct MemoryOutputsState {+ produced: Vec<(String, OutputMessage)>,+ fail_outputs: Vec<String>,+}++impl MemoryOutputs {+ pub fn new() -> Self {+ Self::default()+ }++ pub fn produced(&self) -> Vec<(String, OutputMessage)> {+ self.inner.lock().unwrap().produced.clone()+ }++ pub fn produced_to(&self, output: &str) -> Vec<OutputMessage> {+ self.inner+ .lock()+ .unwrap()+ .produced+ .iter()+ .filter(|(o, _)| o == output)+ .map(|(_, m)| m.clone())+ .collect()+ }++ pub fn fail_output(&self, output: &str) {+ self.inner+ .lock()+ .unwrap()+ .fail_outputs+ .push(output.to_string());+ }++ pub fn clear_failures(&self) {+ self.inner.lock().unwrap().fail_outputs.clear();+ }++ fn record(&self, output: &str, messages: Vec<OutputMessage>) -> anyhow::Result<()> {+ let mut state = self.inner.lock().unwrap();+ if state.fail_outputs.iter().any(|o| o == output) {+ anyhow::bail!("produce to {output} failed");+ }+ state+ .produced+ .extend(messages.into_iter().map(|m| (output.to_string(), m)));+ Ok(())+ }+}++#[async_trait]+impl IngestionOutputs for MemoryOutputs {+ async fn produce(&self, output: &str, message: OutputMessage) -> anyhow::Result<()> {+ self.record(output, vec![message])+ }++ async fn queue_messages(+ &self,+ output: &str,+ messages: Vec<OutputMessage>,+ ) -> anyhow::Result<()> {+ self.record(output, messages)+ }+}++pub trait HasMessage {+ fn message(&self) -> &KafkaMessage;+}++pub trait HasTeamId {+ fn team_id(&self) -> i64;+}diff --git a/rust/ingestion-framework/src/per_element.rs b/rust/ingestion-framework/src/per_element.rsnew file mode 100644index 000000000000..ca3488d65e16--- /dev/null+++ b/rust/ingestion-framework/src/per_element.rs@@ -0,0 +1,75 @@+use std::future::Future;+use std::sync::{Arc, Mutex};++use futures::stream::{FuturesOrdered, StreamExt};++use crate::steps::{ChunkProcessingStep, ProcessingStep};+use crate::sync::FifoLimiter;++/// Sized up front, because above `JOIN_ALL_SMALL` `join_all` grows its output one push at a time.+async fn join_all_presized<F: Future>(runs: Vec<F>) -> Vec<F::Output> {+ if runs.len() <= JOIN_ALL_SMALL {+ return futures::future::join_all(runs).await;+ }+ let mut results = Vec::with_capacity(runs.len());+ let mut ordered: FuturesOrdered<F> = runs.into_iter().collect();+ while let Some(result) = ordered.next().await {+ results.push(result);+ }+ results+}++/// Mirrors `futures_util::future::join_all::SMALL`.+const JOIN_ALL_SMALL: usize = 30;++/// Steps run inside one future, not as tasks, as Node runs them on one event loop. The first error fails the chunk after the rest end.+/// Panics on `Some(0)`, as Node's `pLimit(0)` throws.+pub fn per_element<T, U>(+ step: ProcessingStep<T, U>,+ max_concurrency: Option<usize>,+) -> ChunkProcessingStep<T, U>+where+ T: Send + 'static,+ U: Send + 'static,+{+ let limit = max_concurrency.map(FifoLimiter::new);+ let name = step.name().clone();+ ChunkProcessingStep::new(&name, move |values: Vec<T>| {+ let step = step.clone();+ let limit = limit.clone();+ let first_error: Arc<Mutex<Option<anyhow::Error>>> = Arc::default();+ // Every slot is taken here, in input order, before any step starts.+ let runs: Vec<_> = values+ .into_iter()+ .map(|value| {+ let permit = limit.as_ref().map(|limit| limit.acquire());+ let step = step.clone();+ let first_error = first_error.clone();+ async move {+ let _permit = match permit {+ Some(permit) => Some(permit.await),+ None => None,+ };+ let result = step.call(value).await;+ match result {+ Ok(result) => Some(result),+ Err(error) => {+ first_error.lock().unwrap().get_or_insert(error);+ None+ }+ }+ }+ })+ .collect();+ async move {+ let results = join_all_presized(runs).await;+ if let Some(error) = first_error.lock().unwrap().take() {+ return Err(error);+ }+ Ok(results+ .into_iter()+ .map(|result| result.expect("no error was recorded"))+ .collect())+ }+ })+}diff --git a/rust/ingestion-framework/src/pipeline.rs b/rust/ingestion-framework/src/pipeline.rsnew file mode 100644index 000000000000..9b8f89491a2c--- /dev/null+++ b/rust/ingestion-framework/src/pipeline.rs@@ -0,0 +1,111 @@+use std::sync::Arc;+use std::time::Instant;++use async_trait::async_trait;++use crate::context::{OkResultWithContext, ResultWithContext};+use crate::metrics::{record_budget_checkpoint, record_step_duration, BudgetCheckpoint};+use crate::results::{NonOk, Outcome};+use crate::steps::ProcessingStep;++/// An `Err` is the Node `throw`: it crashes the batch.+#[async_trait]+pub trait Pipeline<I, O, C>: Send + Sync {+ async fn process(+ &self,+ input: OkResultWithContext<I, C>,+ ) -> anyhow::Result<ResultWithContext<O, C>>;+}++pub type DynPipeline<I, O, C> = Arc<dyn Pipeline<I, O, C>>;++pub struct StartPipeline;++#[async_trait]+impl<T: Send + 'static, C: Send + 'static> Pipeline<T, T, C> for StartPipeline {+ async fn process(+ &self,+ input: OkResultWithContext<T, C>,+ ) -> anyhow::Result<ResultWithContext<T, C>> {+ Ok(input.into_result())+ }+}++pub struct StepPipeline<I, M, O, C> {+ step: ProcessingStep<M, O>,+ previous: DynPipeline<I, M, C>,+}++impl<I, M, O, C> StepPipeline<I, M, O, C> {+ pub fn new(step: ProcessingStep<M, O>, previous: DynPipeline<I, M, C>) -> Self {+ Self { step, previous }+ }+}++#[async_trait]+impl<I, M, O, C> Pipeline<I, O, C> for StepPipeline<I, M, O, C>+where+ I: Send + 'static,+ M: Send + 'static,+ O: Send + 'static,+ C: Send + 'static,+{+ async fn process(+ &self,+ input: OkResultWithContext<I, C>,+ ) -> anyhow::Result<ResultWithContext<O, C>> {+ let previous = self.previous.process(input).await?;+ let (value, mut context) = match previous.into_ok() {+ Ok(ok) => (ok.value, ok.context),+ Err((non_ok, context)) => {+ return Ok(ResultWithContext::<M, C>::non_ok(non_ok, context))+ }+ };++ let step_name = self.step.name();+ // lastStep keeps the step that spent the time; the reason names the step that never started.+ if context.budget.as_ref().is_some_and(|b| b.exhausted()) {+ record_budget_checkpoint(BudgetCheckpoint::Step, step_name.clone());+ return Ok(ResultWithContext {+ result: Outcome::NonOk(NonOk::Timeout {+ reason: format!("budget exceeded before {step_name}"),+ }),+ context,+ });+ }++ let started = Instant::now();+ let result = match self.step.call(value).await {+ Ok(result) => {+ record_step_duration(+ step_name.clone(),+ "element",+ result.result_type().as_label(),+ started.elapsed().as_secs_f64(),+ );+ result+ }+ Err(error) => {+ record_step_duration(+ step_name.clone(),+ "element",+ "exception",+ started.elapsed().as_secs_f64(),+ );+ tracing::error!(+ step = %step_name,+ error = %error,+ debug_context = ?context.debug_context,+ "Step threw"+ );+ return Err(error);+ }+ };++ let outcome = context.absorb_step(step_name, result);+ Ok(ResultWithContext {+ result: outcome,+ context,+ })+ }+}diff --git a/rust/ingestion-framework/src/result_handling.rs b/rust/ingestion-framework/src/result_handling.rsnew file mode 100644index 000000000000..1501fed4de05--- /dev/null+++ b/rust/ingestion-framework/src/result_handling.rs@@ -0,0 +1,270 @@+use std::sync::Arc;++use async_trait::async_trait;+use futures::FutureExt;++use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};+use crate::context::{ChunkResults, OkResultWithContext};+use crate::errors::SharedError;+use crate::metrics::INGESTION_PIPELINE_RESULTS_COUNTER;+use crate::outputs::{+ HasMessage, Headers, IngestionOutputs, KafkaMessage, OutputMessage, DLQ_OUTPUT,+};+use crate::results::{NonOk, Outcome};+use crate::side_effects::{SideEffect, SideEffectScheduler};++#[derive(Clone)]+pub struct PipelineConfig {+ pub outputs: Arc<dyn IngestionOutputs>,+ pub scheduler: Arc<dyn SideEffectScheduler>,+ /// Set it only when the consumer stores no offsets after a failed side effect.+ pub reject_on_dlq_failure: bool,+}++fn now_iso() -> String {+ chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true)+}++/// Node stringifies a non-buffer, non-string value, so null is `"null"`.+pub fn copy_and_extend_headers(message: &KafkaMessage, additional: Headers) -> Headers {+ let mut headers = Headers::new();+ for (key, value) in &message.headers {+ let value = match value {+ Some(bytes) => String::from_utf8_lossy(bytes).into_owned(),+ None => "null".to_string(),+ };+ headers.set(key.clone(), value);+ }+ for (key, value) in additional.0 {+ headers.set(key, value);+ }+ headers+}++pub fn dlq_output_message(message: &KafkaMessage, error: &str, step: &str) -> OutputMessage {+ let headers = copy_and_extend_headers(+ message,+ [+ ("dlq_reason", error.to_string()),+ ("dlq_step", step.to_string()),+ ("dlq_timestamp", now_iso()),+ ("dlq_topic", message.topic.clone()),+ ("dlq_partition", message.partition.to_string()),+ ("dlq_offset", message.offset.to_string()),+ ]+ .into_iter()+ .collect(),+ );+ OutputMessage {+ value: message.value.clone(),+ key: message.key.clone(),+ headers,+ }+}++async fn produce_dlq_message(+ outputs: &dyn IngestionOutputs,+ out: OutputMessage,+ team_id: Option<String>,+ step: &str,+ rethrow_on_failure: bool,+) -> anyhow::Result<()> {+ if let Err(dlq_error) = outputs.produce(DLQ_OUTPUT, out).await {+ tracing::error!(step, team_id = ?team_id, error = %dlq_error, "Failed to send event to DLQ");+ if rethrow_on_failure {+ return Err(dlq_error);+ }+ }+ Ok(())+}++pub async fn produce_message_to_dlq(+ outputs: &dyn IngestionOutputs,+ message: &KafkaMessage,+ error: &str,+ step: &str,+ rethrow_on_failure: bool,+) -> anyhow::Result<()> {+ let team_id = message.header("team_id");+ tracing::warn!(step, team_id = ?team_id, error, "Event sent to DLQ");+ let out = dlq_output_message(message, error, step);+ produce_dlq_message(outputs, out, team_id, step, rethrow_on_failure).await+}++pub fn redirect_output_message(+ message: &KafkaMessage,+ step: &str,+ preserve_key: bool,+) -> OutputMessage {+ let mut extra = Headers::new();+ extra.set("redirect-step", step);+ extra.set("redirect-timestamp", now_iso());+ // Dropping the key spreads the stream across partitions, so keep the original key recoverable.+ if !preserve_key {+ if let Some(key) = &message.key {+ extra.set(+ "redirect-original-key",+ String::from_utf8_lossy(key).into_owned(),+ );+ }+ }+ OutputMessage {+ value: message.value.clone(),+ key: if preserve_key {+ message.key.clone()+ } else {+ None+ },+ headers: copy_and_extend_headers(message, extra),+ }+}++async fn produce_redirect(+ outputs: Arc<dyn IngestionOutputs>,+ output: String,+ scheduler: &dyn SideEffectScheduler,+ out: OutputMessage,+ await_ack: bool,+) -> anyhow::Result<()> {+ // Node calls `outputs.produce` before it schedules the promise, so the produce starts here.+ let produce = async move { outputs.produce(&output, out).await }+ .map(|result| result.map_err(|error| SharedError(Arc::new(error))))+ .boxed()+ .shared();+ let scheduled = produce.clone();+ scheduler.schedule(SideEffect::spawn(async move {+ scheduled.await.map_err(anyhow::Error::new)+ }));+ if await_ack {+ produce.await.map_err(anyhow::Error::new)?;+ }+ Ok(())+}++pub async fn redirect_message_to_output(+ outputs: Arc<dyn IngestionOutputs>,+ output: &str,+ scheduler: &dyn SideEffectScheduler,+ message: &KafkaMessage,+ step: &str,+ preserve_key: bool,+ await_ack: bool,+) -> anyhow::Result<()> {+ let out = redirect_output_message(message, step, preserve_key);+ produce_redirect(outputs, output.to_string(), scheduler, out, await_ack).await+}++fn result_details(result: &NonOk) -> (&'static str, String) {+ match result {+ NonOk::Drop { reason } => ("drop", reason.clone()),+ NonOk::Dlq { reason, .. } => ("dlq", reason.clone()),+ NonOk::Redirect {+ output,+ preserve_key,+ ..+ } => ("redirect", format!("{output}(preserve_key={preserve_key})")),+ NonOk::Timeout { reason } => ("timeout", reason.clone()),+ }+}++/// A timeout produces nothing, because the redelivery produces it.+pub struct ResultHandlingPipeline<I, O, CI, CO> {+ pipeline: DynChunkPipeline<I, O, CI, CO>,+ config: PipelineConfig,+}++impl<I, O, CI, CO> ResultHandlingPipeline<I, O, CI, CO> {+ pub fn new(pipeline: DynChunkPipeline<I, O, CI, CO>, config: PipelineConfig) -> Self {+ Self { pipeline, config }+ }+}++#[async_trait]+impl<I, O, CI, CO> ChunkPipeline<I, O, CI, CO> for ResultHandlingPipeline<I, O, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: HasMessage + Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.pipeline.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ let Some(mut results) = self.pipeline.next().await? else {+ return Ok(None);+ };+ for item in results.iter_mut() {+ let step: Arc<str> = item+ .context+ .last_step+ .clone()+ .unwrap_or_else(|| Arc::from("unknown"));+ let non_ok = match &item.result {+ Outcome::Ok(_) => {+ metrics::counter!(+ INGESTION_PIPELINE_RESULTS_COUNTER,+ "result" => "ok",+ "last_step_name" => step,+ "details" => ""+ )+ .increment(1);+ continue;+ }+ Outcome::NonOk(non_ok) => non_ok,+ };+ let (result_label, details) = result_details(non_ok);+ metrics::counter!(+ INGESTION_PIPELINE_RESULTS_COUNTER,+ "result" => result_label,+ "last_step_name" => step.clone(),+ "details" => details+ )+ .increment(1);++ // Produces start on creation, so they are issued in item order, as in Node's loop.+ let side_effect = match non_ok {+ NonOk::Dlq { reason, error } => {+ let error_text = match error {+ Some(error) => error.to_string(),+ None => reason.clone(),+ };+ let message = item.context.data.message();+ let team_id = message.header("team_id");+ tracing::warn!(step = %step, team_id = ?team_id, error = %error_text, "Event sent to DLQ");+ let out = dlq_output_message(message, &error_text, &step);+ let outputs = self.config.outputs.clone();+ let rethrow = self.config.reject_on_dlq_failure;+ SideEffect::spawn(async move {+ produce_dlq_message(outputs.as_ref(), out, team_id, &step, rethrow).await+ })+ }+ NonOk::Drop { reason } => {+ let message = item.context.data.message();+ tracing::debug!(step = %step, reason, team_id = ?message.header("team_id"), "Event dropped");+ continue;+ }+ NonOk::Redirect {+ output,+ preserve_key,+ await_ack,+ ..+ } => {+ let out =+ redirect_output_message(item.context.data.message(), &step, *preserve_key);+ let output = output.clone();+ let await_ack = *await_ack;+ let outputs = self.config.outputs.clone();+ let scheduler = self.config.scheduler.clone();+ SideEffect::spawn(async move {+ produce_redirect(outputs, output, scheduler.as_ref(), out, await_ack).await+ })+ }+ NonOk::Timeout { .. } => continue,+ };+ item.context.side_effects.push(side_effect);+ }+ Ok(Some(results))+ }+}diff --git a/rust/ingestion-framework/src/results.rs b/rust/ingestion-framework/src/results.rsnew file mode 100644index 000000000000..06e2ae3ae22b--- /dev/null+++ b/rust/ingestion-framework/src/results.rs@@ -0,0 +1,222 @@+use std::fmt;+use std::sync::Arc;++use crate::context::PipelineWarning;+use crate::side_effects::SideEffect;++/// Mirrors the Node `PipelineResultType` enum, including its numeric values.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum PipelineResultType {+ Ok = 0,+ Dlq = 1,+ Drop = 2,+ Redirect = 3,+ Timeout = 4,+}++impl PipelineResultType {+ /// The lowercase label the Node metrics use (`PipelineResultType[type].toLowerCase()`).+ pub fn as_label(&self) -> &'static str {+ match self {+ PipelineResultType::Ok => "ok",+ PipelineResultType::Dlq => "dlq",+ PipelineResultType::Drop => "drop",+ PipelineResultType::Redirect => "redirect",+ PipelineResultType::Timeout => "timeout",+ }+ }+}++#[derive(Debug, Clone)]+pub enum NonOk {+ Dlq {+ reason: String,+ error: Option<Arc<anyhow::Error>>,+ },+ Drop {+ reason: String,+ },+ Redirect {+ reason: String,+ output: String,+ preserve_key: bool,+ await_ack: bool,+ },+ /// The budget cut the element off. Its message is not acked, so its source redelivers it.+ Timeout {+ reason: String,+ },+}++impl NonOk {+ pub fn reason(&self) -> &str {+ match self {+ NonOk::Dlq { reason, .. }+ | NonOk::Drop { reason }+ | NonOk::Redirect { reason, .. }+ | NonOk::Timeout { reason } => reason,+ }+ }++ pub fn result_type(&self) -> PipelineResultType {+ match self {+ NonOk::Dlq { .. } => PipelineResultType::Dlq,+ NonOk::Drop { .. } => PipelineResultType::Drop,+ NonOk::Redirect { .. } => PipelineResultType::Redirect,+ NonOk::Timeout { .. } => PipelineResultType::Timeout,+ }+ }+}++#[derive(Debug, Clone)]+pub enum Outcome<T> {+ Ok(T),+ NonOk(NonOk),+}++impl<T> Outcome<T> {+ pub fn is_ok(&self) -> bool {+ matches!(self, Outcome::Ok(_))+ }++ pub fn result_type(&self) -> PipelineResultType {+ match self {+ Outcome::Ok(_) => PipelineResultType::Ok,+ Outcome::NonOk(n) => n.result_type(),+ }+ }++ pub fn value(&self) -> Option<&T> {+ match self {+ Outcome::Ok(v) => Some(v),+ Outcome::NonOk(_) => None,+ }+ }++ pub fn into_value(self) -> Option<T> {+ match self {+ Outcome::Ok(v) => Some(v),+ Outcome::NonOk(_) => None,+ }+ }++ pub fn non_ok(&self) -> Option<&NonOk> {+ match self {+ Outcome::Ok(_) => None,+ Outcome::NonOk(n) => Some(n),+ }+ }++ pub fn reason(&self) -> Option<&str> {+ self.non_ok().map(|n| n.reason())+ }++ pub fn map<U>(self, f: impl FnOnce(T) -> U) -> Outcome<U> {+ match self {+ Outcome::Ok(v) => Outcome::Ok(f(v)),+ Outcome::NonOk(n) => Outcome::NonOk(n),+ }+ }+}++/// A `SideEffect` is a task handle that cannot be copied, so the pipeline moves it where Node keeps a copy.+pub struct PipelineResult<T> {+ pub outcome: Outcome<T>,+ pub side_effects: Vec<SideEffect>,+ pub warnings: Vec<PipelineWarning>,+}++impl<T: fmt::Debug> fmt::Debug for PipelineResult<T> {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("PipelineResult")+ .field("outcome", &self.outcome)+ .field("side_effects", &self.side_effects.len())+ .field("warnings", &self.warnings)+ .finish()+ }+}++impl<T> PipelineResult<T> {+ pub fn new(outcome: Outcome<T>) -> Self {+ Self {+ outcome,+ side_effects: Vec::new(),+ warnings: Vec::new(),+ }+ }++ pub fn with_side_effects(mut self, side_effects: Vec<SideEffect>) -> Self {+ self.side_effects.extend(side_effects);+ self+ }++ pub fn with_side_effect(mut self, side_effect: SideEffect) -> Self {+ self.side_effects.push(side_effect);+ self+ }++ pub fn with_warnings(mut self, warnings: Vec<PipelineWarning>) -> Self {+ self.warnings.extend(warnings);+ self+ }++ pub fn with_warning(mut self, warning: PipelineWarning) -> Self {+ self.warnings.push(warning);+ self+ }++ pub fn is_ok(&self) -> bool {+ self.outcome.is_ok()+ }++ pub fn result_type(&self) -> PipelineResultType {+ self.outcome.result_type()+ }++ pub fn value(&self) -> Option<&T> {+ self.outcome.value()+ }+}++pub fn ok<T>(value: T) -> PipelineResult<T> {+ PipelineResult::new(Outcome::Ok(value))+}++/// `dlq(reason, error?)`. Node defaults a missing error to `new Error(reason)` at DLQ time.+pub fn dlq<T>(reason: impl Into<String>, error: Option<anyhow::Error>) -> PipelineResult<T> {+ PipelineResult::new(Outcome::NonOk(NonOk::Dlq {+ reason: reason.into(),+ error: error.map(Arc::new),+ }))+}++pub fn drop<T>(reason: impl Into<String>) -> PipelineResult<T> {+ PipelineResult::new(Outcome::NonOk(NonOk::Drop {+ reason: reason.into(),+ }))+}++pub fn timeout<T>(reason: impl Into<String>) -> PipelineResult<T> {+ PipelineResult::new(Outcome::NonOk(NonOk::Timeout {+ reason: reason.into(),+ }))+}++/// `redirect(reason, output)` with the Node defaults `preserveKey = true`, `awaitAck = true`.+pub fn redirect<T>(reason: impl Into<String>, output: impl Into<String>) -> PipelineResult<T> {+ redirect_with(reason, output, true, true)+}++pub fn redirect_with<T>(+ reason: impl Into<String>,+ output: impl Into<String>,+ preserve_key: bool,+ await_ack: bool,+) -> PipelineResult<T> {+ PipelineResult::new(Outcome::NonOk(NonOk::Redirect {+ reason: reason.into(),+ output: output.into(),+ preserve_key,+ await_ack,+ }))+}diff --git a/rust/ingestion-framework/src/retry.rs b/rust/ingestion-framework/src/retry.rsnew file mode 100644index 000000000000..cc478f28f8a3--- /dev/null+++ b/rust/ingestion-framework/src/retry.rs@@ -0,0 +1,218 @@+use std::sync::Arc;+use std::time::Duration;++use rand::Rng;+use tokio::time::Instant;++use crate::errors::is_retriable;+use crate::metrics::record_retry_attempts;+use crate::results::{dlq, PipelineResult};+use crate::steps::{ChunkProcessingStep, ProcessingStep};++/// Defaults match `defaultRetryConfig`.+#[derive(Debug, Clone, Default)]+pub struct RetryOptions {+ pub tries: Option<u32>,+ pub sleep_ms: Option<u64>,+ pub backoff_factor: Option<f64>,+ pub max_sleep_ms: Option<u64>,+ pub jitter: Option<f64>,+ pub soft_deadline_ms: Option<u64>,+ pub name: Option<String>,+}++pub const DEFAULT_TRIES: u32 = 3;+pub const DEFAULT_SLEEP_MS: u64 = 100;+pub const DEFAULT_BACKOFF_FACTOR: f64 = 2.0;+pub const DEFAULT_MAX_SLEEP_MS: u64 = 10_000;+pub const DEFAULT_JITTER: f64 = 0.05;++pub async fn retry_if_retriable<T, F, Fut>(+ mut f: F,+ options: &RetryOptions,+) -> (anyhow::Result<T>, u32)+where+ F: FnMut() -> Fut,+ Fut: std::future::Future<Output = anyhow::Result<T>>,+{+ retry_with_input((), |()| f(), options).await+}++/// Every attempt but the last gets a clone, because a retry needs the input again.+pub async fn retry_with_input<V, T, F, Fut>(+ input: V,+ mut f: F,+ options: &RetryOptions,+) -> (anyhow::Result<T>, u32)+where+ V: Clone,+ F: FnMut(V) -> Fut,+ Fut: std::future::Future<Output = anyhow::Result<T>>,+{+ let tries = options.tries.unwrap_or(DEFAULT_TRIES);+ if tries == 0 {+ // Node's loop runs no attempt and reaches this throw.+ return (Err(anyhow::anyhow!("Unreachable error in retry")), 0);+ }+ let backoff = options.backoff_factor.unwrap_or(DEFAULT_BACKOFF_FACTOR);+ let max_sleep = options.max_sleep_ms.unwrap_or(DEFAULT_MAX_SLEEP_MS) as f64;+ let jitter = options.jitter.unwrap_or(DEFAULT_JITTER);+ let soft_deadline = options.soft_deadline_ms.map(Duration::from_millis);+ let started = Instant::now();+ let past_deadline = || soft_deadline.is_some_and(|d| started.elapsed() >= d);++ let mut sleep_ms = options.sleep_ms.unwrap_or(DEFAULT_SLEEP_MS) as f64;+ let mut input = Some(input);+ let mut attempts = 0;+ loop {+ attempts += 1;+ let value = if attempts >= tries {+ input.take().expect("the last attempt takes the input once")+ } else {+ input.clone().expect("input is kept until the last attempt")+ };+ match f(value).await {+ Ok(value) => return (Ok(value), attempts),+ Err(error) => {+ if is_retriable(&error) == Some(false) || attempts >= tries || past_deadline() {+ return (Err(error), attempts);+ }+ let jittered = if jitter > 0.0 {+ sleep_ms * (1.0 - jitter + rand::thread_rng().gen::<f64>() * jitter)+ } else {+ sleep_ms+ };+ tokio::time::sleep(node_timer_delay(jittered)).await;+ sleep_ms = js_min(sleep_ms * backoff, max_sleep);+ if past_deadline() {+ return (Err(error), attempts);+ }+ }+ }+ }+}++// Node's `TIMEOUT_MAX`, the largest delay that `setTimeout` keeps.+const NODE_TIMEOUT_MAX_MS: f64 = 2_147_483_647.0;++/// Node's `setTimeout` uses 1 ms for a NaN, sub-1 or over-`TIMEOUT_MAX` delay, so bad options cannot fail a retry.+fn node_timer_delay(ms: f64) -> Duration {+ let ms = if (1.0..=NODE_TIMEOUT_MAX_MS).contains(&ms) {+ ms+ } else {+ 1.0+ };+ Duration::from_secs_f64(ms / 1000.0)+}++/// `Math.min(a, b)`, which returns NaN when an input is NaN. `f64::min` returns the other input.+fn js_min(a: f64, b: f64) -> f64 {+ if a.is_nan() || b.is_nan() {+ f64::NAN+ } else {+ a.min(b)+ }+}++const NON_RETRIABLE_REASON: &str = "Processing error - non-retriable";++pub fn with_step_retry<T, U>(+ step: ProcessingStep<T, U>,+ options: RetryOptions,+) -> ProcessingStep<T, U>+where+ T: Clone + Send + Sync + 'static,+ U: Send + 'static,+{+ let name: Arc<str> = Arc::from(+ options+ .name+ .clone()+ .unwrap_or_else(|| step.name().to_string()),+ );+ let step_name = step.name().to_string();+ let options = Arc::new(options);+ ProcessingStep::new(&step_name, move |value: T| {+ let step = step.clone();+ let options = options.clone();+ let name = name.clone();+ async move {+ let (result, attempts) =+ retry_with_input(value, |value| step.call(value), &options).await;+ match result {+ Ok(result) => {+ record_retry_attempts(name, "completed", attempts);+ Ok(result)+ }+ Err(error) => {+ let retriable = is_retriable(&error);+ tracing::error!(step = %name, error = %error, ?retriable, "Step failed");+ if retriable == Some(false) {+ record_retry_attempts(name, "non_retriable", attempts);+ Ok(dlq(NON_RETRIABLE_REASON, Some(error)))+ } else {+ record_retry_attempts(name, "exhausted", attempts);+ Err(error)+ }+ }+ }+ }+ })+}++pub fn with_chunk_retry<T, U>(+ step: ChunkProcessingStep<T, U>,+ options: RetryOptions,+) -> ChunkProcessingStep<T, U>+where+ T: Clone + Send + Sync + 'static,+ U: Send + 'static,+{+ let name: Arc<str> = Arc::from(+ options+ .name+ .clone()+ .unwrap_or_else(|| step.name().to_string()),+ );+ let step_name = step.name().to_string();+ let options = Arc::new(options);+ ChunkProcessingStep::new(&step_name, move |values: Vec<T>| {+ let step = step.clone();+ let options = options.clone();+ let name = name.clone();+ async move {+ let count = values.len();+ let (result, attempts) =+ retry_with_input(values, |values| step.call(values), &options).await;+ match result {+ Ok(result) => {+ record_retry_attempts(name, "completed", attempts);+ Ok(result)+ }+ Err(error) => {+ let retriable = is_retriable(&error);+ tracing::error!(step = %name, error = %error, chunk_size = count, ?retriable, "Chunk step failed");+ if retriable == Some(false) {+ record_retry_attempts(name, "non_retriable", attempts);+ let shared = Arc::new(error);+ Ok((0..count)+ .map(|_| {+ let mut r: PipelineResult<U> = dlq(NON_RETRIABLE_REASON, None);+ if let crate::results::Outcome::NonOk(+ crate::results::NonOk::Dlq { error, .. },+ ) = &mut r.outcome+ {+ *error = Some(shared.clone());+ }+ r+ })+ .collect())+ } else {+ record_retry_attempts(name, "exhausted", attempts);+ Err(error)+ }+ }+ }+ }+ })+}diff --git a/rust/ingestion-framework/src/side_effect_handling.rs b/rust/ingestion-framework/src/side_effect_handling.rsnew file mode 100644index 000000000000..637612b2c521--- /dev/null+++ b/rust/ingestion-framework/src/side_effect_handling.rs@@ -0,0 +1,118 @@+use std::sync::Arc;++use async_trait::async_trait;++use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};+use crate::context::{ChunkResults, OkResultWithContext, ResultWithContext};+use crate::metrics::record_side_effect_result;+use crate::pipeline::{DynPipeline, Pipeline};+use crate::side_effects::{SideEffect, SideEffectScheduler};++#[derive(Debug, Clone, Copy, Default)]+pub struct SideEffectHandlingConfig {+ pub await_side_effects: bool,+}++async fn handle(+ scheduler: &dyn SideEffectScheduler,+ config: SideEffectHandlingConfig,+ side_effects: Vec<SideEffect>,+) {+ if side_effects.is_empty() {+ return;+ }+ if config.await_side_effects {+ let settled =+ futures::future::join_all(side_effects.into_iter().map(SideEffect::wait)).await;+ for result in settled {+ record_side_effect_result(result.is_ok());+ }+ } else {+ for side_effect in side_effects {+ scheduler.schedule(side_effect);+ }+ }+}++pub struct SideEffectHandlingPipeline<I, O, CI, CO> {+ sub: DynChunkPipeline<I, O, CI, CO>,+ scheduler: Arc<dyn SideEffectScheduler>,+ config: SideEffectHandlingConfig,+}++impl<I, O, CI, CO> SideEffectHandlingPipeline<I, O, CI, CO> {+ pub fn new(+ sub: DynChunkPipeline<I, O, CI, CO>,+ scheduler: Arc<dyn SideEffectScheduler>,+ config: SideEffectHandlingConfig,+ ) -> Self {+ Self {+ sub,+ scheduler,+ config,+ }+ }+}++#[async_trait]+impl<I, O, CI, CO> ChunkPipeline<I, O, CI, CO> for SideEffectHandlingPipeline<I, O, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.sub.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ let Some(mut results) = self.sub.next().await? else {+ return Ok(None);+ };+ let side_effects: Vec<SideEffect> = results+ .iter_mut()+ .flat_map(|r| std::mem::take(&mut r.context.side_effects))+ .collect();+ handle(self.scheduler.as_ref(), self.config, side_effects).await;+ Ok(Some(results))+ }+}++pub struct SideEffectHandlingProcessor<I, O, C> {+ sub: DynPipeline<I, O, C>,+ scheduler: Arc<dyn SideEffectScheduler>,+ config: SideEffectHandlingConfig,+}++impl<I, O, C> SideEffectHandlingProcessor<I, O, C> {+ pub fn new(+ sub: DynPipeline<I, O, C>,+ scheduler: Arc<dyn SideEffectScheduler>,+ config: SideEffectHandlingConfig,+ ) -> Self {+ Self {+ sub,+ scheduler,+ config,+ }+ }+}++#[async_trait]+impl<I, O, C> Pipeline<I, O, C> for SideEffectHandlingProcessor<I, O, C>+where+ I: Send + 'static,+ O: Send + 'static,+ C: Send + 'static,+{+ async fn process(+ &self,+ input: OkResultWithContext<I, C>,+ ) -> anyhow::Result<ResultWithContext<O, C>> {+ let mut result = self.sub.process(input).await?;+ let side_effects = std::mem::take(&mut result.context.side_effects);+ handle(self.scheduler.as_ref(), self.config, side_effects).await;+ Ok(result)+ }+}diff --git a/rust/ingestion-framework/src/side_effects.rs b/rust/ingestion-framework/src/side_effects.rsnew file mode 100644index 000000000000..efdd69c5d261--- /dev/null+++ b/rust/ingestion-framework/src/side_effects.rs@@ -0,0 +1,245 @@+//! `spawn` polls once on the caller's thread, as a Node async function runs to its first `await`, so produces start in creation order.++use std::collections::BTreeMap;+use std::future::Future;+use std::panic::AssertUnwindSafe;+use std::sync::{Arc, Mutex};+use std::task::{Context, Poll};++use futures::future::{BoxFuture, Shared};+use futures::FutureExt;+use tokio::task::JoinHandle;++enum State {+ Running(JoinHandle<anyhow::Result<()>>),+ Settled(Option<anyhow::Result<()>>),+}++pub struct SideEffect {+ state: State,+}++fn panic_error() -> anyhow::Error {+ anyhow::anyhow!("side effect task failed: panicked")+}++impl SideEffect {+ pub fn spawn<F>(fut: F) -> Self+ where+ F: Future<Output = anyhow::Result<()>> + Send + 'static,+ {+ let mut fut = Box::pin(fut);+ let mut cx = Context::from_waker(futures::task::noop_waker_ref());+ // A Node async function that throws before its first await rejects instead of throwing.+ match std::panic::catch_unwind(AssertUnwindSafe(|| fut.as_mut().poll(&mut cx))) {+ Ok(Poll::Ready(result)) => Self::settled(result),+ // The task polls again with a real waker, so the no-op waker loses no wake-up.+ Ok(Poll::Pending) => Self {+ state: State::Running(tokio::spawn(fut)),+ },+ Err(_) => Self::settled(Err(panic_error())),+ }+ }++ fn settled(result: anyhow::Result<()>) -> Self {+ Self {+ state: State::Settled(Some(result)),+ }+ }++ pub fn resolved() -> Self {+ Self::settled(Ok(()))+ }++ pub fn rejected(error: anyhow::Error) -> Self {+ Self::settled(Err(error))+ }++ pub fn is_finished(&self) -> bool {+ match &self.state {+ State::Running(handle) => handle.is_finished(),+ State::Settled(_) => true,+ }+ }++ /// `None` while the work runs. After a `Some`, no result is left and `wait()` succeeds.+ pub fn try_take(&mut self) -> Option<anyhow::Result<()>> {+ match &mut self.state {+ State::Settled(result) => result.take(),+ State::Running(handle) => {+ if !handle.is_finished() {+ return None;+ }+ // Unconstrained: a long prune exhausts the coop budget and reads done tasks as pending.+ let result = match tokio::task::unconstrained(handle).now_or_never()? {+ Ok(result) => result,+ Err(join_error) => Err(join_failure(join_error)),+ };+ self.state = State::Settled(None);+ Some(result)+ }+ }+ }++ pub async fn wait(self) -> anyhow::Result<()> {+ match self.state {+ State::Settled(result) => result.unwrap_or(Ok(())),+ State::Running(handle) => match handle.await {+ Ok(result) => result,+ Err(join_error) => Err(join_failure(join_error)),+ },+ }+ }++ pub fn into_future(self) -> BoxFuture<'static, anyhow::Result<()>> {+ self.wait().boxed()+ }+}++fn join_failure(join_error: tokio::task::JoinError) -> anyhow::Error {+ anyhow::anyhow!("side effect task failed: {join_error}")+}++impl std::fmt::Debug for SideEffect {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("SideEffect")+ .field("finished", &self.is_finished())+ .finish()+ }+}++pub trait SideEffectScheduler: Send + Sync {+ fn schedule(&self, side_effect: SideEffect);+}++/// Unlike Node, it latches the first failure, so the consumer never stores offsets for a batch whose side effect failed early.+#[derive(Default, Clone)]+pub struct PromiseScheduler {+ inner: Arc<Mutex<SchedulerState>>,+}++// Prune at this size, then at twice the surviving size, so schedule() is amortized O(1).+const MIN_PRUNE_AT: usize = 64;++type Pending = Shared<BoxFuture<'static, Result<(), Arc<anyhow::Error>>>>;++struct SchedulerState {+ pending: BTreeMap<u64, Pending>,+ next_id: u64,+ prune_at: usize,+ failure: Option<Arc<anyhow::Error>>,+}++impl Default for SchedulerState {+ fn default() -> Self {+ Self {+ pending: BTreeMap::new(),+ next_id: 0,+ prune_at: MIN_PRUNE_AT,+ failure: None,+ }+ }+}++fn poll_settled(pending: &Pending) -> Option<Result<(), Arc<anyhow::Error>>> {+ if let Some(result) = pending.peek() {+ return Some(result.clone());+ }+ // Unconstrained: a long prune exhausts the coop budget and reads done tasks as pending.+ tokio::task::unconstrained(pending.clone()).now_or_never()+}++impl SchedulerState {+ fn latch(&mut self, error: Arc<anyhow::Error>) {+ if self.failure.is_none() {+ self.failure = Some(error);+ }+ }++ fn prune(&mut self) {+ let mut failures = Vec::new();+ self.pending+ .retain(|_, pending| match poll_settled(pending) {+ None => true,+ Some(Ok(())) => false,+ Some(Err(error)) => {+ failures.push(error);+ false+ }+ });+ for error in failures {+ self.latch(error);+ }+ self.prune_at = (self.pending.len() * 2).max(MIN_PRUNE_AT);+ }+}++impl PromiseScheduler {+ pub fn new() -> Self {+ Self::default()+ }++ pub fn pending_count(&self) -> usize {+ let mut state = self.inner.lock().unwrap();+ state.prune();+ state.pending.len()+ }++ /// Entries leave only when they settle, so an overlapping call also waits for them and a cancelled call leaves them.+ pub async fn wait_for_all(&self) -> anyhow::Result<()> {+ let snapshot: Vec<(u64, Pending)> = self+ .inner+ .lock()+ .unwrap()+ .pending+ .iter()+ .map(|(id, pending)| (*id, pending.clone()))+ .collect();+ let mut first_failure = None;+ for (id, pending) in snapshot {+ let result = pending.await;+ let mut state = self.inner.lock().unwrap();+ state.pending.remove(&id);+ if let Err(error) = result {+ // The latch keeps the failure if this call is cancelled before it returns.+ state.latch(error.clone());+ first_failure.get_or_insert(error);+ }+ }+ // An overlapping call can take the latched failure first, so keep the one seen here.+ match self.inner.lock().unwrap().failure.take().or(first_failure) {+ Some(error) => Err(crate::errors::shared_error(error)),+ None => Ok(()),+ }+ }++ pub async fn wait_for_all_settled(&self) {+ let _ = self.wait_for_all().await;+ }+}++impl SideEffectScheduler for PromiseScheduler {+ fn schedule(&self, mut side_effect: SideEffect) {+ // Settled on creation, so it skips the set, as a settled Node promise leaves it.+ match side_effect.try_take() {+ Some(Ok(())) => return,+ Some(Err(error)) => {+ self.inner.lock().unwrap().latch(Arc::new(error));+ return;+ }+ None => {}+ }+ let pending = side_effect+ .wait()+ .map(|r| r.map_err(Arc::new))+ .boxed()+ .shared();+ let mut state = self.inner.lock().unwrap();+ let id = state.next_id;+ state.next_id += 1;+ state.pending.insert(id, pending);+ if state.pending.len() >= state.prune_at {+ state.prune();+ }+ }+}diff --git a/rust/ingestion-framework/src/steps.rs b/rust/ingestion-framework/src/steps.rsnew file mode 100644index 000000000000..25c435910579--- /dev/null+++ b/rust/ingestion-framework/src/steps.rs@@ -0,0 +1,136 @@+//! Rust closures have no `.name`, so a step carries its name.++use std::future::Future;+use std::sync::Arc;++use futures::future::BoxFuture;+use futures::FutureExt;++use crate::results::PipelineResult;++pub type StepFuture<U> = BoxFuture<'static, anyhow::Result<PipelineResult<U>>>;+pub type ChunkStepFuture<U> = BoxFuture<'static, anyhow::Result<Vec<PipelineResult<U>>>>;++/// An `Err` is the Node `throw`: it propagates and is not converted into a DLQ.+pub struct ProcessingStep<T, U> {+ name: Arc<str>,+ f: Arc<dyn Fn(T) -> StepFuture<U> + Send + Sync>,+}++impl<T, U> Clone for ProcessingStep<T, U> {+ fn clone(&self) -> Self {+ Self {+ name: self.name.clone(),+ f: self.f.clone(),+ }+ }+}++impl<T: 'static, U: 'static> ProcessingStep<T, U> {+ pub fn new<F, Fut>(name: &str, f: F) -> Self+ where+ F: Fn(T) -> Fut + Send + Sync + 'static,+ Fut: Future<Output = anyhow::Result<PipelineResult<U>>> + Send + 'static,+ {+ Self {+ name: Arc::from(name),+ f: Arc::new(move |value| f(value).boxed()),+ }+ }++ pub fn sync<F>(name: &str, f: F) -> Self+ where+ F: Fn(T) -> anyhow::Result<PipelineResult<U>> + Send + Sync + 'static,+ U: Send,+ {+ Self {+ name: Arc::from(name),+ f: Arc::new(move |value| futures::future::ready(f(value)).boxed()),+ }+ }++ pub fn name(&self) -> &Arc<str> {+ &self.name+ }++ pub fn call(&self, value: T) -> StepFuture<U> {+ (self.f)(value)+ }++ pub fn renamed(mut self, name: &str) -> Self {+ self.name = Arc::from(name);+ self+ }+}++pub struct ChunkProcessingStep<T, U> {+ name: Arc<str>,+ f: Arc<dyn Fn(Vec<T>) -> ChunkStepFuture<U> + Send + Sync>,+}++impl<T, U> Clone for ChunkProcessingStep<T, U> {+ fn clone(&self) -> Self {+ Self {+ name: self.name.clone(),+ f: self.f.clone(),+ }+ }+}++impl<T: 'static, U: 'static> ChunkProcessingStep<T, U> {+ pub fn new<F, Fut>(name: &str, f: F) -> Self+ where+ F: Fn(Vec<T>) -> Fut + Send + Sync + 'static,+ Fut: Future<Output = anyhow::Result<Vec<PipelineResult<U>>>> + Send + 'static,+ {+ Self {+ name: Arc::from(name),+ f: Arc::new(move |values| f(values).boxed()),+ }+ }++ pub fn sync<F>(name: &str, f: F) -> Self+ where+ F: Fn(Vec<T>) -> anyhow::Result<Vec<PipelineResult<U>>> + Send + Sync + 'static,+ U: Send,+ {+ Self {+ name: Arc::from(name),+ f: Arc::new(move |values| futures::future::ready(f(values)).boxed()),+ }+ }++ pub fn name(&self) -> &Arc<str> {+ &self.name+ }++ pub fn call(&self, values: Vec<T>) -> ChunkStepFuture<U> {+ (self.f)(values)+ }+}++impl<T: Send + 'static, U: Send + 'static> ChunkProcessingStep<T, U> {+ pub fn per_item(step: ProcessingStep<T, U>) -> Self {+ crate::per_element::per_element(step, None)+ }+}++pub fn step<T, U, F, Fut>(name: &str, f: F) -> ProcessingStep<T, U>+where+ T: 'static,+ U: 'static,+ F: Fn(T) -> Fut + Send + Sync + 'static,+ Fut: Future<Output = anyhow::Result<PipelineResult<U>>> + Send + 'static,+{+ ProcessingStep::new(name, f)+}++pub fn chunk_step<T, U, F, Fut>(name: &str, f: F) -> ChunkProcessingStep<T, U>+where+ T: 'static,+ U: 'static,+ F: Fn(Vec<T>) -> Fut + Send + Sync + 'static,+ Fut: Future<Output = anyhow::Result<Vec<PipelineResult<U>>>> + Send + 'static,+{+ ChunkProcessingStep::new(name, f)+}diff --git a/rust/ingestion-framework/src/sync.rs b/rust/ingestion-framework/src/sync.rsnew file mode 100644index 000000000000..70f80e0f3224--- /dev/null+++ b/rust/ingestion-framework/src/sync.rs@@ -0,0 +1,135 @@+use std::collections::VecDeque;+use std::future::Future;+use std::sync::{Arc, Mutex};++use tokio::sync::{oneshot, watch};++/// A re-armable wake-up. A `resolve()` between `reset()` and `wait()` is not lost.+pub struct ResettableSignal {+ tx: watch::Sender<u64>,+ baseline: Mutex<u64>,+}++impl Default for ResettableSignal {+ fn default() -> Self {+ let (tx, _) = watch::channel(0);+ Self {+ tx,+ baseline: Mutex::new(0),+ }+ }+}++impl ResettableSignal {+ pub fn new() -> Self {+ Self::default()+ }++ pub fn resolve(&self) {+ self.tx.send_modify(|generation| *generation += 1);+ }++ pub fn reset(&self) {+ *self.baseline.lock().unwrap() = *self.tx.borrow();+ }++ pub fn wait(&self) -> impl Future<Output = ()> + Send + 'static {+ let baseline = *self.baseline.lock().unwrap();+ let mut rx = self.tx.subscribe();+ async move {+ let _ = rx.wait_for(|generation| *generation != baseline).await;+ }+ }+}++/// A caller queues when it calls `acquire()`, not when it first polls, so admission order is call order.+pub struct FifoLimiter {+ max: usize,+ state: Mutex<LimiterState>,+}++struct LimiterState {+ running: usize,+ waiters: VecDeque<oneshot::Sender<LimiterPermit>>,+}++/// Reserved at `acquire()`, so a dropped future or a permit dropped in transit releases the slot.+pub struct LimiterPermit {+ limiter: Option<Arc<FifoLimiter>>,+}++impl FifoLimiter {+ /// Panics when `max` is 0, as `pLimit(0)` throws when the Node pipeline is built.+ pub fn new(max: usize) -> Arc<Self> {+ assert!(+ max > 0,+ "Expected `concurrency` to be a number from 1 and up"+ );+ Arc::new(Self {+ max,+ state: Mutex::new(LimiterState {+ running: 0,+ waiters: VecDeque::new(),+ }),+ })+ }++ pub fn acquire(self: &Arc<Self>) -> impl Future<Output = LimiterPermit> + Send + 'static {+ enum Reserved {+ Now(LimiterPermit),+ Queued(oneshot::Receiver<LimiterPermit>),+ }+ let reserved = {+ let mut state = self.state.lock().unwrap();+ if state.running < self.max && state.waiters.is_empty() {+ state.running += 1;+ Reserved::Now(LimiterPermit {+ limiter: Some(self.clone()),+ })+ } else {+ let (tx, rx) = oneshot::channel();+ state.waiters.push_back(tx);+ Reserved::Queued(rx)+ }+ };+ async move {+ match reserved {+ Reserved::Now(permit) => permit,+ // Every queued sender gets a send, so this receive does not fail.+ Reserved::Queued(rx) => rx.await.unwrap_or(LimiterPermit { limiter: None }),+ }+ }+ }++ fn release(self: &Arc<Self>) {+ loop {+ let waiter = {+ let mut state = self.state.lock().unwrap();+ match state.waiters.pop_front() {+ Some(waiter) => waiter,+ None => {+ state.running -= 1;+ return;+ }+ }+ };+ // The slot moves to the next waiter. A waiter that is gone hands it back.+ match waiter.send(LimiterPermit {+ limiter: Some(self.clone()),+ }) {+ Ok(()) => return,+ Err(mut permit) => {+ permit.limiter = None;+ }+ }+ }+ }+}++impl Drop for LimiterPermit {+ fn drop(&mut self) {+ if let Some(limiter) = self.limiter.take() {+ limiter.release();+ }+ }+}diff --git a/rust/ingestion-framework/src/timeout_barrier.rs b/rust/ingestion-framework/src/timeout_barrier.rsnew file mode 100644index 000000000000..97b3e36d8f7f--- /dev/null+++ b/rust/ingestion-framework/src/timeout_barrier.rs@@ -0,0 +1,56 @@+use std::sync::Arc;++use async_trait::async_trait;++use crate::budget::BatchBudget;+use crate::context::{OkResultWithContext, ResultWithContext};+use crate::metrics::{record_budget_checkpoint, BudgetCheckpoint};+use crate::pipeline::{DynPipeline, Pipeline};+use crate::results::{NonOk, Outcome};++const TIMEOUT_BARRIER: &str = "timeoutBarrier";++/// An element with time left continues with an unlimited budget, so it always finishes.+pub struct TimeoutBarrierPipeline<I, O, C> {+ previous: DynPipeline<I, O, C>,+}++impl<I, O, C> TimeoutBarrierPipeline<I, O, C> {+ pub fn new(previous: DynPipeline<I, O, C>) -> Self {+ Self { previous }+ }+}++#[async_trait]+impl<I, O, C> Pipeline<I, O, C> for TimeoutBarrierPipeline<I, O, C>+where+ I: Send + 'static,+ O: Send + 'static,+ C: Send + 'static,+{+ async fn process(+ &self,+ input: OkResultWithContext<I, C>,+ ) -> anyhow::Result<ResultWithContext<O, C>> {+ let mut result = self.previous.process(input).await?;+ let Some(budget) = result.context.budget.as_ref() else {+ return Ok(result);+ };+ if !result.is_ok() {+ return Ok(result);+ }+ if budget.exhausted() {+ record_budget_checkpoint(BudgetCheckpoint::Step, Arc::from(TIMEOUT_BARRIER));+ return Ok(ResultWithContext {+ result: Outcome::NonOk(NonOk::Timeout {+ reason: format!("budget exceeded before {TIMEOUT_BARRIER}"),+ }),+ context: result.context,+ });+ }+ if !budget.is_unlimited() {+ result.context.budget = Some(BatchBudget::unlimited());+ }+ Ok(result)+ }+}diff --git a/rust/ingestion-framework/src/warning_handling.rs b/rust/ingestion-framework/src/warning_handling.rsnew file mode 100644index 000000000000..6d52995e07f9--- /dev/null+++ b/rust/ingestion-framework/src/warning_handling.rs@@ -0,0 +1,190 @@+use std::collections::HashMap;+use std::sync::{Arc, OnceLock};++use async_trait::async_trait;++use crate::chunk_pipeline::{ChunkPipeline, DynChunkPipeline};+use crate::context::{ChunkResults, OkResultWithContext, PipelineWarning};+use crate::js_json;+use crate::metrics::INGESTION_WARNING_COUNTER;+use crate::outputs::{HasTeamId, IngestionOutputs, OutputMessage, INGESTION_WARNINGS_OUTPUT};+use crate::results::{NonOk, Outcome};+use crate::side_effects::SideEffect;++/// Node logs a failed produce and does not reject, so that is `Ok(false)`. `Err` fails the side effect, as Node's emit rejects.+#[async_trait]+pub trait IngestionWarningEmitter: Send + Sync {+ async fn emit(&self, team_id: i64, warning: PipelineWarning) -> anyhow::Result<bool>;+}++pub struct OutputsWarningEmitter {+ pub outputs: Arc<dyn IngestionOutputs>,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct WarningClassification {+ pub category: &'static str,+ pub severity: &'static str,+}++// The Node registry exports this file, and a Node test fails when the two differ.+const WARNING_TYPES_JSON: &str =+ include_str!("../../common/ingestion_warnings/warning_types.generated.json");++pub fn warning_classification(warning_type: &str) -> Option<WarningClassification> {+ #[derive(serde::Deserialize)]+ struct Entry<'a> {+ #[serde(rename = "type")]+ warning_type: &'a str,+ category: &'a str,+ severity: &'a str,+ }+ #[derive(serde::Deserialize)]+ struct Registry<'a> {+ #[serde(borrow)]+ types: Vec<Entry<'a>>,+ }+ static REGISTRY: OnceLock<HashMap<&'static str, WarningClassification>> = OnceLock::new();+ REGISTRY+ .get_or_init(|| {+ let registry: Registry<'static> =+ serde_json::from_str(WARNING_TYPES_JSON).expect("valid warning types JSON");+ registry+ .types+ .into_iter()+ .map(|entry| {+ let classification = WarningClassification {+ category: entry.category,+ severity: entry.severity,+ };+ (entry.warning_type, classification)+ })+ .collect()+ })+ .get(warning_type)+ .copied()+}++/// Byte for byte as Node. A key the caller already set keeps its place and takes the new value, as in an object spread.+pub fn serialize_ingestion_warning(+ team_id: i64,+ warning: &PipelineWarning,+) -> anyhow::Result<String> {+ let Some(classification) = warning_classification(&warning.warning_type) else {+ anyhow::bail!(+ "ingestion warning type {:?} is not in INGESTION_WARNING_TYPES",+ warning.warning_type+ );+ };+ let mut details = warning.details.clone();+ details.insert("category".into(), classification.category.into());+ details.insert("severity".into(), classification.severity.into());+ if let Some(step) = warning.pipeline_step.as_deref().filter(|s| !s.is_empty()) {+ details.insert("pipelineStep".into(), step.into());+ }+ let mut details_json = String::new();+ js_json::write_object(&mut details_json, details.iter());++ let mut out = String::with_capacity(details_json.len() + 128);+ out.push_str("{\"team_id\":");+ js_json::write_number(&mut out, team_id as f64);+ out.push_str(",\"type\":");+ js_json::write_string(&mut out, &warning.warning_type);+ out.push_str(",\"source\":");+ js_json::write_string(+ &mut out,+ warning.source.as_deref().unwrap_or("plugin-server"),+ );+ out.push_str(",\"details\":");+ js_json::write_string(&mut out, &details_json);+ out.push_str(",\"timestamp\":");+ js_json::write_string(+ &mut out,+ &chrono::Utc::now()+ .format("%Y-%m-%d %H:%M:%S%.3f")+ .to_string(),+ );+ out.push('}');+ Ok(out)+}++#[async_trait]+impl IngestionWarningEmitter for OutputsWarningEmitter {+ async fn emit(&self, team_id: i64, warning: PipelineWarning) -> anyhow::Result<bool> {+ metrics::counter!(INGESTION_WARNING_COUNTER, "type" => warning.warning_type.clone(), "emitted" => "true")+ .increment(1);+ let value = serialize_ingestion_warning(team_id, &warning)?.into_bytes();+ match self+ .outputs+ .queue_messages(+ INGESTION_WARNINGS_OUTPUT,+ vec![OutputMessage {+ value: Some(value.into()),+ ..Default::default()+ }],+ )+ .await+ {+ Ok(()) => Ok(true),+ Err(error) => {+ tracing::warn!(team_id, error = %error, "Failed to produce ingestion warning");+ Ok(false)+ }+ }+ }+}++/// A timed-out result drops its warnings, because the redelivery emits them again.+pub struct IngestionWarningHandlingChunkPipeline<I, O, CI, CO> {+ emitter: Arc<dyn IngestionWarningEmitter>,+ previous: DynChunkPipeline<I, O, CI, CO>,+}++impl<I, O, CI, CO> IngestionWarningHandlingChunkPipeline<I, O, CI, CO> {+ pub fn new(+ emitter: Arc<dyn IngestionWarningEmitter>,+ previous: DynChunkPipeline<I, O, CI, CO>,+ ) -> Self {+ Self { emitter, previous }+ }+}++#[async_trait]+impl<I, O, CI, CO> ChunkPipeline<I, O, CI, CO>+ for IngestionWarningHandlingChunkPipeline<I, O, CI, CO>+where+ I: Send + 'static,+ O: Send + 'static,+ CI: Send + 'static,+ CO: HasTeamId + Send + 'static,+{+ fn feed(&self, elements: Vec<OkResultWithContext<I, CI>>) {+ self.previous.feed(elements)+ }++ async fn next(&self) -> anyhow::Result<Option<ChunkResults<O, CO>>> {+ let Some(mut results) = self.previous.next().await? else {+ return Ok(None);+ };+ for item in results.iter_mut() {+ if item.context.warnings.is_empty() {+ continue;+ }+ let warnings = std::mem::take(&mut item.context.warnings);+ if matches!(item.result, Outcome::NonOk(NonOk::Timeout { .. })) {+ continue;+ }+ let team_id = item.context.data.team_id();+ for warning in warnings {+ let emitter = self.emitter.clone();+ // The emit starts on creation, so warnings are queued in item order.+ item.context+ .side_effects+ .push(SideEffect::spawn(async move {+ emitter.emit(team_id, warning).await.map(|_| ())+ }));+ }+ }+ Ok(Some(results))+ }+}diff --git a/rust/ingestion-framework/tests/common/mod.rs b/rust/ingestion-framework/tests/common/mod.rsnew file mode 100644index 000000000000..4223dda424f4--- /dev/null+++ b/rust/ingestion-framework/tests/common/mod.rs@@ -0,0 +1,153 @@+#![allow(dead_code)]++use std::sync::{Arc, Mutex};+use std::time::Duration;++use ingestion_framework::{create_ok_context, ChunkPipeline, OkResultWithContext};++pub type Ctx = ();++pub fn items<T>(values: impl IntoIterator<Item = T>) -> Vec<OkResultWithContext<T, Ctx>> {+ values+ .into_iter()+ .map(|v| create_ok_context(v, ()))+ .collect()+}++/// A paused tokio clock advances when all tasks wait, like `jest.advanceTimersByTimeAsync`.+pub async fn consume_all<I, O, CI, CO>(pipeline: &dyn ChunkPipeline<I, O, CI, CO>) -> Vec<O> {+ let mut values = Vec::new();+ while let Some(chunk) = pipeline.next().await.expect("next") {+ for r in chunk {+ if let Some(v) = r.result.into_value() {+ values.push(v);+ }+ }+ }+ values+}++/// The bound is 1 ms past `advance_ms`, because tokio fires a timer and this timeout in the same millisecond tick.+pub async fn consume_all_within<I, O, CI, CO>(+ pipeline: &dyn ChunkPipeline<I, O, CI, CO>,+ advance_ms: u64,+) -> Vec<O> {+ within_ms(advance_ms, consume_all(pipeline)).await+}++pub async fn within_ms<T>(advance_ms: u64, fut: impl std::future::Future<Output = T>) -> T {+ tokio::time::timeout(Duration::from_millis(advance_ms + 1), fut)+ .await+ .unwrap_or_else(|_| panic!("did not finish within {advance_ms} ms of virtual time"))+}++pub async fn collect_chunks<I, O, CI, CO>(+ pipeline: &dyn ChunkPipeline<I, O, CI, CO>,+) -> Vec<Vec<O>> {+ let mut chunks = Vec::new();+ while let Some(chunk) = pipeline.next().await.expect("next") {+ chunks.push(+ chunk+ .into_iter()+ .filter_map(|r| r.result.into_value())+ .collect(),+ );+ }+ chunks+}++pub async fn sleep_ms(ms: u64) {+ tokio::time::sleep(Duration::from_millis(ms)).await+}++pub fn shared<T>(value: T) -> Arc<Mutex<T>> {+ Arc::new(Mutex::new(value))+}++use async_trait::async_trait;+use ingestion_framework::{IngestionOutputs, KafkaMessage, OutputMessage};++#[derive(Clone)]+pub struct MockOutputs {+ pub calls: Arc<Mutex<Vec<(String, OutputMessage)>>>,+ gate: Arc<tokio::sync::watch::Sender<bool>>,+}++impl Default for MockOutputs {+ fn default() -> Self {+ let (tx, _) = tokio::sync::watch::channel(true);+ Self {+ calls: Arc::default(),+ gate: Arc::new(tx),+ }+ }+}++impl MockOutputs {+ pub fn close_gate(&self) {+ self.gate.send_replace(false);+ }+ pub fn open_gate(&self) {+ self.gate.send_replace(true);+ }+ pub fn calls(&self) -> Vec<(String, OutputMessage)> {+ self.calls.lock().unwrap().clone()+ }+}++#[async_trait]+impl IngestionOutputs for MockOutputs {+ async fn produce(&self, output: &str, message: OutputMessage) -> anyhow::Result<()> {+ self.calls+ .lock()+ .unwrap()+ .push((output.to_string(), message));+ let mut rx = self.gate.subscribe();+ let _ = rx.wait_for(|open| *open).await;+ Ok(())+ }+ async fn queue_messages(+ &self,+ output: &str,+ messages: Vec<OutputMessage>,+ ) -> anyhow::Result<()> {+ for m in messages {+ self.produce(output, m).await?;+ }+ Ok(())+ }+}++pub fn test_message(key: Option<&[u8]>) -> KafkaMessage {+ KafkaMessage {+ topic: "test-topic".into(),+ partition: 0,+ offset: 0,+ key: key.map(|k| k.to_vec()),+ value: Some(b"test-value".to_vec().into()),+ headers: vec![],+ timestamp: Some(0),+ }+}++#[derive(Debug, Clone)]+pub struct MsgCtx {+ pub message: KafkaMessage,+}++impl ingestion_framework::HasMessage for MsgCtx {+ fn message(&self) -> &KafkaMessage {+ &self.message+ }+}++#[derive(Debug, Clone)]+pub struct TeamCtx {+ pub team_id: i64,+}++impl ingestion_framework::HasTeamId for TeamCtx {+ fn team_id(&self) -> i64 {+ self.team_id+ }+}diff --git a/rust/ingestion-framework/tests/docs_01_introduction.rs b/rust/ingestion-framework/tests/docs_01_introduction.rsnew file mode 100644index 000000000000..77a0acf2cb05--- /dev/null+++ b/rust/ingestion-framework/tests/docs_01_introduction.rs@@ -0,0 +1,255 @@+mod common;++use std::sync::Arc;++use common::{items, shared};+use ingestion_framework::{+ chunk_step, create_ok_context, dlq, drop, new_chunk_pipeline, new_pipeline, ok, redirect, step,+ PipelineResultType, ProcessingStep,+};++#[tokio::test]+async fn steps_use_the_factory_pattern_for_dependency_injection_and_naming() {+ fn create_uppercase_step() -> ProcessingStep<String, String> {+ step("uppercaseStep", |input: String| async move {+ Ok(ok(input.to_uppercase()))+ })+ }+ struct MultiplierConfig {+ factor: i64,+ }+ fn create_multiply_step(config: MultiplierConfig) -> ProcessingStep<i64, i64> {+ step("multiplyStep", move |input: i64| async move {+ Ok(ok(input * config.factor))+ })+ }++ let string_pipeline = new_pipeline::<String, ()>()+ .pipe(create_uppercase_step())+ .build();+ let number_pipeline = new_pipeline::<i64, ()>()+ .pipe(create_multiply_step(MultiplierConfig { factor: 3 }))+ .build();++ let r = string_pipeline+ .process(create_ok_context("hello".into(), ()))+ .await+ .unwrap();+ assert_eq!(r.value().map(String::as_str), Some("HELLO"));+ let r = number_pipeline+ .process(create_ok_context(5, ()))+ .await+ .unwrap();+ assert_eq!(r.value(), Some(&15));+}++#[tokio::test]+async fn steps_accumulate_data_through_the_pipeline() {+ #[derive(Clone)]+ struct Input {+ event_id: String,+ }+ struct Validated {+ event_id: String,+ is_valid: bool,+ }+ struct Enriched {+ event_id: String,+ is_valid: bool,+ timestamp: i64,+ }+ let pipeline = new_pipeline::<Input, ()>()+ .pipe(step("validateStep", |i: Input| async move {+ Ok(ok(Validated {+ event_id: i.event_id,+ is_valid: true,+ }))+ }))+ .pipe(step("enrichStep", |i: Validated| async move {+ Ok(ok(Enriched {+ event_id: i.event_id,+ is_valid: i.is_valid,+ timestamp: chrono::Utc::now().timestamp_millis(),+ }))+ }))+ .pipe(step("finalizeStep", |i: Enriched| async move {+ Ok(ok(format!(+ "Event {} valid={} at {}",+ i.event_id, i.is_valid, i.timestamp+ )))+ }))+ .build();++ let r = pipeline+ .process(create_ok_context(+ Input {+ event_id: "evt-123".into(),+ },+ (),+ ))+ .await+ .unwrap();+ let summary = r.value().unwrap();+ assert!(summary.contains("evt-123"));+ assert!(summary.contains("valid=true"));+}++#[tokio::test]+async fn a_step_can_return_ok_to_pass_data_forward() {+ let pipeline = new_pipeline::<String, ()>()+ .pipe(step("processDataStep", |i: String| async move {+ Ok(ok(i.to_uppercase()))+ }))+ .build();+ let r = pipeline+ .process(create_ok_context("hello".into(), ()))+ .await+ .unwrap();+ assert_eq!(r.value().unwrap(), "HELLO");+}++#[tokio::test]+async fn a_pipeline_transforms_input_through_a_sequence_of_steps() {+ let pipeline = new_pipeline::<String, ()>()+ .pipe(step("uppercaseStep", |i: String| async move {+ Ok(ok(i.to_uppercase()))+ }))+ .pipe(step("addExclamationStep", |i: String| async move {+ Ok(ok(i + "!"))+ }))+ .build();+ let r = pipeline+ .process(create_ok_context("hello".into(), ()))+ .await+ .unwrap();+ assert_eq!(r.value().unwrap(), "HELLO!");+}++#[tokio::test]+async fn a_step_can_return_dlq_to_signal_an_error() {+ let pipeline = new_pipeline::<i64, ()>()+ .pipe(step("validateDataStep", |v: i64| async move {+ if v < 0 {+ return Ok(dlq(+ "Invalid value: must be non-negative",+ Some(anyhow::anyhow!("Validation failed")),+ ));+ }+ Ok(ok(v))+ }))+ .build();+ let r = pipeline.process(create_ok_context(-1, ())).await.unwrap();+ assert_eq!(r.result.result_type(), PipelineResultType::Dlq);+ assert_eq!(r.result.result_type() as i32, 1);+}++#[tokio::test]+async fn a_step_can_return_drop_to_silently_discard_an_item() {+ let pipeline = new_pipeline::<&'static str, ()>()+ .pipe(step("filterInternalStep", |t: &'static str| async move {+ if t == "internal" {+ return Ok(drop("Internal events are not processed"));+ }+ Ok(ok(t))+ }))+ .build();+ let r = pipeline+ .process(create_ok_context("internal", ()))+ .await+ .unwrap();+ assert_eq!(r.result.result_type() as i32, 2);+}++#[tokio::test]+async fn a_step_can_return_redirect_to_send_an_item_elsewhere() {+ let pipeline = new_pipeline::<&'static str, ()>()+ .pipe(step("routeByPriorityStep", |p: &'static str| async move {+ if p == "high" {+ return Ok(redirect("High priority event", "high_priority"));+ }+ Ok(ok(p))+ }))+ .build();+ let r = pipeline+ .process(create_ok_context("high", ()))+ .await+ .unwrap();+ assert_eq!(r.result.result_type() as i32, 3);+}++#[tokio::test]+async fn non_ok_results_short_circuit_the_pipeline_ok_results_propagate() {+ let pipeline = new_pipeline::<&'static str, ()>()+ .pipe(step("decisionStep", |action: &'static str| async move {+ Ok(match action {+ "dlq" => dlq("Error condition", None),+ "drop" => drop("Filtered out"),+ "redirect" => redirect("Redirecting", "other"),+ _ => ok(action),+ })+ }))+ .pipe(step("processStep", |action: &'static str| async move {+ if action != "ok" {+ anyhow::bail!("Unexpected action: {action}");+ }+ Ok(ok((action, true)))+ }))+ .build();++ let r = pipeline.process(create_ok_context("ok", ())).await.unwrap();+ assert!(r.is_ok());+ assert_eq!(r.context.last_step(), Some("processStep"));+ for (action, expected) in [("dlq", 1), ("drop", 2), ("redirect", 3)] {+ let r = pipeline+ .process(create_ok_context(action, ()))+ .await+ .unwrap();+ assert_eq!(r.result.result_type() as i32, expected);+ assert_eq!(r.context.last_step(), Some("decisionStep"));+ }+}++#[tokio::test]+async fn chunk_pipelines_are_created_using_new_chunk_pipeline_builder() {+ let pipeline = new_chunk_pipeline::<String, ()>()+ .pipe_chunk(chunk_step(+ "uppercaseChunkStep",+ |items: Vec<String>| async move {+ Ok(items.into_iter().map(|s| ok(s.to_uppercase())).collect())+ },+ ))+ .build();+ pipeline.feed(items(["a", "b", "c"].map(String::from)));+ let results = pipeline.next().await.unwrap().unwrap();+ let values: Vec<_> = results.iter().map(|r| r.value().unwrap().clone()).collect();+ assert_eq!(values, ["A", "B", "C"]);+}++#[tokio::test]+async fn pipe_chunk_processes_all_items_in_a_single_call() {+ let call_counts = shared(Vec::<usize>::new());+ let counts = call_counts.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("chunkEnrichStep", move |items: Vec<i64>| {+ counts.lock().unwrap().push(items.len());+ async move { Ok(items.into_iter().map(|n| ok(n * 10)).collect()) }+ }))+ .build();+ pipeline.feed(items([1, 2, 3, 4, 5]));+ pipeline.next().await.unwrap();+ assert_eq!(*call_counts.lock().unwrap(), vec![5]);+}++#[tokio::test]+async fn feed_accepts_a_batch_and_next_returns_processed_chunks() {+ let pipeline = new_chunk_pipeline::<&'static str, ()>()+ .pipe_chunk(chunk_step(+ "passthroughStep",+ |items: Vec<&'static str>| async move { Ok(items.into_iter().map(ok).collect()) },+ ))+ .build();+ pipeline.feed(items(["x", "y"]));+ assert_eq!(pipeline.next().await.unwrap().unwrap().len(), 2);+ assert!(pipeline.next().await.unwrap().is_none());+ let _ = Arc::new(());+}diff --git a/rust/ingestion-framework/tests/docs_02_chunk_pipelines.rs b/rust/ingestion-framework/tests/docs_02_chunk_pipelines.rsnew file mode 100644index 000000000000..66346a6c2ceb--- /dev/null+++ b/rust/ingestion-framework/tests/docs_02_chunk_pipelines.rs@@ -0,0 +1,131 @@+mod common;++use common::{collect_chunks, items, shared};+use ingestion_framework::{chunk_step, dlq, new_chunk_pipeline, ok, step};++#[tokio::test]+async fn chunk_pipelines_process_multiple_items_at_once() {+ let calls = shared(0);+ let c = calls.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("chunkDoubleStep", move |items: Vec<i64>| {+ *c.lock().unwrap() += 1;+ async move { Ok(items.into_iter().map(|n| ok(n * 2)).collect()) }+ }))+ .build();+ pipeline.feed(items([1, 2, 3, 4, 5]));+ let results = pipeline.next().await.unwrap().unwrap();+ assert_eq!(*calls.lock().unwrap(), 1);+ let v: Vec<_> = results.iter().map(|r| *r.value().unwrap()).collect();+ assert_eq!(v, [2, 4, 6, 8, 10]);+}++#[tokio::test]+async fn feed_accepts_items_and_next_returns_results_then_null() {+ let pipeline = new_chunk_pipeline::<String, ()>()+ .pipe_chunk(chunk_step(+ "uppercaseStep",+ |items: Vec<String>| async move {+ Ok(items.into_iter().map(|s| ok(s.to_uppercase())).collect())+ },+ ))+ .build();+ pipeline.feed(items(["a", "b", "c"].map(String::from)));+ let v: Vec<_> = pipeline+ .next()+ .await+ .unwrap()+ .unwrap()+ .into_iter()+ .map(|r| r.result.into_value().unwrap())+ .collect();+ assert_eq!(v, ["A", "B", "C"]);+ assert!(pipeline.next().await.unwrap().is_none());+}++#[tokio::test]+async fn chunk_steps_must_return_same_number_of_results_as_inputs() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("validChunkStep", |items: Vec<i64>| async move {+ Ok(items.into_iter().map(|n| ok(n * 2)).collect())+ }))+ .build();+ pipeline.feed(items([1, 2, 3]));+ assert_eq!(pipeline.next().await.unwrap().unwrap().len(), 3);+}++#[tokio::test]+async fn mismatched_result_count_throws_an_error() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("badChunkStep", |items: Vec<i64>| async move {+ Ok(vec![ok(items[0])])+ }))+ .build();+ pipeline.feed(items([1, 2, 3]));+ let err = pipeline.next().await.unwrap_err();+ assert!(+ err.to_string()+ .contains("different number of results than input values"),+ "{err}"+ );+}++#[tokio::test]+async fn chunk_steps_receive_only_ok_items_from_previous_steps() {+ let received = shared(Vec::<i64>::new());+ let r = received.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("filterStep", |items: Vec<i64>| async move {+ Ok(items+ .into_iter()+ .map(|n| {+ if n > 3 {+ dlq(format!("Number {n} is too large"), None)+ } else {+ ok(n)+ }+ })+ .collect())+ }))+ .pipe_chunk(chunk_step("processStep", move |items: Vec<i64>| {+ r.lock().unwrap().extend(items.iter().copied());+ async move { Ok(items.into_iter().map(|n| ok(n * 10)).collect()) }+ }))+ .build();+ pipeline.feed(items([1, 2, 3, 4, 5]));+ pipeline.next().await.unwrap();+ assert_eq!(*received.lock().unwrap(), vec![1, 2, 3]);+}++#[tokio::test]+async fn two_batches_emerge_as_a_single_chunk() {+ let lengths = shared(Vec::<usize>::new());+ let l = lengths.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("recordingStep", move |items: Vec<i64>| {+ l.lock().unwrap().push(items.len());+ async move { Ok(items.into_iter().map(ok).collect()) }+ }))+ .build();+ pipeline.feed(items([1, 2]));+ pipeline.feed(items([3, 4, 5]));+ assert_eq!(pipeline.next().await.unwrap().unwrap().len(), 5);+ assert_eq!(*lengths.lock().unwrap(), vec![5]);+}++#[tokio::test]+async fn one_batch_spreads_across_multiple_chunks() {+ let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()+ .concurrently_per_group(+ |e: &(&'static str, i64)| e.0,+ |group| {+ group.sequentially(|b| b.pipe(step("identityStep", |e| async move { Ok(ok(e)) })))+ },+ None,+ )+ .build();+ pipeline.feed(items([("alice", 1), ("bob", 2), ("alice", 3), ("bob", 4)]));+ let chunks = collect_chunks(pipeline.as_ref()).await;+ assert_eq!(chunks.len(), 2);+ assert_eq!(chunks.iter().map(Vec::len).collect::<Vec<_>>(), [2, 2]);+}diff --git a/rust/ingestion-framework/tests/docs_03_concurrent_processing.rs b/rust/ingestion-framework/tests/docs_03_concurrent_processing.rsnew file mode 100644index 000000000000..6fd7c3c11c32--- /dev/null+++ b/rust/ingestion-framework/tests/docs_03_concurrent_processing.rs@@ -0,0 +1,106 @@+mod common;++use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::Arc;++use common::{consume_all_within, items, shared, sleep_ms};+use ingestion_framework::{new_chunk_pipeline, ok, step};++#[tokio::test(start_paused = true)]+async fn concurrently_processes_items_in_parallel() {+ let processed = shared(Vec::<i64>::new());+ let p = processed.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently(+ move |b| {+ b.pipe(step("slowStep", move |n: i64| {+ let p = p.clone();+ async move {+ sleep_ms(100).await;+ p.lock().unwrap().push(n);+ Ok(ok(n * 2))+ }+ }))+ },+ None,+ )+ .build();+ pipeline.feed(items([1, 2, 3]));+ let started = tokio::time::Instant::now();+ let values = consume_all_within(pipeline.as_ref(), 100).await;+ assert_eq!(processed.lock().unwrap().len(), 3);+ assert_eq!(values, [2, 4, 6]);+ // In parallel: the three 100 ms steps take 100 ms in total, not 300 ms.+ assert!(started.elapsed().as_millis() < 200);+}++#[tokio::test(start_paused = true)]+async fn items_are_returned_one_by_one_as_they_complete() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently(+ |b| {+ b.pipe(step("slowStep", |n: i64| async move {+ sleep_ms(10).await;+ Ok(ok(n * 10))+ }))+ },+ None,+ )+ .build();+ pipeline.feed(items([1, 2, 3]));+ let chunks = common::collect_chunks(pipeline.as_ref()).await;+ assert_eq!(chunks, vec![vec![10], vec![20], vec![30]]);+}++#[tokio::test(start_paused = true)]+async fn result_ordering_matches_input_order() {+ let order = shared(Vec::<i64>::new());+ let o = order.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently(+ move |b| {+ b.pipe(step("variableDelayStep", move |n: i64| {+ let o = o.clone();+ async move {+ sleep_ms(((4 - n) * 10) as u64).await;+ o.lock().unwrap().push(n);+ Ok(ok(n * 10))+ }+ }))+ },+ None,+ )+ .build();+ pipeline.feed(items([1, 2, 3]));+ let values = consume_all_within(pipeline.as_ref(), 30).await;+ assert_eq!(*order.lock().unwrap(), vec![3, 2, 1]);+ assert_eq!(values, [10, 20, 30]);+}++#[tokio::test(start_paused = true)]+async fn max_concurrency_caps_how_many_items_run_at_once() {+ let running = Arc::new(AtomicUsize::new(0));+ let peak = Arc::new(AtomicUsize::new(0));+ let (r, p) = (running.clone(), peak.clone());+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently(+ move |b| {+ b.pipe(step("trackingStep", move |n: i64| {+ let (r, p) = (r.clone(), p.clone());+ async move {+ let now = r.fetch_add(1, Ordering::SeqCst) + 1;+ p.fetch_max(now, Ordering::SeqCst);+ sleep_ms(100).await;+ r.fetch_sub(1, Ordering::SeqCst);+ Ok(ok(n * 2))+ }+ }))+ },+ Some(2),+ )+ .build();+ pipeline.feed(items([1, 2, 3, 4, 5]));+ let values = consume_all_within(pipeline.as_ref(), 300).await;+ assert_eq!(peak.load(Ordering::SeqCst), 2);+ assert_eq!(values, [2, 4, 6, 8, 10]);+}diff --git a/rust/ingestion-framework/tests/docs_04_sequential_processing.rs b/rust/ingestion-framework/tests/docs_04_sequential_processing.rsnew file mode 100644index 000000000000..611a4e76572c--- /dev/null+++ b/rust/ingestion-framework/tests/docs_04_sequential_processing.rs@@ -0,0 +1,67 @@+mod common;++use common::{collect_chunks, consume_all, items, shared, sleep_ms};+use ingestion_framework::{new_chunk_pipeline, ok, step};++#[tokio::test(start_paused = true)]+async fn sequentially_processes_items_one_at_a_time() {+ let order = shared(Vec::<i64>::new());+ let o = order.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .sequentially(move |b| {+ b.pipe(step("orderTrackingStep", move |n: i64| {+ let o = o.clone();+ async move {+ sleep_ms(10).await;+ o.lock().unwrap().push(n);+ Ok(ok(n))+ }+ }))+ })+ .build();+ pipeline.feed(items([1, 2, 3]));+ let started = tokio::time::Instant::now();+ let values = consume_all(pipeline.as_ref()).await;+ assert_eq!(*order.lock().unwrap(), vec![1, 2, 3]);+ assert_eq!(values, [1, 2, 3]);+ assert!(started.elapsed().as_millis() >= 30);+}++#[tokio::test(start_paused = true)]+async fn all_items_are_returned_together_after_sequential_processing() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .sequentially(|b| {+ b.pipe(step("slowStep", |n: i64| async move {+ sleep_ms(10).await;+ Ok(ok(n * 10))+ }))+ })+ .build();+ pipeline.feed(items([1, 2, 3]));+ assert_eq!(+ collect_chunks(pipeline.as_ref()).await,+ vec![vec![10, 20, 30]]+ );+}++#[tokio::test(start_paused = true)]+async fn result_ordering_matches_input_order() {+ let order = shared(Vec::<i64>::new());+ let o = order.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .sequentially(move |b| {+ b.pipe(step("trackingStep", move |n: i64| {+ let o = o.clone();+ async move {+ sleep_ms(10).await;+ o.lock().unwrap().push(n);+ Ok(ok(n * 10))+ }+ }))+ })+ .build();+ pipeline.feed(items([3, 1, 2]));+ let values = consume_all(pipeline.as_ref()).await;+ assert_eq!(*order.lock().unwrap(), vec![3, 1, 2]);+ assert_eq!(values, [30, 10, 20]);+}diff --git a/rust/ingestion-framework/tests/docs_05_grouping.rs b/rust/ingestion-framework/tests/docs_05_grouping.rsnew file mode 100644index 000000000000..454970ec4636--- /dev/null+++ b/rust/ingestion-framework/tests/docs_05_grouping.rs@@ -0,0 +1,161 @@+mod common;++use std::collections::HashMap;++use common::{collect_chunks, consume_all_within, items, shared, sleep_ms};+use ingestion_framework::{new_chunk_pipeline, ok, step};++type Event = (&'static str, i64);++fn events(list: &[Event]) -> Vec<ingestion_framework::OkResultWithContext<Event, ()>> {+ items(list.iter().copied())+}++#[tokio::test(start_paused = true)]+async fn concurrently_per_group_partitions_items_by_a_key_function() {+ let groups = shared(HashMap::<&'static str, Vec<i64>>::new());+ let g = groups.clone();+ let pipeline = new_chunk_pipeline::<Event, ()>()+ .concurrently_per_group(+ |e: &Event| e.0,+ move |group| {+ group.sequentially(move |b| {+ b.pipe(step("processEventStep", move |e: Event| {+ let g = g.clone();+ async move {+ g.lock().unwrap().entry(e.0).or_default().push(e.1);+ sleep_ms(5).await;+ Ok(ok(e))+ }+ }))+ })+ },+ None,+ )+ .build();+ pipeline.feed(events(&[+ ("alice", 1),+ ("bob", 2),+ ("alice", 3),+ ("bob", 4),+ ]));+ consume_all_within(pipeline.as_ref(), 20).await;+ let groups = groups.lock().unwrap();+ assert_eq!(groups["alice"], vec![1, 3]);+ assert_eq!(groups["bob"], vec![2, 4]);+}++#[tokio::test(start_paused = true)]+async fn items_in_the_same_group_are_processed_sequentially() {+ let order = shared(Vec::<i64>::new());+ let o = order.clone();+ let pipeline = new_chunk_pipeline::<Event, ()>()+ .concurrently_per_group(+ |e: &Event| e.0,+ move |group| {+ group.sequentially(move |b| {+ b.pipe(step("processEventStep", move |e: Event| {+ let o = o.clone();+ async move {+ o.lock().unwrap().push(e.1);+ sleep_ms(10).await;+ Ok(ok(e))+ }+ }))+ })+ },+ None,+ )+ .build();+ pipeline.feed(events(&[("alice", 1), ("alice", 2), ("alice", 3)]));+ consume_all_within(pipeline.as_ref(), 30).await;+ assert_eq!(*order.lock().unwrap(), vec![1, 2, 3]);+}++fn delay_for(user: &str) -> u64 {+ match user {+ "alice" => 30,+ "bob" => 10,+ "charlie" => 20,+ _ => 0,+ }+}++#[tokio::test(start_paused = true)]+async fn items_in_different_groups_can_process_concurrently() {+ let pipeline = new_chunk_pipeline::<Event, ()>()+ .concurrently_per_group(+ |e: &Event| e.0,+ |group| {+ group.sequentially(|b| {+ b.pipe(step("variableDelayStep", |e: Event| async move {+ sleep_ms(delay_for(e.0)).await;+ Ok(ok(e))+ }))+ })+ },+ None,+ )+ .build();+ pipeline.feed(events(&[+ ("alice", 1),+ ("bob", 2),+ ("charlie", 3),+ ("alice", 4),+ ("bob", 5),+ ("charlie", 6),+ ]));+ let chunks: Vec<Vec<String>> = collect_chunks(pipeline.as_ref())+ .await+ .into_iter()+ .map(|c| c.into_iter().map(|(u, id)| format!("{u}:{id}")).collect())+ .collect();+ assert_eq!(+ chunks,+ vec![+ vec!["bob:2", "bob:5"],+ vec!["charlie:3", "charlie:6"],+ vec!["alice:1", "alice:4"],+ ]+ );+}++#[tokio::test(start_paused = true)]+async fn complex_group_keys_with_token_and_distinct_id() {+ type IngestionEvent = (&'static str, &'static str, &'static str);+ let groups = shared(HashMap::<String, Vec<&'static str>>::new());+ let g = groups.clone();+ let pipeline = new_chunk_pipeline::<IngestionEvent, ()>()+ .concurrently_per_group(+ |e: &IngestionEvent| format!("{}:{}", e.0, e.1),+ move |group| {+ group.sequentially(move |b| {+ b.pipe(step("processEventStep", move |e: IngestionEvent| {+ let g = g.clone();+ async move {+ g.lock()+ .unwrap()+ .entry(format!("{}:{}", e.0, e.1))+ .or_default()+ .push(e.2);+ sleep_ms(5).await;+ Ok(ok(e))+ }+ }))+ })+ },+ None,+ )+ .build();+ pipeline.feed(items([+ ("token-A", "user-1", "login"),+ ("token-B", "user-1", "signup"),+ ("token-A", "user-2", "pageview"),+ ("token-A", "user-1", "purchase"),+ ]));+ consume_all_within(pipeline.as_ref(), 10).await;+ let groups = groups.lock().unwrap();+ assert_eq!(groups["token-A:user-1"], vec!["login", "purchase"]);+ assert_eq!(groups["token-B:user-1"], vec!["signup"]);+ assert_eq!(groups["token-A:user-2"], vec!["pageview"]);+}diff --git a/rust/ingestion-framework/tests/docs_06_gathering.rs b/rust/ingestion-framework/tests/docs_06_gathering.rsnew file mode 100644index 000000000000..f960767ca759--- /dev/null+++ b/rust/ingestion-framework/tests/docs_06_gathering.rs@@ -0,0 +1,81 @@+mod common;++use common::{collect_chunks, items, sleep_ms};+use ingestion_framework::{new_chunk_pipeline, ok, step, ProcessingStep};++fn variable_delay_numbers() -> ProcessingStep<i64, i64> {+ step("variableDelayStep", |n: i64| async move {+ sleep_ms(match n {+ 1 => 30,+ 2 => 10,+ _ => 20,+ })+ .await;+ Ok(ok(n * 10))+ })+}++#[tokio::test(start_paused = true)]+async fn gather_after_concurrently_collects_items_into_one_chunk() {+ let streaming = new_chunk_pipeline::<i64, ()>()+ .concurrently(|b| b.pipe(variable_delay_numbers()), None)+ .build();+ streaming.feed(items([1, 2, 3]));+ assert_eq!(+ collect_chunks(streaming.as_ref()).await,+ vec![vec![10], vec![20], vec![30]]+ );++ let gathering = new_chunk_pipeline::<i64, ()>()+ .concurrently(|b| b.pipe(variable_delay_numbers()), None)+ .gather()+ .build();+ gathering.feed(items([1, 2, 3]));+ assert_eq!(+ collect_chunks(gathering.as_ref()).await,+ vec![vec![10, 20, 30]]+ );+}++type Event = (&'static str, i64);++fn variable_delay_events() -> ProcessingStep<Event, Event> {+ step("variableDelayStep", |e: Event| async move {+ sleep_ms(if e.0 == "alice" { 30 } else { 10 }).await;+ Ok(ok(e))+ })+}++#[tokio::test(start_paused = true)]+async fn gather_after_concurrently_per_group_collects_groups_into_one_chunk() {+ let events = [("alice", 1), ("bob", 2), ("alice", 3), ("bob", 4)];+ let streaming = new_chunk_pipeline::<Event, ()>()+ .concurrently_per_group(+ |e: &Event| e.0,+ |g| g.sequentially(|b| b.pipe(variable_delay_events())),+ None,+ )+ .build();+ streaming.feed(items(events));+ assert_eq!(+ collect_chunks(streaming.as_ref()).await,+ vec![+ vec![("bob", 2), ("bob", 4)],+ vec![("alice", 1), ("alice", 3)]+ ]+ );++ let gathering = new_chunk_pipeline::<Event, ()>()+ .concurrently_per_group(+ |e: &Event| e.0,+ |g| g.sequentially(|b| b.pipe(variable_delay_events())),+ None,+ )+ .gather()+ .build();+ gathering.feed(items(events));+ assert_eq!(+ collect_chunks(gathering.as_ref()).await,+ vec![vec![("bob", 2), ("bob", 4), ("alice", 1), ("alice", 3)]]+ );+}diff --git a/rust/ingestion-framework/tests/docs_07_result_handling.rs b/rust/ingestion-framework/tests/docs_07_result_handling.rsnew file mode 100644index 000000000000..7185a522dd09--- /dev/null+++ b/rust/ingestion-framework/tests/docs_07_result_handling.rs@@ -0,0 +1,242 @@+mod common;++use std::sync::Arc;+use std::time::Duration;++use common::{test_message, MockOutputs, MsgCtx};+use ingestion_framework::{+ chunk_step, create_ok_context, dlq, drop, new_chunk_pipeline, ok, redirect, redirect_with,+ PipelineConfig, PipelineResultType, PromiseScheduler, SideEffectHandlingConfig, DLQ_OUTPUT,+};++fn config(outputs: &MockOutputs, scheduler: &PromiseScheduler) -> PipelineConfig {+ PipelineConfig {+ outputs: Arc::new(outputs.clone()),+ scheduler: Arc::new(scheduler.clone()),+ reject_on_dlq_failure: false,+ }+}++const AWAIT: SideEffectHandlingConfig = SideEffectHandlingConfig {+ await_side_effects: true,+};++#[tokio::test]+async fn dlq_sends_failed_items_to_dead_letter_topic_with_error_details() {+ let outputs = MockOutputs::default();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()+ .pipe_chunk(chunk_step(+ "validationStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|d| {+ if d == "invalid" {+ dlq(+ "Validation failed",+ Some(anyhow::anyhow!("Invalid data format")),+ )+ } else {+ ok(d)+ }+ })+ .collect())+ },+ ))+ .message_aware(|b| b)+ .handle_results(config(&outputs, &scheduler))+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)+ .build();+ let message = test_message(None);+ pipeline.feed(vec![create_ok_context(+ "invalid",+ MsgCtx {+ message: message.clone(),+ },+ )]);+ let results = pipeline.next().await.unwrap().unwrap();+ assert_eq!(results.len(), 1);+ assert_eq!(results[0].result.result_type(), PipelineResultType::Dlq);+ let calls = outputs.calls();+ assert_eq!(calls.len(), 1);+ let (output, produced) = &calls[0];+ assert_eq!(output, DLQ_OUTPUT);+ assert_eq!(produced.value, message.value);+ assert_eq!(produced.key, message.key);+ assert_eq!(+ produced.headers.get("dlq_reason"),+ Some("Invalid data format")+ );+ assert_eq!(produced.headers.get("dlq_step"), Some("validationStep"));+ assert!(produced.headers.get("dlq_timestamp").is_some());+}++#[tokio::test]+async fn drop_discards_items_without_sending_to_kafka() {+ let outputs = MockOutputs::default();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()+ .pipe_chunk(chunk_step(+ "filterStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|t| {+ if t.starts_with("internal-") {+ drop("Internal event filtered")+ } else {+ ok(t)+ }+ })+ .collect())+ },+ ))+ .message_aware(|b| b)+ .handle_results(config(&outputs, &scheduler))+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)+ .build();+ pipeline.feed(vec![create_ok_context(+ "internal-heartbeat",+ MsgCtx {+ message: test_message(None),+ },+ )]);+ let results = pipeline.next().await.unwrap().unwrap();+ assert_eq!(results.len(), 1);+ assert_eq!(results[0].result.result_type(), PipelineResultType::Drop);+ assert!(outputs.calls().is_empty());+}++#[tokio::test]+async fn redirect_sends_items_to_a_named_output_with_optional_key_preservation() {+ let outputs = MockOutputs::default();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()+ .pipe_chunk(chunk_step(+ "routingStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|p| match p {+ "high" => {+ redirect_with("High priority routing", "high_priority", true, true)+ }+ "broadcast" => redirect_with("Broadcast routing", "broadcast", false, true),+ _ => ok(p),+ })+ .collect())+ },+ ))+ .message_aware(|b| b)+ .handle_results(config(&outputs, &scheduler))+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)+ .build();+ let message1 = test_message(Some(b"my-partition-key"));+ let message2 = test_message(Some(b"another-key"));+ pipeline.feed(vec![+ create_ok_context(+ "high",+ MsgCtx {+ message: message1.clone(),+ },+ ),+ create_ok_context(+ "broadcast",+ MsgCtx {+ message: message2.clone(),+ },+ ),+ ]);+ let results = pipeline.next().await.unwrap().unwrap();+ assert_eq!(results.len(), 2);+ assert!(results+ .iter()+ .all(|r| r.result.result_type() == PipelineResultType::Redirect));++ // Node produces in item order, so the call order is the feed order.+ let calls = outputs.calls();+ assert_eq!(calls.len(), 2);+ let (high_out, high) = &calls[0];+ let (broadcast_out, broadcast) = &calls[1];+ assert_eq!(high_out, "high_priority");+ assert_eq!(high.value, message1.value);+ assert_eq!(high.key.as_deref(), Some(&b"my-partition-key"[..]));+ assert_eq!(high.headers.get("redirect-step"), Some("routingStep"));+ assert!(high.headers.get("redirect-timestamp").is_some());+ assert_eq!(broadcast_out, "broadcast");+ assert_eq!(broadcast.value, message2.value);+ assert_eq!(broadcast.key, None);+ assert_eq!(broadcast.headers.get("redirect-step"), Some("routingStep"));+ assert!(broadcast.headers.get("redirect-timestamp").is_some());+}++#[tokio::test]+async fn redirect_with_await_ack_true_blocks_on_the_produce_ack() {+ let outputs = MockOutputs::default();+ outputs.close_gate();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()+ .pipe_chunk(chunk_step(+ "routingStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|_| redirect::<&str>("Overflow", "overflow"))+ .collect())+ },+ ))+ .message_aware(|b| b)+ .handle_results(config(&outputs, &scheduler))+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)+ .build();+ pipeline.feed(vec![create_ok_context(+ "x",+ MsgCtx {+ message: test_message(None),+ },+ )]);+ let mut next = Box::pin(pipeline.next());+ assert!(tokio::time::timeout(Duration::from_millis(50), &mut next)+ .await+ .is_err());+ outputs.open_gate();+ let results = next.await.unwrap().unwrap();+ assert_eq!(+ results[0].result.result_type(),+ PipelineResultType::Redirect+ );+}++#[tokio::test]+async fn redirect_with_await_ack_false_does_not_block_on_the_produce_ack() {+ let outputs = MockOutputs::default();+ outputs.close_gate();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<&'static str, MsgCtx>()+ .pipe_chunk(chunk_step(+ "routingStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|_| redirect_with::<&str>("Overflow", "overflow", true, false))+ .collect())+ },+ ))+ .message_aware(|b| b)+ .handle_results(config(&outputs, &scheduler))+ .handle_side_effects(Arc::new(scheduler.clone()), AWAIT)+ .build();+ pipeline.feed(vec![create_ok_context(+ "x",+ MsgCtx {+ message: test_message(None),+ },+ )]);+ let results = pipeline.next().await.unwrap().unwrap();+ assert_eq!(+ results[0].result.result_type(),+ PipelineResultType::Redirect+ );+ assert_eq!(outputs.calls().len(), 1);+}diff --git a/rust/ingestion-framework/tests/docs_08_side_effects.rs b/rust/ingestion-framework/tests/docs_08_side_effects.rsnew file mode 100644index 000000000000..04f728787404--- /dev/null+++ b/rust/ingestion-framework/tests/docs_08_side_effects.rs@@ -0,0 +1,181 @@+mod common;++use std::sync::Arc;++use common::{items, shared, sleep_ms};+use ingestion_framework::{+ create_ok_context, new_chunk_pipeline, new_pipeline, ok, step, ChunkProcessingStep,+ PromiseScheduler, SideEffect, SideEffectHandlingConfig,+};++async fn wait_all(side_effects: Vec<SideEffect>) {+ for s in side_effects {+ s.wait().await.unwrap();+ }+}++#[tokio::test]+async fn steps_can_add_side_effects_to_context() {+ let log = shared(Vec::<String>::new());+ let l = log.clone();+ let pipeline = new_pipeline::<String, ()>()+ .pipe(step("processWithLoggingStep", move |input: String| {+ let l = l.clone();+ async move {+ let result = input.to_uppercase();+ let message = format!("Processed: {input} -> {result}");+ let effect = SideEffect::spawn(async move {+ l.lock().unwrap().push(message);+ Ok(())+ });+ Ok(ok((result, true)).with_side_effect(effect))+ }+ }))+ .build();+ let mut r = pipeline+ .process(create_ok_context("hello".into(), ()))+ .await+ .unwrap();+ assert!(r.is_ok());+ assert_eq!(r.context.side_effects.len(), 1);+ // The side effect started when it was created, as a Node promise does.+ assert!(log+ .lock()+ .unwrap()+ .contains(&"Processed: hello -> HELLO".to_string()));+ wait_all(std::mem::take(&mut r.context.side_effects)).await;+ assert!(log+ .lock()+ .unwrap()+ .contains(&"Processed: hello -> HELLO".to_string()));+}++#[tokio::test]+async fn side_effects_accumulate_through_the_pipeline() {+ let audit = shared(Vec::<&'static str>::new());+ let audited = |name: &'static str, f: fn(i64) -> i64| {+ let a = audit.clone();+ step(name, move |v: i64| {+ let a = a.clone();+ async move {+ let effect = SideEffect::spawn(async move {+ a.lock().unwrap().push(match name {+ "step1" => "step1 completed",+ "step2" => "step2 completed",+ _ => "step3 completed",+ });+ Ok(())+ });+ Ok(ok(f(v)).with_side_effect(effect))+ }+ })+ };+ let pipeline = new_pipeline::<i64, ()>()+ .pipe(audited("step1", |v| v * 2))+ .pipe(audited("step2", |v| v + 10))+ .pipe(audited("step3", |v| v * v))+ .build();+ let mut r = pipeline.process(create_ok_context(5, ())).await.unwrap();+ assert_eq!(r.context.side_effects.len(), 3);+ wait_all(std::mem::take(&mut r.context.side_effects)).await;+ let audit = audit.lock().unwrap();+ assert_eq!(audit.len(), 3);+ for expected in ["step1 completed", "step2 completed", "step3 completed"] {+ assert!(audit.contains(&expected));+ }+}++#[tokio::test]+async fn a_step_can_add_multiple_side_effects() {+ let sinks = shared((0, 0, 0));+ let s = sinks.clone();+ let pipeline = new_pipeline::<(&'static str, &'static str), ()>()+ .pipe(step(+ "processEventStep",+ move |e: (&'static str, &'static str)| {+ let (s1, s2, s3) = (s.clone(), s.clone(), s.clone());+ async move {+ Ok(ok((e, true))+ .with_side_effect(SideEffect::spawn(async move {+ s1.lock().unwrap().0 += 1;+ Ok(())+ }))+ .with_side_effect(SideEffect::spawn(async move {+ s2.lock().unwrap().1 += 1;+ Ok(())+ }))+ .with_side_effect(SideEffect::spawn(async move {+ s3.lock().unwrap().2 += 1;+ Ok(())+ })))+ }+ },+ ))+ .build();+ let mut r = pipeline+ .process(create_ok_context(("purchase", "user-1"), ()))+ .await+ .unwrap();+ assert_eq!(r.context.side_effects.len(), 3);+ wait_all(std::mem::take(&mut r.context.side_effects)).await;+ assert_eq!(*sinks.lock().unwrap(), (1, 1, 1));+}++fn delayed_effect_step(+ completed: Arc<std::sync::Mutex<Vec<&'static str>>>,+ delay: u64,+) -> ChunkProcessingStep<&'static str, &'static str> {+ ChunkProcessingStep::per_item(step("processStep", move |id: &'static str| {+ let completed = completed.clone();+ async move {+ let effect = SideEffect::spawn(async move {+ sleep_ms(delay).await;+ completed.lock().unwrap().push(id);+ Ok(())+ });+ Ok(ok(id).with_side_effect(effect))+ }+ }))+}++#[tokio::test(start_paused = true)]+async fn handle_side_effects_with_await_true_blocks_until_completion() {+ let completed = shared(Vec::new());+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<&'static str, ()>()+ .pipe_chunk(delayed_effect_step(completed.clone(), 10))+ .handle_side_effects(+ Arc::new(scheduler),+ SideEffectHandlingConfig {+ await_side_effects: true,+ },+ )+ .build();+ pipeline.feed(items(["a", "b", "c"]));+ pipeline.next().await.unwrap();+ let mut done = completed.lock().unwrap().clone();+ done.sort();+ assert_eq!(done, ["a", "b", "c"]);+}++#[tokio::test(start_paused = true)]+async fn handle_side_effects_with_await_false_schedules_for_background_execution() {+ let completed = shared(Vec::new());+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<&'static str, ()>()+ .pipe_chunk(delayed_effect_step(completed.clone(), 50))+ .handle_side_effects(+ Arc::new(scheduler.clone()),+ SideEffectHandlingConfig {+ await_side_effects: false,+ },+ )+ .build();+ pipeline.feed(items(["x", "y"]));+ pipeline.next().await.unwrap();+ assert!(completed.lock().unwrap().is_empty());+ scheduler.wait_for_all().await.unwrap();+ let mut done = completed.lock().unwrap().clone();+ done.sort();+ assert_eq!(done, ["x", "y"]);+}diff --git a/rust/ingestion-framework/tests/docs_09_ingestion_warnings.rs b/rust/ingestion-framework/tests/docs_09_ingestion_warnings.rsnew file mode 100644index 000000000000..5ba6a633d016--- /dev/null+++ b/rust/ingestion-framework/tests/docs_09_ingestion_warnings.rs@@ -0,0 +1,311 @@+mod common;++use std::sync::Arc;++use common::{shared, MockOutputs, TeamCtx};+use ingestion_framework::warning_handling::OutputsWarningEmitter;+use ingestion_framework::{+ chunk_step, create_ok_context, new_chunk_pipeline, ok, PipelineWarning, PromiseScheduler,+ SideEffect, SideEffectHandlingConfig, INGESTION_WARNINGS_OUTPUT,+};+use serde_json::json;++fn team() -> TeamCtx {+ TeamCtx { team_id: 2 }+}++#[tokio::test]+async fn steps_can_add_warnings_to_results() {+ let pipeline = new_chunk_pipeline::<(&'static str, Option<()>), TeamCtx>()+ .pipe_chunk(chunk_step(+ "validationStep",+ |items: Vec<(&'static str, Option<()>)>| async move {+ Ok(items+ .into_iter()+ .map(|item| {+ let mut r = ok(item);+ if item.1.is_none() {+ r = r.with_warning(PipelineWarning::new(+ "schema_validation_failed",+ json!({"eventName": item.0}),+ ));+ }+ r+ })+ .collect())+ },+ ))+ .build();+ pipeline.feed(vec![create_ok_context(("pageview", None), team())]);+ let results = pipeline.next().await.unwrap().unwrap();+ assert_eq!(results.len(), 1);+ assert!(results[0].is_ok());+ assert_eq!(+ results[0].context.warnings,+ vec![PipelineWarning::new(+ "schema_validation_failed",+ json!({"eventName": "pageview"})+ )]+ );+}++#[tokio::test]+async fn warnings_accumulate_through_the_pipeline() {+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()+ .pipe_chunk(chunk_step(+ "timestampCheckStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|n| {+ ok(n).with_warning(PipelineWarning::new(+ "ignored_invalid_timestamp",+ json!({"eventName": n}),+ ))+ })+ .collect())+ },+ ))+ .pipe_chunk(chunk_step(+ "propertiesCheckStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|n| {+ ok(n).with_warning(PipelineWarning::new(+ "schema_validation_failed",+ json!({"eventName": n}),+ ))+ })+ .collect())+ },+ ))+ .build();+ pipeline.feed(vec![create_ok_context("click", team())]);+ let results = pipeline.next().await.unwrap().unwrap();+ let types: Vec<_> = results[0]+ .context+ .warnings+ .iter()+ .map(|w| w.warning_type.as_str())+ .collect();+ assert_eq!(+ types,+ ["ignored_invalid_timestamp", "schema_validation_failed"]+ );+}++#[tokio::test]+async fn a_step_can_add_multiple_warnings() {+ struct Event {+ timestamp: &'static str,+ group_key: String,+ process_person: serde_json::Value,+ }+ let pipeline = new_chunk_pipeline::<Event, TeamCtx>()+ .pipe_chunk(chunk_step(+ "comprehensiveValidationStep",+ |items: Vec<Event>| async move {+ Ok(items+ .into_iter()+ .map(|item| {+ let mut warnings = Vec::new();+ if item.group_key.len() > 400 {+ warnings.push(PipelineWarning::new(+ "group_key_too_long",+ json!({"groupKey": &item.group_key[..20], "max": 400}),+ ));+ }+ if chrono::DateTime::parse_from_rfc3339(item.timestamp).is_err() {+ warnings.push(PipelineWarning::new(+ "ignored_invalid_timestamp",+ json!({"value": item.timestamp}),+ ));+ }+ if !item.process_person.is_null() && !item.process_person.is_boolean() {+ warnings.push(PipelineWarning::new(+ "invalid_process_person_profile",+ json!({"received": "string"}),+ ));+ }+ ok(item).with_warnings(warnings)+ })+ .collect())+ },+ ))+ .build();+ pipeline.feed(vec![create_ok_context(+ Event {+ timestamp: "not-a-date",+ group_key: "g".repeat(500),+ process_person: json!("yes"),+ },+ team(),+ )]);+ let results = pipeline.next().await.unwrap().unwrap();+ let types: Vec<_> = results[0]+ .context+ .warnings+ .iter()+ .map(|w| w.warning_type.clone())+ .collect();+ assert_eq!(types.len(), 3);+ for t in [+ "group_key_too_long",+ "ignored_invalid_timestamp",+ "invalid_process_person_profile",+ ] {+ assert!(types.contains(&t.to_string()));+ }+}++#[tokio::test]+async fn handle_ingestion_warnings_converts_warnings_to_side_effects() {+ let outputs = MockOutputs::default();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()+ .pipe_chunk(chunk_step(+ "warningStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|n| {+ ok(n).with_warning(PipelineWarning::new(+ "client_ingestion_warning",+ json!({"eventName": n}),+ ))+ })+ .collect())+ },+ ))+ .team_aware(|b| b)+ .handle_ingestion_warnings(Arc::new(OutputsWarningEmitter {+ outputs: Arc::new(outputs.clone()),+ }))+ .handle_side_effects(+ Arc::new(scheduler),+ SideEffectHandlingConfig {+ await_side_effects: true,+ },+ )+ .build();+ pipeline.feed(vec![create_ok_context("pageview", TeamCtx { team_id: 42 })]);+ let results = pipeline.next().await.unwrap().unwrap();+ assert!(results[0].context.warnings.is_empty());+ let calls = outputs.calls();+ assert_eq!(calls.len(), 1);+ assert_eq!(calls[0].0, INGESTION_WARNINGS_OUTPUT);+ let body: serde_json::Value =+ serde_json::from_slice(calls[0].1.value.as_ref().unwrap()).unwrap();+ assert_eq!(body["team_id"], 42);+ assert_eq!(body["type"], "client_ingestion_warning");+}++#[tokio::test]+async fn handle_ingestion_warnings_preserves_existing_side_effects() {+ let outputs = MockOutputs::default();+ let log = shared(Vec::<String>::new());+ let l = log.clone();+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()+ .pipe_chunk(chunk_step(+ "stepWithBothSideEffectsAndWarnings",+ move |items: Vec<&'static str>| {+ let l = l.clone();+ async move {+ Ok(items+ .into_iter()+ .map(|n| {+ let l = l.clone();+ ok(n)+ .with_side_effect(SideEffect::spawn(async move {+ l.lock().unwrap().push(format!("processed: {n}"));+ Ok(())+ }))+ .with_warning(PipelineWarning::new(+ "client_ingestion_warning",+ json!({}),+ ))+ })+ .collect())+ }+ },+ ))+ .team_aware(|b| b)+ .handle_ingestion_warnings(Arc::new(OutputsWarningEmitter {+ outputs: Arc::new(outputs.clone()),+ }))+ .handle_side_effects(+ Arc::new(PromiseScheduler::new()),+ SideEffectHandlingConfig {+ await_side_effects: true,+ },+ )+ .build();+ pipeline.feed(vec![create_ok_context("click", team())]);+ pipeline.next().await.unwrap();+ assert!(log+ .lock()+ .unwrap()+ .contains(&"processed: click".to_string()));+ assert_eq!(outputs.calls().len(), 1);+}++#[tokio::test]+async fn warnings_can_include_a_key_for_debouncing() {+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()+ .pipe_chunk(chunk_step(+ "userWarningStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|d| {+ let mut w = PipelineWarning::new(+ "cannot_merge_already_identified",+ json!({"distinctId": d}),+ );+ w.key = Some(d.to_string());+ ok(d).with_warning(w)+ })+ .collect())+ },+ ))+ .build();+ pipeline.feed(vec![create_ok_context("user-123", team())]);+ let results = pipeline.next().await.unwrap().unwrap();+ let mut expected = PipelineWarning::new(+ "cannot_merge_already_identified",+ json!({"distinctId": "user-123"}),+ );+ expected.key = Some("user-123".into());+ assert_eq!(results[0].context.warnings, vec![expected]);+}++#[tokio::test]+async fn always_send_bypasses_debouncing_for_critical_warnings() {+ let pipeline = new_chunk_pipeline::<&'static str, TeamCtx>()+ .pipe_chunk(chunk_step(+ "criticalWarningStep",+ |items: Vec<&'static str>| async move {+ Ok(items+ .into_iter()+ .map(|n| {+ let mut w =+ PipelineWarning::new("message_size_too_large", json!({"eventName": n}));+ w.key = Some("quota".into());+ w.always_send = true;+ ok(n).with_warning(w)+ })+ .collect())+ },+ ))+ .build();+ pipeline.feed(vec![create_ok_context("important_event", team())]);+ let results = pipeline.next().await.unwrap().unwrap();+ let mut expected = PipelineWarning::new(+ "message_size_too_large",+ json!({"eventName": "important_event"}),+ );+ expected.key = Some("quota".into());+ expected.always_send = true;+ assert_eq!(results[0].context.warnings, vec![expected]);+}diff --git a/rust/ingestion-framework/tests/docs_10_branching.rs b/rust/ingestion-framework/tests/docs_10_branching.rsnew file mode 100644index 000000000000..42b77209f7de--- /dev/null+++ b/rust/ingestion-framework/tests/docs_10_branching.rs@@ -0,0 +1,112 @@+//! Branch names are strings, so only the run-time DLQ path exists.++mod common;++use common::shared;+use ingestion_framework::{create_ok_context, new_pipeline, ok, step, PipelineResultType};++type Event = (&'static str, &'static str);++#[tokio::test]+async fn branching_routes_items_to_different_sub_pipelines() {+ let calls = shared(Vec::<&'static str>::new());+ let (c1, c2, c3) = (calls.clone(), calls.clone(), calls.clone());+ let pipeline = new_pipeline::<Event, ()>()+ .branching(+ |e: &Event| e.0.to_string(),+ move |b| {+ b.branch("pageview", move |s| {+ s.pipe(step("pageviewStep", move |e: Event| {+ c1.lock().unwrap().push("pageview");+ async move { Ok(ok(format!("PAGE: {}", e.1))) }+ }))+ })+ .branch("click", move |s| {+ s.pipe(step("clickStep", move |e: Event| {+ c2.lock().unwrap().push("click");+ async move { Ok(ok(format!("CLICK: {}", e.1))) }+ }))+ })+ .branch("custom", move |s| {+ s.pipe(step("customStep", move |e: Event| {+ c3.lock().unwrap().push("custom");+ async move { Ok(ok(e.1.to_string())) }+ }))+ })+ },+ )+ .build();+ let page = pipeline+ .process(create_ok_context(("pageview", "home"), ()))+ .await+ .unwrap();+ let click = pipeline+ .process(create_ok_context(("click", "button"), ()))+ .await+ .unwrap();+ assert_eq!(page.value().unwrap(), "PAGE: home");+ assert_eq!(click.value().unwrap(), "CLICK: button");+ assert_eq!(*calls.lock().unwrap(), vec!["pageview", "click"]);+}++#[tokio::test]+async fn unknown_branch_routes_to_dlq() {+ let pipeline = new_pipeline::<Event, ()>()+ .branching(+ |e: &Event| e.0.to_string(),+ |b| {+ b.branch("pageview", |s| {+ s.pipe(step("pageviewStep", |e: Event| async move { Ok(ok(e)) }))+ })+ },+ )+ .build();+ let r = pipeline+ .process(create_ok_context(("unknown", "test"), ()))+ .await+ .unwrap();+ assert_eq!(r.result.result_type(), PipelineResultType::Dlq);+ assert_eq!(r.result.reason(), Some("Unknown branch: unknown"));+}++#[tokio::test]+async fn branches_must_converge_to_the_same_output_type() {+ #[derive(Debug, PartialEq)]+ enum Output {+ Number(i64),+ Str(String),+ }+ let pipeline = new_pipeline::<(&'static str, &'static str), ()>()+ .branching(+ |i: &(&'static str, &'static str)| i.0.to_string(),+ |b| {+ b.branch("toNumber", |s| {+ s.pipe(step(+ "toNumberStep",+ |i: (&'static str, &'static str)| async move {+ Ok(ok(Output::Number(i.1.parse().unwrap())))+ },+ ))+ })+ .branch("toString", |s| {+ s.pipe(step(+ "toStringStep",+ |i: (&'static str, &'static str)| async move {+ Ok(ok(Output::Str(i.1.to_uppercase())))+ },+ ))+ })+ },+ )+ .build();+ let n = pipeline+ .process(create_ok_context(("toNumber", "42"), ()))+ .await+ .unwrap();+ let s = pipeline+ .process(create_ok_context(("toString", "hello"), ()))+ .await+ .unwrap();+ assert_eq!(n.value(), Some(&Output::Number(42)));+ assert_eq!(s.value(), Some(&Output::Str("HELLO".into())));+}diff --git a/rust/ingestion-framework/tests/docs_11_retries.rs b/rust/ingestion-framework/tests/docs_11_retries.rsnew file mode 100644index 000000000000..9cefb7ff7063--- /dev/null+++ b/rust/ingestion-framework/tests/docs_11_retries.rs@@ -0,0 +1,230 @@+mod common;++use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::Arc;++use common::{items, sleep_ms, within_ms};+use ingestion_framework::{+ chunk_step, create_ok_context, new_chunk_pipeline, new_pipeline, ok, step, PipelineError,+ PipelineResultType, RetryOptions,+};++fn opts(tries: u32, sleep_ms: u64) -> RetryOptions {+ RetryOptions {+ tries: Some(tries),+ sleep_ms: Some(sleep_ms),+ ..Default::default()+ }+}++#[tokio::test(start_paused = true)]+async fn retriable_errors_are_retried_until_the_step_succeeds() {+ let attempts = Arc::new(AtomicU32::new(0));+ let a = attempts.clone();+ let pipeline = new_pipeline::<String, ()>()+ .pipe_with_retry(+ step("flakyStep", move |v: String| {+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;+ async move {+ if n < 3 {+ return Err(PipelineError::retriable("Temporary failure"));+ }+ Ok(ok(v.to_uppercase()))+ }+ }),+ opts(5, 100),+ )+ .build();+ let r = within_ms(300, pipeline.process(create_ok_context("hello".into(), ())))+ .await+ .unwrap();+ assert_eq!(attempts.load(Ordering::SeqCst), 3);+ assert_eq!(r.value().unwrap(), "HELLO");+}++#[tokio::test(start_paused = true)]+async fn non_retriable_errors_go_directly_to_dlq() {+ let attempts = Arc::new(AtomicU32::new(0));+ let a = attempts.clone();+ let pipeline = new_pipeline::<String, ()>()+ .pipe_with_retry(+ step("validationStep", move |_v: String| {+ a.fetch_add(1, Ordering::SeqCst);+ async move {+ Err::<ingestion_framework::PipelineResult<String>, _>(+ PipelineError::non_retriable("Validation failed - invalid format"),+ )+ }+ }),+ opts(5, 100),+ )+ .build();+ let r = within_ms(+ 0,+ pipeline.process(create_ok_context("bad-data".into(), ())),+ )+ .await+ .unwrap();+ assert_eq!(attempts.load(Ordering::SeqCst), 1);+ assert_eq!(r.result.result_type(), PipelineResultType::Dlq);+ assert!(r.result.reason().unwrap().contains("non-retriable"));+}++#[tokio::test(start_paused = true)]+async fn after_max_retries_the_error_is_rethrown() {+ let attempts = Arc::new(AtomicU32::new(0));+ let a = attempts.clone();+ let pipeline = new_pipeline::<String, ()>()+ .pipe_with_retry(+ step("alwaysFailsStep", move |_v: String| {+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;+ async move {+ Err::<ingestion_framework::PipelineResult<String>, _>(PipelineError::retriable(+ format!("Attempt {n} failed"),+ ))+ }+ }),+ opts(3, 100),+ )+ .build();+ let err = within_ms(300, pipeline.process(create_ok_context("data".into(), ())))+ .await+ .unwrap_err();+ assert_eq!(attempts.load(Ordering::SeqCst), 3);+ assert_eq!(err.to_string(), "Attempt 3 failed");+}++#[tokio::test(start_paused = true)]+async fn errors_without_is_retriable_are_retried_then_rethrown() {+ let attempts = Arc::new(AtomicU32::new(0));+ let a = attempts.clone();+ let pipeline = new_pipeline::<String, ()>()+ .pipe_with_retry(+ step("unexpectedErrorStep", move |_v: String| {+ a.fetch_add(1, Ordering::SeqCst);+ async move {+ Err::<ingestion_framework::PipelineResult<String>, _>(anyhow::anyhow!(+ "Unexpected internal error"+ ))+ }+ }),+ opts(3, 100),+ )+ .build();+ let err = within_ms(300, pipeline.process(create_ok_context("data".into(), ())))+ .await+ .unwrap_err();+ assert_eq!(attempts.load(Ordering::SeqCst), 3);+ assert_eq!(err.to_string(), "Unexpected internal error");+}++#[tokio::test(start_paused = true)]+async fn tries_and_sleep_ms_control_attempts_and_backoff() {+ let attempts = Arc::new(AtomicU32::new(0));+ let a = attempts.clone();+ let pipeline = new_pipeline::<String, ()>()+ .pipe_with_retry(+ step("slowlyRecoveringStep", move |v: String| {+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;+ async move {+ if n < 4 {+ return Err(PipelineError::retriable("Still recovering"));+ }+ Ok(ok(v))+ }+ }),+ opts(5, 50),+ )+ .build();+ let handle =+ tokio::spawn(async move { pipeline.process(create_ok_context("data".into(), ())).await });+ async fn advance(ms: u64) {+ sleep_ms(ms).await;+ for _ in 0..10 {+ tokio::task::yield_now().await;+ }+ }+ advance(0).await;+ assert_eq!(attempts.load(Ordering::SeqCst), 1);+ advance(50).await;+ assert_eq!(attempts.load(Ordering::SeqCst), 2);+ advance(100).await;+ assert_eq!(attempts.load(Ordering::SeqCst), 3);+ advance(200).await;+ assert_eq!(attempts.load(Ordering::SeqCst), 4);+ assert!(handle.await.unwrap().unwrap().is_ok());+}++#[tokio::test(start_paused = true)]+async fn defaults_to_3_tries_and_100ms_sleep() {+ let attempts = Arc::new(AtomicU32::new(0));+ let a = attempts.clone();+ let pipeline = new_pipeline::<String, ()>()+ .pipe_with_retry(+ step("flakyStep", move |v: String| {+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;+ async move {+ if n < 3 {+ return Err(PipelineError::retriable("retry"));+ }+ Ok(ok(v))+ }+ }),+ RetryOptions::default(),+ )+ .build();+ let started = tokio::time::Instant::now();+ let r = within_ms(300, pipeline.process(create_ok_context("data".into(), ())))+ .await+ .unwrap();+ assert!(r.is_ok());+ assert_eq!(attempts.load(Ordering::SeqCst), 3);+ // 100 ms + 200 ms nominal, minus at most 5% jitter each.+ let elapsed = started.elapsed().as_millis();+ assert!((285..=300).contains(&elapsed), "{elapsed}");+}++#[tokio::test(start_paused = true)]+async fn pipe_chunk_retries_the_chunk_step_until_it_succeeds() {+ let attempts = Arc::new(AtomicU32::new(0));+ let a = attempts.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk_with_retry(+ chunk_step("flakyChunkStep", move |events: Vec<i64>| {+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;+ async move {+ if n < 3 {+ return Err(PipelineError::retriable("Downstream unavailable"));+ }+ Ok(events.into_iter().map(ok).collect())+ }+ }),+ opts(5, 100),+ )+ .build();+ pipeline.feed(items([1, 2]));+ let results = within_ms(300, pipeline.next()).await.unwrap().unwrap();+ assert_eq!(attempts.load(Ordering::SeqCst), 3);+ assert_eq!(results.len(), 2);+ assert!(results.iter().all(|r| r.is_ok()));+}++#[tokio::test(start_paused = true)]+async fn pipe_chunk_sends_one_dlq_per_input_on_a_non_retriable_error() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk_with_retry(+ chunk_step("failingChunkStep", |_events: Vec<i64>| async move {+ Err::<Vec<ingestion_framework::PipelineResult<i64>>, _>(+ PipelineError::non_retriable("Chunk permanently invalid"),+ )+ }),+ opts(5, 100),+ )+ .build();+ pipeline.feed(items([1, 2, 3]));+ let results = within_ms(0, pipeline.next()).await.unwrap().unwrap();+ assert_eq!(results.len(), 3);+ assert!(results+ .iter()+ .all(|r| r.result.result_type() == PipelineResultType::Dlq));+}diff --git a/rust/ingestion-framework/tests/docs_12_filter_map.rs b/rust/ingestion-framework/tests/docs_12_filter_map.rsnew file mode 100644index 000000000000..f6f37561a87b--- /dev/null+++ b/rust/ingestion-framework/tests/docs_12_filter_map.rs@@ -0,0 +1,172 @@+mod common;++use std::sync::Arc;++use common::{shared, test_message, MockOutputs};+use ingestion_framework::warning_handling::OutputsWarningEmitter;+use ingestion_framework::{+ chunk_step, create_ok_context, dlq, new_chunk_pipeline, ok, redirect, HasMessage, HasTeamId,+ KafkaMessage, OkResultWithContext, PipelineConfig, PipelineWarning, PromiseScheduler,+ SideEffectHandlingConfig, DLQ_OUTPUT, INGESTION_WARNINGS_OUTPUT,+};+use serde_json::json;++#[derive(Debug, Clone)]+struct MsgCtx {+ message: KafkaMessage,+}++#[derive(Debug, Clone)]+struct TeamMsgCtx {+ message: KafkaMessage,+ team_id: Option<i64>,+}++impl From<MsgCtx> for TeamMsgCtx {+ fn from(c: MsgCtx) -> Self {+ Self {+ message: c.message,+ team_id: None,+ }+ }+}+impl HasMessage for TeamMsgCtx {+ fn message(&self) -> &KafkaMessage {+ &self.message+ }+}+impl HasTeamId for TeamMsgCtx {+ fn team_id(&self) -> i64 {+ self.team_id+ .expect("team set inside the filter_map sub-pipeline")+ }+}++#[derive(Debug, Clone)]+struct RawEvent {+ team_id: i64,+ name: &'static str,+}++#[tokio::test]+async fn filter_map_enables_team_aware_processing_with_non_ok_passthrough() {+ let processed = shared(Vec::<(&'static str, i64)>::new());+ let p = processed.clone();+ let outputs = MockOutputs::default();+ let warning_outputs = MockOutputs::default();+ let scheduler = PromiseScheduler::new();+ let wo = warning_outputs.clone();++ let pipeline = new_chunk_pipeline::<RawEvent, MsgCtx>()+ .pipe_chunk(chunk_step(+ "teamLookupStep",+ |events: Vec<RawEvent>| async move {+ Ok(events+ .into_iter()+ .map(|e| {+ if e.team_id == 999 {+ dlq("Team not found", Some(anyhow::anyhow!("Unknown team")))+ } else {+ ok((e.clone(), e.team_id))+ }+ })+ .collect())+ },+ ))+ .gather()+ .filter_map(+ |el: OkResultWithContext<(RawEvent, i64), MsgCtx>| {+ let team_id = el.value.1;+ OkResultWithContext {+ value: el.value,+ context: el.context.map_data(|c| TeamMsgCtx {+ message: c.message,+ team_id: Some(team_id),+ }),+ }+ },+ move |b| {+ b.team_aware(move |b| {+ b.pipe_chunk(chunk_step(+ "processEventStep",+ move |events: Vec<(RawEvent, i64)>| {+ let p = p.clone();+ async move {+ Ok(events+ .into_iter()+ .map(|item| {+ if item.0.name == "overflow" {+ return redirect("Event overflow", "overflow");+ }+ let mut r = ok(item.clone());+ if item.0.name == "deprecated_event" {+ r = r.with_warning(PipelineWarning::new(+ "event_dropped_by_transformation",+ json!({"eventName": item.0.name}),+ ));+ }+ p.lock().unwrap().push((item.0.name, item.1));+ r+ })+ .collect())+ }+ },+ ))+ .gather()+ })+ .handle_ingestion_warnings(Arc::new(OutputsWarningEmitter {+ outputs: Arc::new(wo),+ }))+ },+ )+ .message_aware(|b| b)+ .handle_results(PipelineConfig {+ outputs: Arc::new(outputs.clone()),+ scheduler: Arc::new(scheduler.clone()),+ reject_on_dlq_failure: false,+ })+ .handle_side_effects(+ Arc::new(scheduler.clone()),+ SideEffectHandlingConfig {+ await_side_effects: true,+ },+ )+ .build();++ let feed = |team_id, name| {+ create_ok_context(+ RawEvent { team_id, name },+ MsgCtx {+ message: test_message(None),+ },+ )+ };+ pipeline.feed(vec![+ feed(1, "pageview"),+ feed(999, "will_fail"),+ feed(2, "overflow"),+ feed(3, "deprecated_event"),+ ]);++ let mut all = Vec::new();+ while let Some(chunk) = pipeline.next().await.unwrap() {+ all.extend(chunk);+ }+ assert_eq!(all.len(), 4);+ assert_eq!(all.iter().filter(|r| r.is_ok()).count(), 2);+ assert_eq!(+ *processed.lock().unwrap(),+ vec![("pageview", 1), ("deprecated_event", 3)]+ );+ let produced: Vec<String> = outputs.calls().into_iter().map(|c| c.0).collect();+ assert!(produced.contains(&DLQ_OUTPUT.to_string()));+ assert!(produced.contains(&"overflow".to_string()));+ let warnings = warning_outputs.calls();+ assert_eq!(warnings.len(), 1);+ assert_eq!(warnings[0].0, INGESTION_WARNINGS_OUTPUT);+ let value = String::from_utf8(warnings[0].1.value.clone().unwrap().to_vec()).unwrap();+ assert!(+ value.contains("\"type\":\"event_dropped_by_transformation\""),+ "{value}"+ );+}diff --git a/rust/ingestion-framework/tests/docs_13_conventions.rs b/rust/ingestion-framework/tests/docs_13_conventions.rsnew file mode 100644index 000000000000..1b6a1f60f520--- /dev/null+++ b/rust/ingestion-framework/tests/docs_13_conventions.rs@@ -0,0 +1,340 @@+mod common;++use std::sync::Arc;++use common::{shared, test_message, MockOutputs};+use ingestion_framework::warning_handling::OutputsWarningEmitter;+use ingestion_framework::{+ create_ok_context, dlq, new_chunk_pipeline, new_pipeline, ok, step, HasMessage, HasTeamId,+ KafkaMessage, OkResultWithContext, PipelineBuilder, PipelineConfig, ProcessingStep,+ PromiseScheduler, SideEffect, SideEffectHandlingConfig, StartPipelineBuilder,+};++#[tokio::test]+async fn steps_are_created_via_factory_functions() {+ fn create_process_step() -> ProcessingStep<String, (String, bool)> {+ step("processStep", |v: String| async move {+ Ok(ok((v.to_uppercase(), true)))+ })+ }+ let pipeline = new_pipeline::<String, ()>()+ .pipe(create_process_step())+ .build();+ let r = pipeline+ .process(create_ok_context("hello".into(), ()))+ .await+ .unwrap();+ assert_eq!(r.value(), Some(&("HELLO".to_string(), true)));+}++#[tokio::test]+async fn pipelines_are_created_via_factory_functions_to_ensure_single_caller() {+ fn create_my_pipeline() -> ingestion_framework::DynPipeline<i64, (i64, bool), ()> {+ new_pipeline::<i64, ()>()+ .pipe(step(+ "processStep",+ |id: i64| async move { Ok(ok((id, true))) },+ ))+ .build()+ }+ let p1 = create_my_pipeline();+ let p2 = create_my_pipeline();+ assert!(!Arc::ptr_eq(&p1, &p2));+ assert_eq!(+ p1.process(create_ok_context(1, ()))+ .await+ .unwrap()+ .value()+ .unwrap()+ .0,+ 1+ );+ assert_eq!(+ p2.process(create_ok_context(2, ()))+ .await+ .unwrap()+ .value()+ .unwrap()+ .0,+ 2+ );+}++#[tokio::test]+async fn factory_functions_accept_configuration_parameters() {+ fn create_multiply_step(multiplier: i64) -> ProcessingStep<i64, (i64, i64)> {+ step("multiplyStep", move |v: i64| async move {+ Ok(ok((v, v * multiplier)))+ })+ }+ let pipeline = new_pipeline::<i64, ()>()+ .pipe(create_multiply_step(3))+ .build();+ let r = pipeline.process(create_ok_context(10, ())).await.unwrap();+ assert_eq!(r.value().unwrap().1, 30);+}++#[tokio::test]+async fn factory_functions_can_inject_multiple_dependencies() {+ struct UserService;+ impl UserService {+ async fn get_user(&self, id: &str) -> String {+ format!("User {id}")+ }+ }+ struct AuditService;+ impl AuditService {+ async fn log(&self, _msg: String) {}+ }+ fn create_enrich_step(+ users: Arc<UserService>,+ audit: Arc<AuditService>,+ ) -> ProcessingStep<String, (String, String, bool)> {+ step("enrichStep", move |user_id: String| {+ let (users, audit) = (users.clone(), audit.clone());+ async move {+ let name = users.get_user(&user_id).await;+ let message = format!("Enriched user {user_id}");+ let effect = SideEffect::spawn(async move {+ audit.log(message).await;+ Ok(())+ });+ Ok(ok((user_id, name, true)).with_side_effect(effect))+ }+ })+ }+ let pipeline = new_pipeline::<String, ()>()+ .pipe(create_enrich_step(+ Arc::new(UserService),+ Arc::new(AuditService),+ ))+ .build();+ let r = pipeline+ .process(create_ok_context("123".into(), ()))+ .await+ .unwrap();+ assert_eq!(r.value().unwrap().1, "User 123");+}++#[tokio::test]+async fn steps_extend_input_types_with_new_properties() {+ // Node extends with intersection types (`T & ValidateOutput`). Rust wraps the input.+ struct Validated<T> {+ inner: T,+ is_valid: bool,+ }+ struct Enriched<T> {+ inner: T,+ enriched_at: i64,+ }+ let pipeline = new_pipeline::<String, ()>()+ .pipe(step("validateStep", |raw: String| async move {+ let is_valid = !raw.is_empty();+ Ok(ok(Validated {+ inner: raw,+ is_valid,+ }))+ }))+ .pipe(step("enrichStep", |v: Validated<String>| async move {+ Ok(ok(Enriched {+ inner: v,+ enriched_at: chrono::Utc::now().timestamp_millis(),+ }))+ }))+ .build();+ let r = pipeline+ .process(create_ok_context("test data".into(), ()))+ .await+ .unwrap();+ let v = r.value().unwrap();+ assert_eq!(v.inner.inner, "test data");+ assert!(v.inner.is_valid);+ assert!(v.enriched_at > 0);+}++#[tokio::test]+async fn terminal_transforms_produce_different_output_shapes() {+ let pipeline = new_pipeline::<(&'static str, bool, i64), ()>()+ .pipe(step(+ "summarizeStep",+ |e: (&'static str, bool, i64)| async move {+ Ok(ok(format!("Event {}: valid={} at {}", e.0, e.1, e.2)))+ },+ ))+ .build();+ let r = pipeline+ .process(create_ok_context(("evt-1", true, 1000), ()))+ .await+ .unwrap();+ assert_eq!(r.value().unwrap(), "Event evt-1: valid=true at 1000");+}++#[tokio::test]+async fn terminal_steps_return_void_with_side_effects() {+ let emitted = shared(Vec::<&'static str>::new());+ let e = emitted.clone();+ let pipeline = new_pipeline::<&'static str, ()>()+ .pipe(step("emitStep", move |id: &'static str| {+ let e = e.clone();+ async move {+ Ok(ok(()).with_side_effect(SideEffect::spawn(async move {+ e.lock().unwrap().push(id);+ Ok(())+ })))+ }+ }))+ .build();+ let mut r = pipeline+ .process(create_ok_context("evt-1", ()))+ .await+ .unwrap();+ assert_eq!(r.value(), Some(&()));+ for s in std::mem::take(&mut r.context.side_effects) {+ s.wait().await.unwrap();+ }+ assert!(emitted.lock().unwrap().contains(&"evt-1"));+}++#[tokio::test]+async fn simple_terminal_steps_return_ok_unit() {+ let pipeline = new_pipeline::<bool, ()>()+ .pipe(step("skipStep", |_skip: bool| async move { Ok(ok(())) }))+ .build();+ assert!(pipeline+ .process(create_ok_context(true, ()))+ .await+ .unwrap()+ .is_ok());+}++#[derive(Debug, Clone)]+struct MsgCtx {+ message: KafkaMessage,+}+#[derive(Debug, Clone)]+struct TeamMsgCtx {+ message: KafkaMessage,+ team_id: Option<i64>,+}+impl From<MsgCtx> for TeamMsgCtx {+ fn from(c: MsgCtx) -> Self {+ Self {+ message: c.message,+ team_id: None,+ }+ }+}+impl HasMessage for MsgCtx {+ fn message(&self) -> &KafkaMessage {+ &self.message+ }+}+impl HasMessage for TeamMsgCtx {+ fn message(&self) -> &KafkaMessage {+ &self.message+ }+}+impl HasTeamId for TeamMsgCtx {+ fn team_id(&self) -> i64 {+ self.team_id.unwrap_or_default()+ }+}++#[derive(Debug, Clone)]+struct PreTeam {+ team_id: i64,+ event_name: String,+}++#[tokio::test]+async fn pipelines_are_organized_into_preprocessing_and_processing_phases() {+ let phase_log = shared(Vec::<&'static str>::new());+ let scheduler = PromiseScheduler::new();++ let (l1, l2, l3) = (phase_log.clone(), phase_log.clone(), phase_log.clone());+ let pre_team = move |b: StartPipelineBuilder<KafkaMessage, MsgCtx>| -> PipelineBuilder<KafkaMessage, PreTeam, MsgCtx> {+ b.pipe(step("parseStep", move |m: KafkaMessage| {+ l1.lock().unwrap().push("pre-team: parse");+ async move {+ match serde_json::from_slice::<serde_json::Value>(m.value.as_deref().unwrap_or(b"{}")) {+ Ok(v) => Ok(ok(PreTeam {+ team_id: v["teamId"].as_i64().unwrap_or_default(),+ event_name: v["eventName"].as_str().unwrap_or_default().to_string(),+ })),+ Err(e) => Ok(dlq("Failed to parse message", Some(e.into()))),+ }+ }+ }))+ };+ let post_team = move |b: StartPipelineBuilder<PreTeam, TeamMsgCtx>| {+ b.pipe(step("validateStep", move |p: PreTeam| {+ l2.lock().unwrap().push("post-team: validate");+ async move { Ok(ok((p, true))) }+ }))+ };+ let processing = move |b: StartPipelineBuilder<(PreTeam, bool), TeamMsgCtx>| {+ b.pipe(step(+ "processStep",+ move |(p, _validated): (PreTeam, bool)| {+ l3.lock().unwrap().push("processing: process");+ async move { Ok(ok(format!("{} for Team {}", p.event_name, p.team_id))) }+ },+ ))+ };++ let pipeline =+ new_chunk_pipeline::<KafkaMessage, MsgCtx>()+ .message_aware(move |b| {+ b.concurrently(pre_team, None).filter_map(+ |el: OkResultWithContext<PreTeam, MsgCtx>| {+ let team_id = el.value.team_id;+ OkResultWithContext {+ value: el.value,+ context: el.context.map_data(|c| TeamMsgCtx {+ message: c.message,+ team_id: Some(team_id),+ }),+ }+ },+ move |b| {+ b.team_aware(move |b| {+ b.concurrently(post_team, None).concurrently_per_group(+ |item: &(PreTeam, bool)| item.0.team_id,+ move |g| g.sequentially(processing),+ None,+ )+ })+ .handle_ingestion_warnings(Arc::new(OutputsWarningEmitter {+ outputs: Arc::new(MockOutputs::default()),+ }))+ },+ )+ })+ .handle_results(PipelineConfig {+ outputs: Arc::new(MockOutputs::default()),+ scheduler: Arc::new(scheduler.clone()),+ reject_on_dlq_failure: false,+ })+ .handle_side_effects(+ Arc::new(scheduler),+ SideEffectHandlingConfig {+ await_side_effects: true,+ },+ )+ .build();++ let mut message = test_message(None);+ message.value = Some(br#"{"teamId":42,"eventName":"pageview"}"#.to_vec().into());+ pipeline.feed(vec![create_ok_context(message.clone(), MsgCtx { message })]);+ let results = pipeline.next().await.unwrap().unwrap();+ assert_eq!(+ *phase_log.lock().unwrap(),+ vec![+ "pre-team: parse",+ "post-team: validate",+ "processing: process"+ ]+ );+ assert_eq!(results.len(), 1);+ assert_eq!(results[0].value().unwrap(), "pageview for Team 42");+}diff --git a/rust/ingestion-framework/tests/docs_14_batching.rs b/rust/ingestion-framework/tests/docs_14_batching.rsnew file mode 100644index 000000000000..491b594d0c4a--- /dev/null+++ b/rust/ingestion-framework/tests/docs_14_batching.rs@@ -0,0 +1,203 @@+mod common;++use common::{items, shared};+use ingestion_framework::{+ new_batching_pipeline, ok, step, AfterBatchInput, BatchingPipelineOptions, BeforeBatchInput,+ BeforeBatchOutput, FeedRejectionKind, FeedResult,+};++fn unbounded() -> BatchingPipelineOptions {+ BatchingPipelineOptions {+ concurrent_batches: usize::MAX,+ }+}++fn passthrough_before<CB: Send + 'static>(+ name: &str,+ make: impl Fn(u64) -> CB + Send + Sync + 'static,+) -> ingestion_framework::ProcessingStep<BeforeBatchInput<i64, ()>, BeforeBatchOutput<i64, (), CB>>+{+ ingestion_framework::before_batch_step(name, make)+}++#[tokio::test]+async fn each_feed_forms_one_batch_with_sequential_batch_id_and_message_id_tagging() {+ let before_ids = shared(Vec::<u64>::new());+ let after_ids = shared(Vec::<u64>::new());+ let (b, a) = (before_ids.clone(), after_ids.clone());+ let pipeline = new_batching_pipeline::<i64, _, (), (), u64, ()>(+ move |builder| {+ builder.pipe(step(+ "recordingBefore",+ move |input: BeforeBatchInput<i64, ()>| {+ b.lock().unwrap().push(input.batch_id);+ async move {+ Ok(ok(BeforeBatchOutput {+ batch_context: input.batch_id,+ elements: input.elements,+ batch_id: input.batch_id,+ }))+ }+ },+ ))+ },+ |builder| builder,+ move |builder| {+ builder.pipe(step(+ "recordingAfter",+ move |input: AfterBatchInput<_, (), u64>| {+ a.lock().unwrap().push(input.batch_id);+ async move { Ok(ok(input)) }+ },+ ))+ },+ unbounded(),+ );+ pipeline.feed(items([10, 20]), (), None).await.unwrap();+ pipeline.feed(items([30]), (), None).await.unwrap();+ let mut message_ids = Vec::new();+ while let Some(batch) = pipeline.next().await.unwrap() {+ for element in batch.elements {+ message_ids.push(element.context.message_id.unwrap());+ }+ }+ assert_eq!(*before_ids.lock().unwrap(), vec![0, 1]);+ assert_eq!(*after_ids.lock().unwrap(), vec![0, 1]);+ assert_eq!(message_ids, vec![0, 1, 2]);+}++#[tokio::test]+async fn batch_context_flows_from_before_batch_to_after_batch() {+ let seen = shared(None::<String>);+ let s = seen.clone();+ let pipeline = new_batching_pipeline::<i64, _, (), (), String, ()>(+ |builder| {+ builder.pipe(passthrough_before("attachStore", |id| {+ format!("store-{id}")+ }))+ },+ |builder| builder,+ move |builder| {+ builder.pipe(step(+ "readStore",+ move |input: AfterBatchInput<_, (), String>| {+ *s.lock().unwrap() = Some((*input.batch_context).clone());+ async move { Ok(ok(input)) }+ },+ ))+ },+ unbounded(),+ );+ pipeline.feed(items([1]), (), None).await.unwrap();+ pipeline.next().await.unwrap();+ assert_eq!(seen.lock().unwrap().as_deref(), Some("store-0"));+}++fn simple_pipeline(+ concurrent_batches: usize,+ before_ran: Option<std::sync::Arc<std::sync::Mutex<bool>>>,+) -> ingestion_framework::BatchingPipeline<+ i64,+ ingestion_framework::WithBatch<i64, ()>,+ (),+ (),+ (),+ (),+> {+ new_batching_pipeline(+ move |builder| {+ builder.pipe(step(+ "recordingBefore",+ move |input: BeforeBatchInput<i64, ()>| {+ if let Some(r) = &before_ran {+ *r.lock().unwrap() = true;+ }+ async move {+ Ok(ok(BeforeBatchOutput {+ batch_context: (),+ elements: input.elements,+ batch_id: input.batch_id,+ }))+ }+ },+ ))+ },+ |builder| builder,+ |builder| {+ builder.pipe(ingestion_framework::passthrough_after_batch_step(+ "passThroughAfter",+ ))+ },+ BatchingPipelineOptions { concurrent_batches },+ )+}++#[tokio::test]+async fn feed_reports_at_capacity_backpressure_when_concurrent_batches_is_exceeded() {+ let pipeline = simple_pipeline(1, None);+ assert_eq!(+ pipeline.feed(items([1]), (), None).await.unwrap(),+ FeedResult::Ok+ );+ match pipeline.feed(items([2]), (), None).await.unwrap() {+ FeedResult::Rejected { kind, .. } => assert_eq!(kind, FeedRejectionKind::AtCapacity),+ other => panic!("expected rejection, got {other:?}"),+ }+ let drained = pipeline.next().await.unwrap().unwrap();+ assert!(drained.elements.iter().all(|e| e.is_ok()));+ assert_eq!(+ pipeline.feed(items([3]), (), None).await.unwrap(),+ FeedResult::Ok+ );+}++#[tokio::test]+async fn an_empty_feed_is_a_no_op_that_runs_no_hooks_and_consumes_no_capacity() {+ let ran = shared(false);+ let pipeline = simple_pipeline(1, Some(ran.clone()));+ assert_eq!(+ pipeline.feed(vec![], (), None).await.unwrap(),+ FeedResult::Ok+ );+ assert!(!*ran.lock().unwrap());+ assert!(pipeline.next().await.unwrap().is_none());+ assert_eq!(+ pipeline.feed(items([1]), (), None).await.unwrap(),+ FeedResult::Ok+ );+ assert!(pipeline.next().await.unwrap().is_some());+}++#[tokio::test]+async fn a_before_batch_that_changes_the_element_count_throws() {+ let pipeline = new_batching_pipeline::<i64, _, (), (), (), ()>(+ |builder| {+ builder.pipe(step(+ "filteringBefore",+ |input: BeforeBatchInput<i64, ()>| async move {+ let mut elements = input.elements;+ elements.remove(0);+ Ok(ok(BeforeBatchOutput {+ batch_context: (),+ elements,+ batch_id: input.batch_id,+ }))+ },+ ))+ },+ |builder| builder,+ |builder| {+ builder.pipe(ingestion_framework::passthrough_after_batch_step(+ "passThroughAfter",+ ))+ },+ BatchingPipelineOptions {+ concurrent_batches: 1,+ },+ );+ let err = pipeline.feed(items([1, 2]), (), None).await.unwrap_err();+ assert!(+ err.to_string().contains("changed element count (2 -> 1)"),+ "{err}"+ );+}diff --git a/rust/ingestion-framework/tests/docs_15_consuming_pipelines.rs b/rust/ingestion-framework/tests/docs_15_consuming_pipelines.rsnew file mode 100644index 000000000000..ade132c63b38--- /dev/null+++ b/rust/ingestion-framework/tests/docs_15_consuming_pipelines.rs@@ -0,0 +1,99 @@+mod common;++use std::sync::{Arc, Mutex};++use common::items;+use ingestion_framework::{+ chunk_step, drop, new_chunk_pipeline, ok, ChunkPipelineUnwrapper, SideEffect,+};++#[tokio::test]+async fn feed_then_next_drives_the_pipeline_until_it_returns_null() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("doubleStep", |events: Vec<i64>| async move {+ Ok(events.into_iter().map(|e| ok(e * 2)).collect())+ }))+ .build();+ pipeline.feed(items([1, 2, 3]));+ let mut values = Vec::new();+ while let Some(results) = pipeline.next().await.unwrap() {+ values.extend(results.into_iter().filter_map(|r| r.result.into_value()));+ }+ assert_eq!(values, [2, 4, 6]);+ assert!(pipeline.next().await.unwrap().is_none());+}++#[tokio::test]+async fn chunk_pipeline_unwrapper_returns_plain_values_and_drops_non_ok_results() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("filterStep", |events: Vec<i64>| async move {+ Ok(events+ .into_iter()+ .map(|e| {+ if e % 2 == 0 {+ drop("even filtered")+ } else {+ ok(e)+ }+ })+ .collect())+ }))+ .build();+ let unwrapper = ChunkPipelineUnwrapper::new(pipeline);+ unwrapper.feed(items([1, 2, 3]));+ let mut values = Vec::new();+ while let Some(chunk) = unwrapper.next().await.unwrap() {+ values.extend(chunk);+ }+ assert_eq!(values, [1, 3]);+}++#[derive(Clone, Default)]+struct Capture(Arc<Mutex<Vec<String>>>);++impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for Capture {+ fn on_event(+ &self,+ event: &tracing::Event<'_>,+ _ctx: tracing_subscriber::layer::Context<'_, S>,+ ) {+ struct V<'a>(&'a mut String);+ impl tracing::field::Visit for V<'_> {+ fn record_debug(&mut self, _f: &tracing::field::Field, value: &dyn std::fmt::Debug) {+ self.0.push_str(&format!("{value:?}"));+ }+ }+ let mut s = String::new();+ event.record(&mut V(&mut s));+ self.0.lock().unwrap().push(s);+ }+}++#[tokio::test]+async fn chunk_pipeline_unwrapper_warns_about_unhandled_side_effects() {+ use tracing_subscriber::layer::SubscriberExt;+ let capture = Capture::default();+ let subscriber = tracing_subscriber::registry().with(capture.clone());+ let _guard = tracing::subscriber::set_default(subscriber);++ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step(+ "stepWithSideEffect",+ |events: Vec<i64>| async move {+ Ok(events+ .into_iter()+ .map(|e| ok(e).with_side_effect(SideEffect::resolved()))+ .collect())+ },+ ))+ .build();+ let unwrapper = ChunkPipelineUnwrapper::new(pipeline);+ unwrapper.feed(items([1]));+ unwrapper.next().await.unwrap();+ assert!(capture+ .0+ .lock()+ .unwrap()+ .iter()+ .any(|m| m.contains("remaining side effects")));+}diff --git a/rust/ingestion-framework/tests/docs_17_fan_out_fan_in.rs b/rust/ingestion-framework/tests/docs_17_fan_out_fan_in.rsnew file mode 100644index 000000000000..01af8f3dc910--- /dev/null+++ b/rust/ingestion-framework/tests/docs_17_fan_out_fan_in.rs@@ -0,0 +1,147 @@+mod common;++use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::Arc;++use common::items;+use ingestion_framework::{drop, new_chunk_pipeline, ok, step, PipelineError, RetryOptions};++#[tokio::test]+async fn uploads_attachments_concurrently_and_folds_the_results_back_in() {+ let in_flight = Arc::new(AtomicUsize::new(0));+ let high_water = Arc::new(AtomicUsize::new(0));+ let (f, h) = (in_flight.clone(), high_water.clone());+ let pipeline = new_chunk_pipeline::<(&'static str, Vec<&'static str>), ()>()+ .fan_out(+ "attachmentsFanOut",+ |e: &(&'static str, Vec<&'static str>)| {+ e.1.iter().map(|a| (e.0, *a)).collect::<Vec<_>>()+ },+ )+ .via(move |sub| {+ sub.concurrently(+ move |b| {+ b.pipe(step(+ "uploadStep",+ move |u: (&'static str, &'static str)| {+ let (f, h) = (f.clone(), h.clone());+ async move {+ let now = f.fetch_add(1, Ordering::SeqCst) + 1;+ h.fetch_max(now, Ordering::SeqCst);+ tokio::task::yield_now().await;+ f.fetch_sub(1, Ordering::SeqCst);+ Ok(ok(format!("s3://bucket/{}", u.1)))+ }+ },+ ))+ },+ Some(2),+ )+ })+ .fan_in(+ "collectUrlsFanIn",+ |e: (&'static str, Vec<&'static str>), urls: Vec<String>| {+ let mut urls = urls;+ urls.sort();+ (e.0, urls)+ },+ )+ .build();+ pipeline.feed(items([+ ("e1", vec!["a.png", "b.png", "c.png"]),+ ("e2", vec![]),+ ]));+ let mut collected = common::consume_all(pipeline.as_ref()).await;+ collected.sort();+ assert_eq!(+ collected,+ vec![+ (+ "e1",+ vec![+ "s3://bucket/a.png".to_string(),+ "s3://bucket/b.png".into(),+ "s3://bucket/c.png".into()+ ]+ ),+ ("e2", vec![]),+ ]+ );+ assert!(high_water.load(Ordering::SeqCst) <= 2);+}++#[tokio::test]+async fn retries_a_transient_sub_step_failure_without_disturbing_the_parent() {+ let attempts = Arc::new(AtomicUsize::new(0));+ let a = attempts.clone();+ let pipeline = new_chunk_pipeline::<(&'static str, Vec<i64>), ()>()+ .fan_out("partsFanOut", |j: &(&'static str, Vec<i64>)| j.1.clone())+ .via(move |sub| {+ sub.concurrently(+ move |b| {+ b.pipe_with_retry(+ step("flakyStep", move |part: i64| {+ let n = a.fetch_add(1, Ordering::SeqCst) + 1;+ async move {+ if n == 1 {+ return Err(PipelineError::retriable("transient"));+ }+ Ok(ok(part))+ }+ }),+ RetryOptions {+ tries: Some(3),+ sleep_ms: Some(1),+ ..Default::default()+ },+ )+ },+ None,+ )+ })+ .fan_in(+ "sumFanIn",+ |j: (&'static str, Vec<i64>), parts: Vec<i64>| (j.0, parts.iter().sum::<i64>()),+ )+ .build();+ pipeline.feed(items([("job", vec![7])]));+ let mut results = Vec::new();+ while let Some(chunk) = pipeline.next().await.unwrap() {+ results.extend(chunk);+ }+ assert_eq!(results.len(), 1);+ assert_eq!(results[0].value(), Some(&("job", 7)));+ assert_eq!(attempts.load(Ordering::SeqCst), 2);+}++#[tokio::test]+async fn drops_a_sub_element_without_failing_its_parent() {+ let pipeline = new_chunk_pipeline::<(&'static str, Vec<i64>), ()>()+ .fan_out("partsFanOut", |j: &(&'static str, Vec<i64>)| j.1.clone())+ .via(|sub| {+ sub.concurrently(+ |b| {+ b.pipe(step("validatePartStep", |part: i64| async move {+ Ok(if part < 0 {+ drop("negative part")+ } else {+ ok(part)+ })+ }))+ },+ None,+ )+ })+ .fan_in(+ "sumFanIn",+ |j: (&'static str, Vec<i64>), parts: Vec<i64>| (j.0, parts.iter().sum::<i64>()),+ )+ .build();+ pipeline.feed(items([+ ("partly-valid", vec![1, -2, 3]),+ ("all-valid", vec![1, 2]),+ ]));+ let mut collected = common::consume_all(pipeline.as_ref()).await;+ collected.sort();+ assert_eq!(collected, vec![("all-valid", 3), ("partly-valid", 4)]);+}diff --git a/rust/ingestion-framework/tests/docs_18_batch_budgets.rs b/rust/ingestion-framework/tests/docs_18_batch_budgets.rsnew file mode 100644index 000000000000..972ab0197a1a--- /dev/null+++ b/rust/ingestion-framework/tests/docs_18_batch_budgets.rs@@ -0,0 +1,301 @@+//! `tokio::time::advance` inside a step replaces the synchronous `jest.advanceTimersByTime`.++mod common;++use std::time::Duration;++use common::{items, shared};+use ingestion_framework::{+ before_batch_step, chunk_step, new_batching_pipeline, ok, passthrough_after_batch_step, step,+ BatchBudget, BatchingPipelineOptions, ChunkResults, NonOk, Outcome, WithBatch,+};+use tokio::time::Instant;++#[derive(Debug, Clone, Copy)]+struct Event {+ key: &'static str,+ seq: i64,+}++type V = WithBatch<Event, ()>;++fn kinds<C>(results: &ChunkResults<V, C>) -> Vec<&'static str> {+ results+ .iter()+ .map(|r| match &r.result {+ Outcome::Ok(_) => "ok",+ Outcome::NonOk(NonOk::Timeout { .. }) => "timeout",+ _ => "other",+ })+ .collect()+}++fn reasons<C>(results: &ChunkResults<V, C>) -> Vec<Option<String>> {+ results+ .iter()+ .map(|r| match &r.result {+ Outcome::NonOk(NonOk::Timeout { reason }) => Some(reason.clone()),+ _ => None,+ })+ .collect()+}++fn events(+ key: &'static str,+ seqs: impl IntoIterator<Item = i64>,+) -> Vec<ingestion_framework::OkResultWithContext<Event, ()>> {+ items(seqs.into_iter().map(|seq| Event { key, seq }))+}++async fn advance_1s() {+ tokio::time::advance(Duration::from_millis(1000)).await;+}++fn one() -> BatchingPipelineOptions {+ BatchingPipelineOptions {+ concurrent_batches: 1,+ }+}++#[tokio::test(start_paused = true)]+async fn stops_each_element_at_its_next_step_once_the_budget_is_exhausted() {+ let ran = shared(Vec::<String>::new());+ let (r1, r2) = (ran.clone(), ran.clone());+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),+ move |b| {+ b.sequentially(move |s| {+ s.pipe(step("enrichStep", move |e: V| {+ r1.lock().unwrap().push(format!("enrich-{}", e.seq));+ async move {+ if e.seq == 2 {+ advance_1s().await;+ }+ Ok(ok(e))+ }+ }))+ .pipe(step("emitStep", move |e: V| {+ r2.lock().unwrap().push(format!("emit-{}", e.seq));+ async move { Ok(ok(e)) }+ }))+ })+ },+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),+ one(),+ );+ pipeline+ .feed(+ events("a", [1, 2, 3]),+ (),+ Some(BatchBudget::soft_deadline(+ Instant::now() + Duration::from_millis(1000),+ )),+ )+ .await+ .unwrap();+ let batch = pipeline.next().await.unwrap().unwrap();+ assert_eq!(*ran.lock().unwrap(), vec!["enrich-1", "emit-1", "enrich-2"]);+ assert_eq!(kinds(&batch.elements), ["ok", "timeout", "timeout"]);+ assert_eq!(+ reasons(&batch.elements),+ [+ None,+ Some("budget exceeded before emitStep".into()),+ Some("budget exceeded before enrichStep".into())+ ]+ );+}++#[tokio::test(start_paused = true)]+async fn runs_a_chunk_step_on_the_elements_whose_batch_still_has_time() {+ let chunked = shared(Vec::<Vec<i64>>::new());+ let c = chunked.clone();+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),+ move |b| {+ b.pipe_chunk(chunk_step("writeChunk", move |events: Vec<V>| {+ c.lock()+ .unwrap()+ .push(events.iter().map(|e| e.seq).collect());+ async move { Ok(events.into_iter().map(ok).collect()) }+ }))+ },+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),+ BatchingPipelineOptions {+ concurrent_batches: 2,+ },+ );+ pipeline+ .feed(+ events("a", [1]),+ (),+ Some(BatchBudget::soft_deadline(+ Instant::now() - Duration::from_millis(1),+ )),+ )+ .await+ .unwrap();+ pipeline+ .feed(+ events("b", [2]),+ (),+ Some(BatchBudget::soft_deadline(+ Instant::now() + Duration::from_secs(60),+ )),+ )+ .await+ .unwrap();+ let spent = pipeline.next().await.unwrap().unwrap();+ let fresh = pipeline.next().await.unwrap().unwrap();+ assert_eq!(*chunked.lock().unwrap(), vec![vec![2]]);+ assert_eq!(kinds(&spent.elements), ["timeout"]);+ assert_eq!(+ reasons(&spent.elements),+ [Some("budget exceeded before writeChunk".into())]+ );+ assert_eq!(kinds(&fresh.elements), ["ok"]);+}++#[tokio::test(start_paused = true)]+async fn runs_an_element_past_the_barrier_to_completion_after_the_budget_expires() {+ let ran = shared(Vec::<String>::new());+ let (r1, r2, r3) = (ran.clone(), ran.clone(), ran.clone());+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),+ move |b| {+ b.sequentially(move |s| {+ s.pipe(step("prepareEvent", move |e: V| {+ r1.lock().unwrap().push(format!("prepare-{}", e.seq));+ async move { Ok(ok(e)) }+ }))+ .timeout_barrier()+ .pipe(step("writePerson", move |e: V| {+ r2.lock().unwrap().push(format!("write-{}", e.seq));+ async move {+ advance_1s().await;+ Ok(ok(e))+ }+ }))+ .pipe(step("emitEvent", move |e: V| {+ r3.lock().unwrap().push(format!("emit-{}", e.seq));+ async move { Ok(ok(e)) }+ }))+ })+ },+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),+ one(),+ );+ pipeline+ .feed(+ events("a", [1, 2]),+ (),+ Some(BatchBudget::soft_deadline(+ Instant::now() + Duration::from_millis(1000),+ )),+ )+ .await+ .unwrap();+ let batch = pipeline.next().await.unwrap().unwrap();+ assert_eq!(*ran.lock().unwrap(), vec!["prepare-1", "write-1", "emit-1"]);+ assert_eq!(kinds(&batch.elements), ["ok", "timeout"]);+ assert_eq!(+ reasons(&batch.elements),+ [None, Some("budget exceeded before prepareEvent".into())]+ );+}++#[tokio::test(start_paused = true)]+async fn returns_one_result_per_message_and_still_runs_after_batch_when_everything_times_out() {+ let flushed = shared(0usize);+ let f = flushed.clone();+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),+ |b| b.sequentially(|s| s.pipe(step("emitStep", |e: V| async move { Ok(ok(e)) }))),+ move |b| {+ b.pipe(step(+ "flushBatch",+ move |input: ingestion_framework::AfterBatchInput<V, (), ()>| {+ *f.lock().unwrap() = input.elements.len();+ async move { Ok(ok(input)) }+ },+ ))+ },+ one(),+ );+ let mut fed = events("a", [1, 2]);+ fed.extend(events("b", [1]));+ pipeline+ .feed(+ fed,+ (),+ Some(BatchBudget::soft_deadline(+ Instant::now() - Duration::from_millis(1),+ )),+ )+ .await+ .unwrap();+ let batch = pipeline.next().await.unwrap().unwrap();+ assert_eq!(batch.elements.len(), 3);+ assert_eq!(kinds(&batch.elements), ["timeout", "timeout", "timeout"]);+ assert_eq!(*flushed.lock().unwrap(), 3);+}++#[tokio::test(start_paused = true)]+async fn completes_a_prefix_of_each_groups_feed_order() {+ let processed = shared(Vec::<i64>::new());+ let p = processed.clone();+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),+ move |b| {+ b.concurrently_per_group(+ |e: &V| e.key,+ move |g| {+ g.sequentially(move |s| {+ s.pipe(step("processStep", move |e: V| {+ let count = {+ let mut p = p.lock().unwrap();+ p.push(e.seq);+ p.len()+ };+ async move {+ if count == 2 {+ advance_1s().await;+ }+ Ok(ok(e))+ }+ }))+ })+ },+ None,+ )+ },+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),+ one(),+ );+ pipeline+ .feed(+ events("a", [1, 2, 3, 4]),+ (),+ Some(BatchBudget::soft_deadline(+ Instant::now() + Duration::from_millis(1000),+ )),+ )+ .await+ .unwrap();+ let batch = pipeline.next().await.unwrap().unwrap();+ assert_eq!(*processed.lock().unwrap(), vec![1, 2]);+ assert_eq!(kinds(&batch.elements), ["ok", "ok", "timeout", "timeout"]);+}++#[tokio::test(start_paused = true)]+async fn never_cancels_anything_under_the_default_unlimited_budget() {+ let pipeline = new_batching_pipeline::<Event, V, (), (), (), ()>(+ |b| b.pipe(before_batch_step("passThroughBefore", |_| ())),+ |b| b.sequentially(|s| s.pipe(step("emitStep", |e: V| async move { Ok(ok(e)) }))),+ |b| b.pipe(passthrough_after_batch_step("passThroughAfter")),+ one(),+ );+ pipeline.feed(events("a", [1]), (), None).await.unwrap();+ let batch = pipeline.next().await.unwrap().unwrap();+ assert_eq!(kinds(&batch.elements), ["ok"]);+}diff --git a/rust/ingestion-framework/tests/per_element_step.rs b/rust/ingestion-framework/tests/per_element_step.rsnew file mode 100644index 000000000000..2713b3c10afd--- /dev/null+++ b/rust/ingestion-framework/tests/per_element_step.rs@@ -0,0 +1,48 @@+use std::sync::{Arc, Mutex};++use ingestion_framework::{ok, per_element, step, PipelineResult};++#[derive(Default)]+struct Track {+ running: usize,+ peak: usize,+ started: Vec<i64>,+}++#[tokio::test]+async fn keeps_input_order_and_never_runs_more_than_max_concurrency_steps_at_once() {+ let track = Arc::new(Mutex::new(Track::default()));+ let t = track.clone();+ let chunk = per_element(+ step("slowStep", move |value: i64| {+ let t = t.clone();+ async move {+ {+ let mut t = t.lock().unwrap();+ t.started.push(value);+ t.running += 1;+ t.peak = t.peak.max(t.running);+ }+ // Later values wait fewer ticks and finish first, so output order cannot follow completion order.+ for _ in 0..(8 - value) {+ tokio::task::yield_now().await;+ }+ t.lock().unwrap().running -= 1;+ Ok(ok(value * 10))+ }+ }),+ Some(3),+ );++ let results: Vec<PipelineResult<i64>> = chunk.call(vec![1, 2, 3, 4, 5, 6, 7]).await.unwrap();++ let values: Vec<i64> = results+ .into_iter()+ .map(|r| r.outcome.into_value().unwrap())+ .collect();+ assert_eq!(values, [10, 20, 30, 40, 50, 60, 70]);+ let track = track.lock().unwrap();+ assert_eq!(track.peak, 3);+ assert_eq!(track.started, [1, 2, 3, 4, 5, 6, 7]);+ assert_eq!(chunk.name().as_ref(), "slowStep");+}diff --git a/rust/ingestion-framework/tests/review_regressions.rs b/rust/ingestion-framework/tests/review_regressions.rsnew file mode 100644index 000000000000..10330958b8c4--- /dev/null+++ b/rust/ingestion-framework/tests/review_regressions.rs@@ -0,0 +1,419 @@+mod common;++use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::Arc;+use std::time::{Duration, Instant};++use common::{items, test_message, MockOutputs, MsgCtx};+use futures::FutureExt;+use ingestion_framework::warning_handling::serialize_ingestion_warning;+use ingestion_framework::{+ chunk_step, create_ok_context, dlq, new_batching_pipeline, new_chunk_pipeline, new_pipeline,+ ok, redirect_with, step, BatchingPipelineOptions, ChunkPipeline, MemoryOutputs, PipelineConfig,+ PipelineError, PipelineWarning, PromiseScheduler, RetryOptions, SideEffect,+ SideEffectHandlingConfig, SideEffectScheduler, DLQ_OUTPUT,+};++const AWAIT: SideEffectHandlingConfig = SideEffectHandlingConfig {+ await_side_effects: true,+};++// The expected prefix is Node's `JSON.stringify({team_id, type, source, details, timestamp})`.+#[test]+fn warning_message_bytes_match_node() {+ let mut w = PipelineWarning::new("client_ingestion_warning", serde_json::json!({}))+ .with_detail("zeta", 1)+ .with_detail("category", "caller")+ .with_detail("alpha", 1.0)+ .with_detail("big", 1e21);+ w.pipeline_step = Some("s".into());+ let s = serialize_ingestion_warning(42, &w).unwrap();+ let expected = r#"{"team_id":42,"type":"client_ingestion_warning","source":"plugin-server","details":"{\"zeta\":1,\"category\":\"event\",\"alpha\":1,\"big\":1e+21,\"severity\":\"info\",\"pipelineStep\":\"s\"}","timestamp":""#;+ assert!(s.starts_with(expected), "{s}");+ let timestamp = &s[expected.len()..s.len() - 2];+ assert!(+ chrono::NaiveDateTime::parse_from_str(timestamp, "%Y-%m-%d %H:%M:%S%.3f").is_ok()+ && timestamp.len() == 23,+ "{timestamp}"+ );+}++#[test]+fn warning_type_outside_the_registry_fails_like_the_node_lookup() {+ let w = PipelineWarning::new("not_a_registered_type", serde_json::json!({}));+ assert!(serialize_ingestion_warning(1, &w).is_err());+}++// Node's DLQ error is `AggregateError(errors, reason)`, and `dlq_reason` is its message.+#[tokio::test]+async fn fan_out_dlq_reason_header_is_the_aggregate_reason_only() {+ let outputs = MockOutputs::default();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()+ .fan_out("split", |n: &i64| vec![*n])+ .via(|b| {+ b.pipe_chunk(chunk_step("check", |v: Vec<i64>| async move {+ Ok(v.into_iter()+ .map(|_| dlq::<i64>("bad sub", Some(anyhow::anyhow!("boom"))))+ .collect())+ }))+ })+ .fan_in("merge", |o: i64, _s: Vec<i64>| o)+ .message_aware(|b| b)+ .handle_results(PipelineConfig {+ outputs: Arc::new(outputs.clone()),+ scheduler: Arc::new(scheduler.clone()),+ reject_on_dlq_failure: false,+ })+ .handle_side_effects(Arc::new(scheduler), AWAIT)+ .build();+ pipeline.feed(vec![create_ok_context(+ 1,+ MsgCtx {+ message: test_message(None),+ },+ )]);+ pipeline.next().await.unwrap().unwrap();+ let calls = outputs.calls();+ assert_eq!(+ calls[0].1.headers.get("dlq_reason"),+ Some("1/1 fan-out sub-elements dlq'd: bad sub")+ );+}++// Node calls `outputs.produce` synchronously, in item order, inside `next()`.+#[test]+fn dlq_produces_follow_item_order_on_a_multi_thread_runtime() {+ let rt = tokio::runtime::Builder::new_multi_thread()+ .worker_threads(4)+ .enable_all()+ .build()+ .unwrap();+ for _ in 0..50 {+ let order = rt+ .block_on(async {+ tokio::spawn(async {+ let outputs = MemoryOutputs::new();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()+ .pipe_chunk(chunk_step("v", |v: Vec<i64>| async move {+ Ok(v.into_iter().map(|_| dlq::<i64>("x", None)).collect())+ }))+ .message_aware(|b| b)+ .handle_results(PipelineConfig {+ outputs: Arc::new(outputs.clone()),+ scheduler: Arc::new(scheduler.clone()),+ reject_on_dlq_failure: false,+ })+ .handle_side_effects(Arc::new(scheduler), AWAIT)+ .build();+ pipeline.feed(+ (0..8)+ .map(|i| {+ let mut m = test_message(Some(b"k"));+ m.offset = i;+ create_ok_context(i, MsgCtx { message: m })+ })+ .collect(),+ );+ pipeline.next().await.unwrap();+ outputs+ .produced_to(DLQ_OUTPUT)+ .iter()+ .map(|m| m.headers.get("dlq_offset").unwrap().parse::<i64>().unwrap())+ .collect::<Vec<_>>()+ })+ .await+ })+ .unwrap();+ assert_eq!(order, (0..8).collect::<Vec<_>>());+ }+}++// Node starts the redirect produce synchronously, so it is called when next() resolves.+#[tokio::test]+async fn redirect_without_await_ack_has_called_produce_when_next_returns() {+ let outputs = MockOutputs::default();+ outputs.close_gate();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()+ .pipe_chunk(chunk_step("r", |v: Vec<i64>| async move {+ Ok(v.into_iter()+ .map(|_| redirect_with::<i64>("o", "overflow", true, false))+ .collect())+ }))+ .message_aware(|b| b)+ .handle_results(PipelineConfig {+ outputs: Arc::new(outputs.clone()),+ scheduler: Arc::new(scheduler.clone()),+ reject_on_dlq_failure: false,+ })+ .handle_side_effects(Arc::new(scheduler), AWAIT)+ .build();+ pipeline.feed(vec![create_ok_context(+ 1,+ MsgCtx {+ message: test_message(None),+ },+ )]);+ pipeline.next().await.unwrap();+ assert_eq!(outputs.calls().len(), 1);+}++// Node has no cancellation, so a batch always comes back, even after a dropped `next()`.+#[tokio::test(start_paused = true)]+async fn dropped_batching_next_does_not_lose_the_batch() {+ let after_runs = Arc::new(AtomicU32::new(0));+ let runs = after_runs.clone();+ let pipeline = new_batching_pipeline::<i64, _, (), (), (), ()>(+ |b| b.pipe(ingestion_framework::before_batch_step("before", |_| ())),+ |b| b,+ move |b| {+ let runs = runs.clone();+ b.pipe(step(+ "slowAfter",+ move |input: ingestion_framework::AfterBatchInput<_, (), ()>| {+ runs.fetch_add(1, Ordering::SeqCst);+ async move {+ tokio::time::sleep(Duration::from_millis(100)).await;+ Ok(ok(input))+ }+ },+ ))+ },+ BatchingPipelineOptions {+ concurrent_batches: 1,+ },+ );+ pipeline.feed(items([1, 2]), (), None).await.unwrap();+ let _ = tokio::time::timeout(Duration::from_millis(10), pipeline.next()).await;+ let batch = pipeline.next().await.unwrap().expect("batch was lost");+ assert_eq!(batch.elements.len(), 2);+ assert_eq!(after_runs.load(Ordering::SeqCst), 1, "afterBatch ran twice");+ assert!(pipeline.next().await.unwrap().is_none());+}++#[tokio::test]+async fn scheduler_releases_settled_side_effects() {+ let scheduler = PromiseScheduler::new();+ for _ in 0..1000 {+ scheduler.schedule(SideEffect::spawn(async {+ tokio::task::yield_now().await;+ Ok(())+ }));+ }+ tokio::time::sleep(Duration::from_millis(20)).await;+ assert_eq!(scheduler.pending_count(), 0);+}++// Node's retryIfRetriable with tries: 0 never calls fn and throws.+#[tokio::test]+async fn retry_with_zero_tries_never_calls_the_step() {+ let calls = Arc::new(AtomicU32::new(0));+ let c = calls.clone();+ let p = new_pipeline::<i64, ()>()+ .pipe_with_retry(+ step("s", move |v: i64| {+ c.fetch_add(1, Ordering::SeqCst);+ async move { Ok(ok(v)) }+ }),+ RetryOptions {+ tries: Some(0),+ ..Default::default()+ },+ )+ .build();+ let r = p.process(create_ok_context(1, ())).await;+ assert_eq!(calls.load(Ordering::SeqCst), 0);+ assert!(r.is_err());+}++// Node reads only error.isRetriable; a wrapper error without the flag is retried.+#[tokio::test(start_paused = true)]+async fn wrapped_non_retriable_error_is_retried_like_node() {+ let calls = Arc::new(AtomicU32::new(0));+ let c = calls.clone();+ let p = new_pipeline::<i64, ()>()+ .pipe_with_retry(+ step("s", move |_v: i64| {+ c.fetch_add(1, Ordering::SeqCst);+ async move {+ Err::<ingestion_framework::PipelineResult<i64>, _>(+ PipelineError::non_retriable("inner").context("outer"),+ )+ }+ }),+ RetryOptions::default(),+ )+ .build();+ let _ = p.process(create_ok_context(1, ())).await;+ assert_eq!(calls.load(Ordering::SeqCst), 3);+}++// A Node fan-in throw rejects next() and poisons the stage without crashing the process.+#[tokio::test]+async fn fan_in_panic_rejects_next_and_keeps_rejecting() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .fan_out("split", |n: &i64| vec![*n])+ .via(|b| b)+ .fan_in("explode", |_o: i64, _s: Vec<i64>| -> i64 {+ panic!("fan-in bug")+ })+ .build();+ pipeline.feed(items([1]));+ for _ in 0..2 {+ let outcome = std::panic::AssertUnwindSafe(pipeline.next())+ .catch_unwind()+ .await;+ let error = outcome.expect("next() must not panic").unwrap_err();+ assert!(error.to_string().contains("fan-in bug"), "{error}");+ }+}++// `pLimit(0)` throws when the Node stage is built.+#[test]+#[should_panic(expected = "from 1 and up")]+fn zero_max_concurrency_fails_like_p_limit() {+ let _ = new_chunk_pipeline::<i64, ()>().concurrently(+ |b| b.pipe(step("s", |v: i64| async move { Ok(ok(v)) })),+ Some(0),+ );+}++// A quadratic start makes 4x the distinct keys cost 16x the time.+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn grouping_start_scales_linearly_in_distinct_keys() {+ async fn run(n: i64) -> Duration {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently_per_group(+ |v: &i64| *v,+ |g| g.sequentially(|b| b.pipe(step("s", |v: i64| async move { Ok(ok(v)) }))),+ None,+ )+ .build();+ pipeline.feed(items(0..n));+ let t = Instant::now();+ common::consume_all(pipeline.as_ref()).await;+ t.elapsed()+ }+ // Warm the runtime and allocator first, so the small run is not inflated.+ run(5_000).await;+ let small = run(10_000).await;+ let big = run(40_000).await;+ assert!(+ big < small * 8,+ "4x keys took {:.1}x ({small:?} vs {big:?})",+ big.as_secs_f64() / small.as_secs_f64()+ );+}++// Node keeps a promise in the scheduler until it settles.+#[tokio::test]+async fn scheduler_keeps_in_flight_side_effects_for_every_waiter() {+ let scheduler = PromiseScheduler::new();+ let (release, gate) = tokio::sync::oneshot::channel::<()>();+ scheduler.schedule(SideEffect::spawn(async move {+ let _ = gate.await;+ Err(anyhow::anyhow!("produce failed"))+ }));+ let mut cancelled = Box::pin(scheduler.wait_for_all());+ assert!(futures::poll!(cancelled.as_mut()).is_pending());+ let mut overlapping = Box::pin(scheduler.wait_for_all());+ assert!(futures::poll!(overlapping.as_mut()).is_pending());+ drop(cancelled);+ release.send(()).unwrap();+ let error = overlapping.await.unwrap_err();+ assert!(error.to_string().contains("produce failed"), "{error}");+ assert!(scheduler.wait_for_all().await.is_ok());+}++// A Node fan-in throw leaves the parents completed before it in readyResults.+#[tokio::test]+async fn fan_in_panic_keeps_the_parents_completed_before_it() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .fan_out("split", |n: &i64| vec![*n])+ .via(|b| b)+ .fan_in("explode_on_2", |o: i64, _s: Vec<i64>| -> i64 {+ assert!(o != 2, "fan-in bug");+ o+ })+ .build();+ pipeline.feed(items([1, 2]));+ let first = std::panic::AssertUnwindSafe(pipeline.next())+ .catch_unwind()+ .await+ .expect("next() must not panic");+ assert!(first.is_err());+ let kept = pipeline.next().await.unwrap().expect("completed parent");+ let values: Vec<_> = kept+ .into_iter()+ .filter_map(|r| r.result.into_value())+ .collect();+ assert_eq!(values, vec![1]);+ assert!(pipeline.next().await.is_err());+}++// A Node group processor that throws before it returns its promise rejects that group.+#[tokio::test]+async fn group_processor_panic_before_its_future_fails_next() {+ let processor: ingestion_framework::grouping::GroupChunkProcessor<i64, i64, ()> =+ Arc::new(|_items| panic!("processor bug"));+ let pipeline = ingestion_framework::grouping::ConcurrentlyGroupingChunkPipeline::new(+ Arc::new(|v: &i64| *v),+ processor,+ new_chunk_pipeline::<i64, ()>().build(),+ None,+ );+ pipeline.feed(items([1]));+ let outcome = tokio::time::timeout(Duration::from_secs(5), pipeline.next())+ .await+ .expect("next() must not wait forever");+ let error = outcome.unwrap_err();+ assert!(error.to_string().contains("processor bug"), "{error}");+}++// p-limit does not hold a slot for a caller that is gone.+#[tokio::test]+async fn limiter_releases_the_slot_of_a_dropped_acquire() {+ let limiter = ingestion_framework::sync::FifoLimiter::new(1);+ drop(limiter.acquire());+ let held = tokio::time::timeout(Duration::from_secs(5), limiter.acquire())+ .await+ .expect("the slot of the dropped acquire was released");+ // This waiter gets the slot when `held` drops, and is gone before it reads it.+ let queued = limiter.acquire();+ drop(held);+ drop(queued);+ tokio::time::timeout(Duration::from_secs(5), limiter.acquire())+ .await+ .expect("the slot was released");+}++// Node's setTimeout uses 1 ms for a negative, NaN or too-large delay.+#[tokio::test(start_paused = true)]+async fn retry_with_out_of_range_sleep_options_does_not_panic() {+ let cases = [+ (Some(-1.0), Some(0.0)),+ (Some(f64::NAN), None),+ (Some(f64::INFINITY), Some(0.0)),+ (None, Some(3.0)),+ (Some(f64::MAX), Some(-1.0)),+ ];+ for (backoff_factor, jitter) in cases {+ let options = RetryOptions {+ tries: Some(4),+ sleep_ms: Some(100),+ max_sleep_ms: Some(u64::MAX),+ backoff_factor,+ jitter,+ ..RetryOptions::default()+ };+ let (result, attempts) = ingestion_framework::retry::retry_if_retriable(+ || async { Err::<(), _>(anyhow::anyhow!("retriable")) },+ &options,+ )+ .await;+ assert!(result.is_err());+ assert_eq!(attempts, 4, "{backoff_factor:?} {jitter:?}");+ }+}diff --git a/rust/ingestion-framework/tests/unit_semantics.rs b/rust/ingestion-framework/tests/unit_semantics.rsnew file mode 100644index 000000000000..674fd08d20e3--- /dev/null+++ b/rust/ingestion-framework/tests/unit_semantics.rs@@ -0,0 +1,471 @@+mod common;++use std::sync::Arc;+use std::time::Duration;++use common::{items, shared, sleep_ms, test_message, MsgCtx};+use ingestion_framework::{+ chunk_step, create_ok_context, dlq, new_chunk_pipeline, ok, step, timeout, MemoryOutputs,+ PipelineConfig, PipelineResultType, PipelineWarning, PromiseScheduler,+ SideEffectHandlingConfig, DLQ_OUTPUT,+};+use tokio::time::Instant;++fn failing_on(n: i64) -> ingestion_framework::ProcessingStep<i64, i64> {+ step("maybeFails", move |v: i64| async move {+ sleep_ms(10 * v as u64).await;+ if v == n {+ anyhow::bail!("boom {v}");+ }+ Ok(ok(v))+ })+}++#[tokio::test(start_paused = true)]+async fn concurrent_poison_drains_in_flight_items_in_order_then_fails_for_good() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently(|b| b.pipe(failing_on(2)), None)+ .build();+ pipeline.feed(items([1, 2, 3]));+ assert_eq!(pipeline.next().await.unwrap().unwrap()[0].value(), Some(&1));+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");+ assert_eq!(pipeline.next().await.unwrap().unwrap()[0].value(), Some(&3));+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");+}++#[tokio::test(start_paused = true)]+async fn concurrent_upstream_failure_poisons_the_stage() {+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .pipe_chunk(chunk_step("explodes", |_v: Vec<i64>| async move {+ Err::<Vec<ingestion_framework::PipelineResult<i64>>, _>(anyhow::anyhow!("upstream"))+ }))+ .concurrently(+ |b| b.pipe(step("id", |v: i64| async move { Ok(ok(v)) })),+ None,+ )+ .build();+ pipeline.feed(items([1]));+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "upstream");+ pipeline.feed(items([2]));+ // Poisoned: no new source pull, so the later feed never runs.+ assert_eq!(pipeline.next().await.unwrap_err().to_string(), "upstream");+}++#[tokio::test(start_paused = true)]+async fn concurrent_feed_while_parked_starts_new_input_without_waiting_for_the_head() {+ let starts = shared(Vec::<(i64, u128)>::new());+ let s = starts.clone();+ let origin = Instant::now();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently(+ move |b| {+ b.pipe(step("slowHead", move |v: i64| {+ s.lock().unwrap().push((v, origin.elapsed().as_millis()));+ async move {+ sleep_ms(if v == 1 { 100 } else { 1 }).await;+ Ok(ok(v))+ }+ }))+ },+ None,+ )+ .build();+ pipeline.feed(items([1]));+ let p = pipeline.clone();+ let consumer = tokio::spawn(async move { common::consume_all(p.as_ref()).await });+ sleep_ms(10).await;+ pipeline.feed(items([2]));+ let values = consumer.await.unwrap();+ assert_eq!(values, [1, 2]);+ let starts = starts.lock().unwrap().clone();+ assert_eq!(starts[1].0, 2);+ assert!(+ starts[1].1 < 100,+ "item 2 started at {}ms, behind the slow head",+ starts[1].1+ );+}++#[tokio::test(start_paused = true)]+async fn max_concurrency_admits_items_in_input_order() {+ let started = shared(Vec::<i64>::new());+ let s = started.clone();+ let pipeline = new_chunk_pipeline::<i64, ()>()+ .concurrently(+ move |b| {+ b.pipe(step("record", move |v: i64| {+ s.lock().unwrap().push(v);+ async move {+ sleep_ms(5).await;+ Ok(ok(v))+ }+ }))+ },+ Some(1),+ )+ .build();+ pipeline.feed(items(1..=20));+ let values = common::consume_all(pipeline.as_ref()).await;+ assert_eq!(values, (1..=20).collect::<Vec<_>>());+ assert_eq!(*started.lock().unwrap(), (1..=20).collect::<Vec<_>>());+}++#[tokio::test(start_paused = true)]+async fn grouping_keeps_per_key_order_across_next_calls_and_feeds() {+ let seen = shared(Vec::<(&'static str, i64)>::new());+ let s = seen.clone();+ let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()+ .concurrently_per_group(+ |e: &(&'static str, i64)| e.0,+ move |g| {+ g.sequentially(move |b| {+ b.pipe(step("record", move |e: (&'static str, i64)| {+ let s = s.clone();+ async move {+ sleep_ms(if e.1 % 2 == 0 { 3 } else { 7 }).await;+ s.lock().unwrap().push(e);+ Ok(ok(e))+ }+ }))+ })+ },+ Some(2),+ )+ .build();+ pipeline.feed(items([("a", 1), ("b", 1), ("a", 2)]));+ let first = pipeline.next().await.unwrap();+ pipeline.feed(items([("a", 3), ("c", 1), ("b", 2), ("a", 4)]));+ let mut rest = common::consume_all(pipeline.as_ref()).await;+ rest.extend(+ first+ .unwrap()+ .into_iter()+ .filter_map(|r| r.result.into_value()),+ );+ assert_eq!(rest.len(), 7);+ let order_of = |key| {+ seen.lock()+ .unwrap()+ .iter()+ .filter(|e| e.0 == key)+ .map(|e| e.1)+ .collect::<Vec<_>>()+ };+ assert_eq!(order_of("a"), [1, 2, 3, 4]);+ assert_eq!(order_of("b"), [1, 2]);+}++#[tokio::test(start_paused = true)]+async fn grouping_failure_drains_completed_groups_then_fails_for_good() {+ let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()+ .concurrently_per_group(+ |e: &(&'static str, i64)| e.0,+ |g| {+ g.sequentially(|b| {+ b.pipe(step("maybeFails", |e: (&'static str, i64)| async move {+ if e.0 == "bad" {+ sleep_ms(10).await;+ anyhow::bail!("group failed");+ }+ sleep_ms(5).await;+ Ok(ok(e))+ }))+ })+ },+ None,+ )+ .build();+ // Only groups that completed before the failure drain ahead of it, as in Node.+ pipeline.feed(items([("bad", 1), ("good", 1)]));+ let good = pipeline.next().await.unwrap().unwrap();+ assert_eq!(good[0].value(), Some(&("good", 1)));+ assert_eq!(+ pipeline.next().await.unwrap_err().to_string(),+ "group failed"+ );+ assert_eq!(+ pipeline.next().await.unwrap_err().to_string(),+ "group failed"+ );+}++#[tokio::test]+async fn filter_map_emits_non_ok_passthroughs_before_sub_pipeline_results() {+ #[derive(Debug)]+ struct Ctx;+ let pipeline = new_chunk_pipeline::<i64, Ctx>()+ .pipe_chunk(chunk_step("split", |v: Vec<i64>| async move {+ Ok(v.into_iter()+ .map(|n| if n % 2 == 0 { dlq("even", None) } else { ok(n) })+ .collect())+ }))+ .filter_map(+ |el| el,+ |b| {+ b.pipe_chunk(chunk_step("double", |v: Vec<i64>| async move {+ Ok(v.into_iter().map(|n| ok(n * 2)).collect())+ }))+ },+ )+ .build();+ pipeline.feed(vec![+ create_ok_context(1, Ctx),+ create_ok_context(2, Ctx),+ create_ok_context(3, Ctx),+ ]);+ let first = pipeline.next().await.unwrap().unwrap();+ assert_eq!(first.len(), 1);+ assert_eq!(first[0].result.result_type(), PipelineResultType::Dlq);+ let second = pipeline.next().await.unwrap().unwrap();+ assert_eq!(+ second+ .iter()+ .map(|r| *r.value().unwrap())+ .collect::<Vec<_>>(),+ [2, 6]+ );+ assert!(pipeline.next().await.unwrap().is_none());+}++#[tokio::test]+async fn failed_dlq_produce_rejects_the_side_effect_only_when_configured() {+ for reject in [false, true] {+ let outputs = MemoryOutputs::new();+ outputs.fail_output(DLQ_OUTPUT);+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()+ .pipe_chunk(chunk_step("fail", |v: Vec<i64>| async move {+ Ok(v.into_iter().map(|_| dlq::<i64>("bad", None)).collect())+ }))+ .message_aware(|b| b)+ .handle_results(PipelineConfig {+ outputs: Arc::new(outputs.clone()),+ scheduler: Arc::new(scheduler.clone()),+ reject_on_dlq_failure: reject,+ })+ .handle_side_effects(+ Arc::new(scheduler.clone()),+ SideEffectHandlingConfig {+ await_side_effects: false,+ },+ )+ .build();+ pipeline.feed(vec![create_ok_context(+ 1,+ MsgCtx {+ message: test_message(None),+ },+ )]);+ pipeline.next().await.unwrap();+ assert_eq!(scheduler.wait_for_all().await.is_err(), reject);+ }+}++#[tokio::test]+async fn dlq_without_error_uses_the_reason_and_copies_source_headers() {+ let outputs = MemoryOutputs::new();+ let scheduler = PromiseScheduler::new();+ let pipeline = new_chunk_pipeline::<i64, MsgCtx>()+ .pipe_chunk(chunk_step("rejectStep", |v: Vec<i64>| async move {+ Ok(v.into_iter()+ .map(|_| dlq::<i64>("plain reason", None))+ .collect())+ }))+ .message_aware(|b| b)+ .handle_results(PipelineConfig {+ outputs: Arc::new(outputs.clone()),+ scheduler: Arc::new(scheduler.clone()),+ reject_on_dlq_failure: false,+ })+ .handle_side_effects(+ Arc::new(scheduler.clone()),+ SideEffectHandlingConfig {+ await_side_effects: true,+ },+ )+ .build();+ let mut message = test_message(Some(b"k"));+ message.topic = "events".into();+ message.partition = 3;+ message.offset = 42;+ message.headers = vec![+ ("token".into(), Some(b"phc_x".to_vec())),+ ("dlq_step".into(), Some(b"stale".to_vec())),+ ];+ pipeline.feed(vec![create_ok_context(1, MsgCtx { message })]);+ pipeline.next().await.unwrap();+ let produced = outputs.produced_to(DLQ_OUTPUT);+ let h = &produced[0].headers;+ assert_eq!(h.get("token"), Some("phc_x"));+ assert_eq!(h.get("dlq_reason"), Some("plain reason"));+ // A source header with the same name keeps its position and takes the new value, as a JS object spread does.+ assert_eq!(h.0[1], ("dlq_step".to_string(), "rejectStep".to_string()));+ assert_eq!(h.get("dlq_topic"), Some("events"));+ assert_eq!(h.get("dlq_partition"), Some("3"));+ assert_eq!(h.get("dlq_offset"), Some("42"));+}++#[tokio::test]+async fn warnings_on_a_timed_out_result_are_dropped_not_emitted() {+ let outputs = MemoryOutputs::new();+ let pipeline = new_chunk_pipeline::<i64, common::TeamCtx>()+ .pipe_chunk(chunk_step("warnThenTimeout", |v: Vec<i64>| async move {+ Ok(v.into_iter()+ .map(|n| {+ let r = if n == 1 { timeout::<i64>("cut") } else { ok(n) };+ r.with_warning(PipelineWarning::new(+ "client_ingestion_warning",+ serde_json::json!({}),+ ))+ })+ .collect())+ }))+ .team_aware(|b| b)+ .handle_ingestion_warnings(Arc::new(+ ingestion_framework::warning_handling::OutputsWarningEmitter {+ outputs: Arc::new(outputs.clone()),+ },+ ))+ .handle_side_effects(+ Arc::new(PromiseScheduler::new()),+ SideEffectHandlingConfig {+ await_side_effects: true,+ },+ )+ .build();+ pipeline.feed(vec![+ create_ok_context(1, common::TeamCtx { team_id: 1 }),+ create_ok_context(2, common::TeamCtx { team_id: 1 }),+ ]);+ let results = pipeline.next().await.unwrap().unwrap();+ assert!(results.iter().all(|r| r.context.warnings.is_empty()));+ assert_eq!(outputs.produced().len(), 1);+}++#[tokio::test]+async fn fan_out_dlq_sub_result_fails_the_parent_with_an_aggregated_reason() {+ let pipeline = new_chunk_pipeline::<Vec<i64>, ()>()+ .fan_out("split", |v: &Vec<i64>| v.clone())+ .via(|sub| {+ sub.sequentially(|b| {+ b.pipe(step("check", |n: i64| async move {+ Ok(if n < 0 { dlq("negative", None) } else { ok(n) })+ }))+ })+ })+ .fan_in("sum", |_v: Vec<i64>, parts: Vec<i64>| {+ parts.iter().sum::<i64>()+ })+ .build();+ pipeline.feed(items([vec![1, -1, -2]]));+ let results = pipeline.next().await.unwrap().unwrap();+ assert_eq!(+ results[0].result.reason(),+ Some("2/3 fan-out sub-elements dlq'd: negative")+ );+ assert_eq!(results[0].context.last_step(), Some("check"));+}++#[tokio::test]+async fn budget_flag_stays_set_after_expiry() {+ let budget =+ ingestion_framework::BatchBudget::soft_deadline(Instant::now() - Duration::from_millis(1));+ assert!(budget.exhausted());+ assert!(ingestion_framework::BatchBudget::unlimited().is_unlimited());+ assert!(!ingestion_framework::BatchBudget::unlimited().exhausted());+ assert_eq!(+ ingestion_framework::budget::budget_deadline(Instant::now(), 0),+ None+ );+ assert_eq!(+ ingestion_framework::budget::budget_deadline(Instant::now(), 1 << 31),+ None+ );+}++#[test]+fn compact_offset_ranges_matches_node() {+ use ingestion_framework::helpers::compact_offset_ranges;+ assert_eq!(compact_offset_ranges(&[1, 2, 3, 7, 10, 9]), "1-3,7,9-10");+ assert_eq!(compact_offset_ranges(&[5, 5, 5]), "5");+ assert_eq!(compact_offset_ranges(&[]), "");+}++#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn batching_with_concurrent_feeds_on_a_multi_thread_runtime_completes_every_batch() {+ use ingestion_framework::{+ before_batch_step, new_batching_pipeline, passthrough_after_batch_step,+ BatchingPipelineOptions, WithBatch,+ };+ let pipeline = Arc::new(new_batching_pipeline::<+ i64,+ WithBatch<i64, u64>,+ (),+ (),+ u64,+ u64,+ >(+ |b| b.pipe(before_batch_step("before", |id| id)),+ |b| {+ b.concurrently_per_group(+ |v: &WithBatch<i64, u64>| v.value % 7,+ |g| {+ g.sequentially(|s| {+ s.pipe(step("work", |v: WithBatch<i64, u64>| async move {+ tokio::task::yield_now().await;+ Ok(ok(v))+ }))+ })+ },+ Some(3),+ )+ },+ |b| b.pipe(passthrough_after_batch_step("after")),+ BatchingPipelineOptions {+ concurrent_batches: 8,+ },+ ));+ let mut feeders = Vec::new();+ for batch in 0..8u64 {+ let p = pipeline.clone();+ feeders.push(tokio::spawn(async move {+ loop {+ let r = p+ .feed(items((0..50).map(|i| i + batch as i64 * 1000)), batch, None)+ .await+ .unwrap();+ if r.is_ok() {+ break;+ }+ tokio::task::yield_now().await;+ }+ }));+ }+ for f in feeders {+ f.await.unwrap();+ }+ let mut seen = Vec::new();+ while let Some(done) = pipeline.next().await.unwrap() {+ assert_eq!(done.elements.len(), 50);+ assert_eq!(+ *done.batch_context, done.batch_id,+ "the batch context is the one built for this batch"+ );+ let values: Vec<i64> = done+ .elements+ .iter()+ .map(|e| e.value().unwrap().value)+ .collect();+ assert_eq!(+ values,+ (0..50)+ .map(|i| i + done.feed_context as i64 * 1000)+ .collect::<Vec<_>>(),+ "feed order kept in a batch"+ );+ seen.push(done.feed_context);+ }+ seen.sort();+ assert_eq!(seen, (0..8).collect::<Vec<_>>());+}