Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

16.2 KB
Raw Blame
mod common;

use std::sync::Arc;
use std::time::Duration;

use common::{items, shared, sleep_ms, test_message, MsgCtx};
use ingestion_framework::{
    chunk_step, create_ok_context, dlq, new_chunk_pipeline, ok, step, timeout, MemoryOutputs,
    PipelineConfig, PipelineResultType, PipelineWarning, PromiseScheduler,
    SideEffectHandlingConfig, DLQ_OUTPUT,
};
use tokio::time::Instant;

fn failing_on(n: i64) -> ingestion_framework::ProcessingStep<i64, i64> {
    step("maybeFails", move |v: i64| async move {
        sleep_ms(10 * v as u64).await;
        if v == n {
            anyhow::bail!("boom {v}");
        }
        Ok(ok(v))
    })
}

#[tokio::test(start_paused = true)]
async fn concurrent_poison_drains_in_flight_items_in_order_then_fails_for_good() {
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .concurrently(|b| b.pipe(failing_on(2)), None)
        .build();
    pipeline.feed(items([1, 2, 3]));
    assert_eq!(pipeline.next().await.unwrap().unwrap()[0].value(), Some(&1));
    assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");
    assert_eq!(pipeline.next().await.unwrap().unwrap()[0].value(), Some(&3));
    assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");
    assert_eq!(pipeline.next().await.unwrap_err().to_string(), "boom 2");
}

#[tokio::test(start_paused = true)]
async fn concurrent_upstream_failure_poisons_the_stage() {
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .pipe_chunk(chunk_step("explodes", |_v: Vec<i64>| async move {
            Err::<Vec<ingestion_framework::PipelineResult<i64>>, _>(anyhow::anyhow!("upstream"))
        }))
        .concurrently(
            |b| b.pipe(step("id", |v: i64| async move { Ok(ok(v)) })),
            None,
        )
        .build();
    pipeline.feed(items([1]));
    assert_eq!(pipeline.next().await.unwrap_err().to_string(), "upstream");
    pipeline.feed(items([2]));
    // Poisoned: no new source pull, so the later feed never runs.
    assert_eq!(pipeline.next().await.unwrap_err().to_string(), "upstream");
}

#[tokio::test(start_paused = true)]
async fn concurrent_feed_while_parked_starts_new_input_without_waiting_for_the_head() {
    let starts = shared(Vec::<(i64, u128)>::new());
    let s = starts.clone();
    let origin = Instant::now();
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .concurrently(
            move |b| {
                b.pipe(step("slowHead", move |v: i64| {
                    s.lock().unwrap().push((v, origin.elapsed().as_millis()));
                    async move {
                        sleep_ms(if v == 1 { 100 } else { 1 }).await;
                        Ok(ok(v))
                    }
                }))
            },
            None,
        )
        .build();
    pipeline.feed(items([1]));
    let p = pipeline.clone();
    let consumer = tokio::spawn(async move { common::consume_all(p.as_ref()).await });
    sleep_ms(10).await;
    pipeline.feed(items([2]));
    let values = consumer.await.unwrap();
    assert_eq!(values, [1, 2]);
    let starts = starts.lock().unwrap().clone();
    assert_eq!(starts[1].0, 2);
    assert!(
        starts[1].1 < 100,
        "item 2 started at {}ms, behind the slow head",
        starts[1].1
    );
}

#[tokio::test(start_paused = true)]
async fn max_concurrency_admits_items_in_input_order() {
    let started = shared(Vec::<i64>::new());
    let s = started.clone();
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .concurrently(
            move |b| {
                b.pipe(step("record", move |v: i64| {
                    s.lock().unwrap().push(v);
                    async move {
                        sleep_ms(5).await;
                        Ok(ok(v))
                    }
                }))
            },
            Some(1),
        )
        .build();
    pipeline.feed(items(1..=20));
    let values = common::consume_all(pipeline.as_ref()).await;
    assert_eq!(values, (1..=20).collect::<Vec<_>>());
    assert_eq!(*started.lock().unwrap(), (1..=20).collect::<Vec<_>>());
}

#[tokio::test(start_paused = true)]
async fn grouping_keeps_per_key_order_across_next_calls_and_feeds() {
    let seen = shared(Vec::<(&'static str, i64)>::new());
    let s = seen.clone();
    let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()
        .concurrently_per_group(
            |e: &(&'static str, i64)| e.0,
            move |g| {
                g.sequentially(move |b| {
                    b.pipe(step("record", move |e: (&'static str, i64)| {
                        let s = s.clone();
                        async move {
                            sleep_ms(if e.1 % 2 == 0 { 3 } else { 7 }).await;
                            s.lock().unwrap().push(e);
                            Ok(ok(e))
                        }
                    }))
                })
            },
            Some(2),
        )
        .build();
    pipeline.feed(items([("a", 1), ("b", 1), ("a", 2)]));
    let first = pipeline.next().await.unwrap();
    pipeline.feed(items([("a", 3), ("c", 1), ("b", 2), ("a", 4)]));
    let mut rest = common::consume_all(pipeline.as_ref()).await;
    rest.extend(
        first
            .unwrap()
            .into_iter()
            .filter_map(|r| r.result.into_value()),
    );
    assert_eq!(rest.len(), 7);
    let order_of = |key| {
        seen.lock()
            .unwrap()
            .iter()
            .filter(|e| e.0 == key)
            .map(|e| e.1)
            .collect::<Vec<_>>()
    };
    assert_eq!(order_of("a"), [1, 2, 3, 4]);
    assert_eq!(order_of("b"), [1, 2]);
}

#[tokio::test(start_paused = true)]
async fn grouping_failure_drains_completed_groups_then_fails_for_good() {
    let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()
        .concurrently_per_group(
            |e: &(&'static str, i64)| e.0,
            |g| {
                g.sequentially(|b| {
                    b.pipe(step("maybeFails", |e: (&'static str, i64)| async move {
                        if e.0 == "bad" {
                            sleep_ms(10).await;
                            anyhow::bail!("group failed");
                        }
                        sleep_ms(5).await;
                        Ok(ok(e))
                    }))
                })
            },
            None,
        )
        .build();
    // Only groups that completed before the failure drain ahead of it, as in Node.
    pipeline.feed(items([("bad", 1), ("good", 1)]));
    let good = pipeline.next().await.unwrap().unwrap();
    assert_eq!(good[0].value(), Some(&("good", 1)));
    assert_eq!(
        pipeline.next().await.unwrap_err().to_string(),
        "group failed"
    );
    assert_eq!(
        pipeline.next().await.unwrap_err().to_string(),
        "group failed"
    );
}

#[tokio::test]
async fn filter_map_emits_non_ok_passthroughs_before_sub_pipeline_results() {
    #[derive(Debug)]
    struct Ctx;
    let pipeline = new_chunk_pipeline::<i64, Ctx>()
        .pipe_chunk(chunk_step("split", |v: Vec<i64>| async move {
            Ok(v.into_iter()
                .map(|n| if n % 2 == 0 { dlq("even", None) } else { ok(n) })
                .collect())
        }))
        .filter_map(
            |el| el,
            |b| {
                b.pipe_chunk(chunk_step("double", |v: Vec<i64>| async move {
                    Ok(v.into_iter().map(|n| ok(n * 2)).collect())
                }))
            },
        )
        .build();
    pipeline.feed(vec![
        create_ok_context(1, Ctx),
        create_ok_context(2, Ctx),
        create_ok_context(3, Ctx),
    ]);
    let first = pipeline.next().await.unwrap().unwrap();
    assert_eq!(first.len(), 1);
    assert_eq!(first[0].result.result_type(), PipelineResultType::Dlq);
    let second = pipeline.next().await.unwrap().unwrap();
    assert_eq!(
        second
            .iter()
            .map(|r| *r.value().unwrap())
            .collect::<Vec<_>>(),
        [2, 6]
    );
    assert!(pipeline.next().await.unwrap().is_none());
}

#[tokio::test]
async fn failed_dlq_produce_rejects_the_side_effect_only_when_configured() {
    for reject in [false, true] {
        let outputs = MemoryOutputs::new();
        outputs.fail_output(DLQ_OUTPUT);
        let scheduler = PromiseScheduler::new();
        let pipeline = new_chunk_pipeline::<i64, MsgCtx>()
            .pipe_chunk(chunk_step("fail", |v: Vec<i64>| async move {
                Ok(v.into_iter().map(|_| dlq::<i64>("bad", None)).collect())
            }))
            .message_aware(|b| b)
            .handle_results(PipelineConfig {
                outputs: Arc::new(outputs.clone()),
                scheduler: Arc::new(scheduler.clone()),
                reject_on_dlq_failure: reject,
            })
            .handle_side_effects(
                Arc::new(scheduler.clone()),
                SideEffectHandlingConfig {
                    await_side_effects: false,
                },
            )
            .build();
        pipeline.feed(vec![create_ok_context(
            1,
            MsgCtx {
                message: test_message(None),
            },
        )]);
        pipeline.next().await.unwrap();
        assert_eq!(scheduler.wait_for_all().await.is_err(), reject);
    }
}

#[tokio::test]
async fn dlq_without_error_uses_the_reason_and_copies_source_headers() {
    let outputs = MemoryOutputs::new();
    let scheduler = PromiseScheduler::new();
    let pipeline = new_chunk_pipeline::<i64, MsgCtx>()
        .pipe_chunk(chunk_step("rejectStep", |v: Vec<i64>| async move {
            Ok(v.into_iter()
                .map(|_| dlq::<i64>("plain reason", None))
                .collect())
        }))
        .message_aware(|b| b)
        .handle_results(PipelineConfig {
            outputs: Arc::new(outputs.clone()),
            scheduler: Arc::new(scheduler.clone()),
            reject_on_dlq_failure: false,
        })
        .handle_side_effects(
            Arc::new(scheduler.clone()),
            SideEffectHandlingConfig {
                await_side_effects: true,
            },
        )
        .build();
    let mut message = test_message(Some(b"k"));
    message.topic = "events".into();
    message.partition = 3;
    message.offset = 42;
    message.headers = vec![
        ("token".into(), Some(b"phc_x".to_vec())),
        ("dlq_step".into(), Some(b"stale".to_vec())),
    ];
    pipeline.feed(vec![create_ok_context(1, MsgCtx { message })]);
    pipeline.next().await.unwrap();
    let produced = outputs.produced_to(DLQ_OUTPUT);
    let h = &produced[0].headers;
    assert_eq!(h.get("token"), Some("phc_x"));
    assert_eq!(h.get("dlq_reason"), Some("plain reason"));
    // A source header with the same name keeps its position and takes the new value, as a JS object spread does.
    assert_eq!(h.0[1], ("dlq_step".to_string(), "rejectStep".to_string()));
    assert_eq!(h.get("dlq_topic"), Some("events"));
    assert_eq!(h.get("dlq_partition"), Some("3"));
    assert_eq!(h.get("dlq_offset"), Some("42"));
}

#[tokio::test]
async fn warnings_on_a_timed_out_result_are_dropped_not_emitted() {
    let outputs = MemoryOutputs::new();
    let pipeline = new_chunk_pipeline::<i64, common::TeamCtx>()
        .pipe_chunk(chunk_step("warnThenTimeout", |v: Vec<i64>| async move {
            Ok(v.into_iter()
                .map(|n| {
                    let r = if n == 1 { timeout::<i64>("cut") } else { ok(n) };
                    r.with_warning(PipelineWarning::new(
                        "client_ingestion_warning",
                        serde_json::json!({}),
                    ))
                })
                .collect())
        }))
        .team_aware(|b| b)
        .handle_ingestion_warnings(Arc::new(
            ingestion_framework::warning_handling::OutputsWarningEmitter {
                outputs: Arc::new(outputs.clone()),
            },
        ))
        .handle_side_effects(
            Arc::new(PromiseScheduler::new()),
            SideEffectHandlingConfig {
                await_side_effects: true,
            },
        )
        .build();
    pipeline.feed(vec![
        create_ok_context(1, common::TeamCtx { team_id: 1 }),
        create_ok_context(2, common::TeamCtx { team_id: 1 }),
    ]);
    let results = pipeline.next().await.unwrap().unwrap();
    assert!(results.iter().all(|r| r.context.warnings.is_empty()));
    assert_eq!(outputs.produced().len(), 1);
}

#[tokio::test]
async fn fan_out_dlq_sub_result_fails_the_parent_with_an_aggregated_reason() {
    let pipeline = new_chunk_pipeline::<Vec<i64>, ()>()
        .fan_out("split", |v: &Vec<i64>| v.clone())
        .via(|sub| {
            sub.sequentially(|b| {
                b.pipe(step("check", |n: i64| async move {
                    Ok(if n < 0 { dlq("negative", None) } else { ok(n) })
                }))
            })
        })
        .fan_in("sum", |_v: Vec<i64>, parts: Vec<i64>| {
            parts.iter().sum::<i64>()
        })
        .build();
    pipeline.feed(items([vec![1, -1, -2]]));
    let results = pipeline.next().await.unwrap().unwrap();
    assert_eq!(
        results[0].result.reason(),
        Some("2/3 fan-out sub-elements dlq'd: negative")
    );
    assert_eq!(results[0].context.last_step(), Some("check"));
}

#[tokio::test]
async fn budget_flag_stays_set_after_expiry() {
    let budget =
        ingestion_framework::BatchBudget::soft_deadline(Instant::now() - Duration::from_millis(1));
    assert!(budget.exhausted());
    assert!(ingestion_framework::BatchBudget::unlimited().is_unlimited());
    assert!(!ingestion_framework::BatchBudget::unlimited().exhausted());
    assert_eq!(
        ingestion_framework::budget::budget_deadline(Instant::now(), 0),
        None
    );
    assert_eq!(
        ingestion_framework::budget::budget_deadline(Instant::now(), 1 << 31),
        None
    );
}

#[test]
fn compact_offset_ranges_matches_node() {
    use ingestion_framework::helpers::compact_offset_ranges;
    assert_eq!(compact_offset_ranges(&[1, 2, 3, 7, 10, 9]), "1-3,7,9-10");
    assert_eq!(compact_offset_ranges(&[5, 5, 5]), "5");
    assert_eq!(compact_offset_ranges(&[]), "");
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn batching_with_concurrent_feeds_on_a_multi_thread_runtime_completes_every_batch() {
    use ingestion_framework::{
        before_batch_step, new_batching_pipeline, passthrough_after_batch_step,
        BatchingPipelineOptions, WithBatch,
    };
    let pipeline = Arc::new(new_batching_pipeline::<
        i64,
        WithBatch<i64, u64>,
        (),
        (),
        u64,
        u64,
    >(
        |b| b.pipe(before_batch_step("before", |id| id)),
        |b| {
            b.concurrently_per_group(
                |v: &WithBatch<i64, u64>| v.value % 7,
                |g| {
                    g.sequentially(|s| {
                        s.pipe(step("work", |v: WithBatch<i64, u64>| async move {
                            tokio::task::yield_now().await;
                            Ok(ok(v))
                        }))
                    })
                },
                Some(3),
            )
        },
        |b| b.pipe(passthrough_after_batch_step("after")),
        BatchingPipelineOptions {
            concurrent_batches: 8,
        },
    ));
    let mut feeders = Vec::new();
    for batch in 0..8u64 {
        let p = pipeline.clone();
        feeders.push(tokio::spawn(async move {
            loop {
                let r = p
                    .feed(items((0..50).map(|i| i + batch as i64 * 1000)), batch, None)
                    .await
                    .unwrap();
                if r.is_ok() {
                    break;
                }
                tokio::task::yield_now().await;
            }
        }));
    }
    for f in feeders {
        f.await.unwrap();
    }
    let mut seen = Vec::new();
    while let Some(done) = pipeline.next().await.unwrap() {
        assert_eq!(done.elements.len(), 50);
        assert_eq!(
            *done.batch_context, done.batch_id,
            "the batch context is the one built for this batch"
        );
        let values: Vec<i64> = done
            .elements
            .iter()
            .map(|e| e.value().unwrap().value)
            .collect();
        assert_eq!(
            values,
            (0..50)
                .map(|i| i + done.feed_context as i64 * 1000)
                .collect::<Vec<_>>(),
            "feed order kept in a batch"
        );
        seen.push(done.feed_context);
    }
    seen.sort();
    assert_eq!(seen, (0..8).collect::<Vec<_>>());
}