Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.5 KB
Raw Blame
mod common;

use common::{collect_chunks, items, shared};
use ingestion_framework::{chunk_step, dlq, new_chunk_pipeline, ok, step};

#[tokio::test]
async fn chunk_pipelines_process_multiple_items_at_once() {
    let calls = shared(0);
    let c = calls.clone();
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .pipe_chunk(chunk_step("chunkDoubleStep", move |items: Vec<i64>| {
            *c.lock().unwrap() += 1;
            async move { Ok(items.into_iter().map(|n| ok(n * 2)).collect()) }
        }))
        .build();
    pipeline.feed(items([1, 2, 3, 4, 5]));
    let results = pipeline.next().await.unwrap().unwrap();
    assert_eq!(*calls.lock().unwrap(), 1);
    let v: Vec<_> = results.iter().map(|r| *r.value().unwrap()).collect();
    assert_eq!(v, [2, 4, 6, 8, 10]);
}

#[tokio::test]
async fn feed_accepts_items_and_next_returns_results_then_null() {
    let pipeline = new_chunk_pipeline::<String, ()>()
        .pipe_chunk(chunk_step(
            "uppercaseStep",
            |items: Vec<String>| async move {
                Ok(items.into_iter().map(|s| ok(s.to_uppercase())).collect())
            },
        ))
        .build();
    pipeline.feed(items(["a", "b", "c"].map(String::from)));
    let v: Vec<_> = pipeline
        .next()
        .await
        .unwrap()
        .unwrap()
        .into_iter()
        .map(|r| r.result.into_value().unwrap())
        .collect();
    assert_eq!(v, ["A", "B", "C"]);
    assert!(pipeline.next().await.unwrap().is_none());
}

#[tokio::test]
async fn chunk_steps_must_return_same_number_of_results_as_inputs() {
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .pipe_chunk(chunk_step("validChunkStep", |items: Vec<i64>| async move {
            Ok(items.into_iter().map(|n| ok(n * 2)).collect())
        }))
        .build();
    pipeline.feed(items([1, 2, 3]));
    assert_eq!(pipeline.next().await.unwrap().unwrap().len(), 3);
}

#[tokio::test]
async fn mismatched_result_count_throws_an_error() {
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .pipe_chunk(chunk_step("badChunkStep", |items: Vec<i64>| async move {
            Ok(vec![ok(items[0])])
        }))
        .build();
    pipeline.feed(items([1, 2, 3]));
    let err = pipeline.next().await.unwrap_err();
    assert!(
        err.to_string()
            .contains("different number of results than input values"),
        "{err}"
    );
}

#[tokio::test]
async fn chunk_steps_receive_only_ok_items_from_previous_steps() {
    let received = shared(Vec::<i64>::new());
    let r = received.clone();
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .pipe_chunk(chunk_step("filterStep", |items: Vec<i64>| async move {
            Ok(items
                .into_iter()
                .map(|n| {
                    if n > 3 {
                        dlq(format!("Number {n} is too large"), None)
                    } else {
                        ok(n)
                    }
                })
                .collect())
        }))
        .pipe_chunk(chunk_step("processStep", move |items: Vec<i64>| {
            r.lock().unwrap().extend(items.iter().copied());
            async move { Ok(items.into_iter().map(|n| ok(n * 10)).collect()) }
        }))
        .build();
    pipeline.feed(items([1, 2, 3, 4, 5]));
    pipeline.next().await.unwrap();
    assert_eq!(*received.lock().unwrap(), vec![1, 2, 3]);
}

#[tokio::test]
async fn two_batches_emerge_as_a_single_chunk() {
    let lengths = shared(Vec::<usize>::new());
    let l = lengths.clone();
    let pipeline = new_chunk_pipeline::<i64, ()>()
        .pipe_chunk(chunk_step("recordingStep", move |items: Vec<i64>| {
            l.lock().unwrap().push(items.len());
            async move { Ok(items.into_iter().map(ok).collect()) }
        }))
        .build();
    pipeline.feed(items([1, 2]));
    pipeline.feed(items([3, 4, 5]));
    assert_eq!(pipeline.next().await.unwrap().unwrap().len(), 5);
    assert_eq!(*lengths.lock().unwrap(), vec![5]);
}

#[tokio::test]
async fn one_batch_spreads_across_multiple_chunks() {
    let pipeline = new_chunk_pipeline::<(&'static str, i64), ()>()
        .concurrently_per_group(
            |e: &(&'static str, i64)| e.0,
            |group| {
                group.sequentially(|b| b.pipe(step("identityStep", |e| async move { Ok(ok(e)) })))
            },
            None,
        )
        .build();
    pipeline.feed(items([("alice", 1), ("bob", 2), ("alice", 3), ("bob", 4)]));
    let chunks = collect_chunks(pipeline.as_ref()).await;
    assert_eq!(chunks.len(), 2);
    assert_eq!(chunks.iter().map(Vec::len).collect::<Vec<_>>(), [2, 2]);
}