:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
mod common;
use std::sync::Arc;
use common::{items, shared, sleep_ms};
use ingestion_framework::{
create_ok_context, new_chunk_pipeline, new_pipeline, ok, step, ChunkProcessingStep,
PromiseScheduler, SideEffect, SideEffectHandlingConfig,
};
async fn wait_all(side_effects: Vec<SideEffect>) {
for s in side_effects {
s.wait().await.unwrap();
}
}
#[tokio::test]
async fn steps_can_add_side_effects_to_context() {
let log = shared(Vec::<String>::new());
let l = log.clone();
let pipeline = new_pipeline::<String, ()>()
.pipe(step("processWithLoggingStep", move |input: String| {
let l = l.clone();
async move {
let result = input.to_uppercase();
let message = format!("Processed: {input} -> {result}");
let effect = SideEffect::spawn(async move {
l.lock().unwrap().push(message);
Ok(())
});
Ok(ok((result, true)).with_side_effect(effect))
}
}))
.build();
let mut r = pipeline
.process(create_ok_context("hello".into(), ()))
.await
.unwrap();
assert!(r.is_ok());
assert_eq!(r.context.side_effects.len(), 1);
// The side effect started when it was created, as a Node promise does.
assert!(log
.lock()
.unwrap()
.contains(&"Processed: hello -> HELLO".to_string()));
wait_all(std::mem::take(&mut r.context.side_effects)).await;
assert!(log
.lock()
.unwrap()
.contains(&"Processed: hello -> HELLO".to_string()));
}
#[tokio::test]
async fn side_effects_accumulate_through_the_pipeline() {
let audit = shared(Vec::<&'static str>::new());
let audited = |name: &'static str, f: fn(i64) -> i64| {
let a = audit.clone();
step(name, move |v: i64| {
let a = a.clone();
async move {
let effect = SideEffect::spawn(async move {
a.lock().unwrap().push(match name {
"step1" => "step1 completed",
"step2" => "step2 completed",
_ => "step3 completed",
});
Ok(())
});
Ok(ok(f(v)).with_side_effect(effect))
}
})
};
let pipeline = new_pipeline::<i64, ()>()
.pipe(audited("step1", |v| v * 2))
.pipe(audited("step2", |v| v + 10))
.pipe(audited("step3", |v| v * v))
.build();
let mut r = pipeline.process(create_ok_context(5, ())).await.unwrap();
assert_eq!(r.context.side_effects.len(), 3);
wait_all(std::mem::take(&mut r.context.side_effects)).await;
let audit = audit.lock().unwrap();
assert_eq!(audit.len(), 3);
for expected in ["step1 completed", "step2 completed", "step3 completed"] {
assert!(audit.contains(&expected));
}
}
#[tokio::test]
async fn a_step_can_add_multiple_side_effects() {
let sinks = shared((0, 0, 0));
let s = sinks.clone();
let pipeline = new_pipeline::<(&'static str, &'static str), ()>()
.pipe(step(
"processEventStep",
move |e: (&'static str, &'static str)| {
let (s1, s2, s3) = (s.clone(), s.clone(), s.clone());
async move {
Ok(ok((e, true))
.with_side_effect(SideEffect::spawn(async move {
s1.lock().unwrap().0 += 1;
Ok(())
}))
.with_side_effect(SideEffect::spawn(async move {
s2.lock().unwrap().1 += 1;
Ok(())
}))
.with_side_effect(SideEffect::spawn(async move {
s3.lock().unwrap().2 += 1;
Ok(())
})))
}
},
))
.build();
let mut r = pipeline
.process(create_ok_context(("purchase", "user-1"), ()))
.await
.unwrap();
assert_eq!(r.context.side_effects.len(), 3);
wait_all(std::mem::take(&mut r.context.side_effects)).await;
assert_eq!(*sinks.lock().unwrap(), (1, 1, 1));
}
fn delayed_effect_step(
completed: Arc<std::sync::Mutex<Vec<&'static str>>>,
delay: u64,
) -> ChunkProcessingStep<&'static str, &'static str> {
ChunkProcessingStep::per_item(step("processStep", move |id: &'static str| {
let completed = completed.clone();
async move {
let effect = SideEffect::spawn(async move {
sleep_ms(delay).await;
completed.lock().unwrap().push(id);
Ok(())
});
Ok(ok(id).with_side_effect(effect))
}
}))
}
#[tokio::test(start_paused = true)]
async fn handle_side_effects_with_await_true_blocks_until_completion() {
let completed = shared(Vec::new());
let scheduler = PromiseScheduler::new();
let pipeline = new_chunk_pipeline::<&'static str, ()>()
.pipe_chunk(delayed_effect_step(completed.clone(), 10))
.handle_side_effects(
Arc::new(scheduler),
SideEffectHandlingConfig {
await_side_effects: true,
},
)
.build();
pipeline.feed(items(["a", "b", "c"]));
pipeline.next().await.unwrap();
let mut done = completed.lock().unwrap().clone();
done.sort();
assert_eq!(done, ["a", "b", "c"]);
}
#[tokio::test(start_paused = true)]
async fn handle_side_effects_with_await_false_schedules_for_background_execution() {
let completed = shared(Vec::new());
let scheduler = PromiseScheduler::new();
let pipeline = new_chunk_pipeline::<&'static str, ()>()
.pipe_chunk(delayed_effect_step(completed.clone(), 50))
.handle_side_effects(
Arc::new(scheduler.clone()),
SideEffectHandlingConfig {
await_side_effects: false,
},
)
.build();
pipeline.feed(items(["x", "y"]));
pipeline.next().await.unwrap();
assert!(completed.lock().unwrap().is_empty());
scheduler.wait_for_all().await.unwrap();
let mut done = completed.lock().unwrap().clone();
done.sort();
assert_eq!(done, ["x", "y"]);
}