:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::future::Future;
use std::sync::{Arc, Mutex};
use futures::stream::{FuturesOrdered, StreamExt};
use crate::steps::{ChunkProcessingStep, ProcessingStep};
use crate::sync::FifoLimiter;
/// Sized up front, because above `JOIN_ALL_SMALL` `join_all` grows its output one push at a time.
async fn join_all_presized<F: Future>(runs: Vec<F>) -> Vec<F::Output> {
if runs.len() <= JOIN_ALL_SMALL {
return futures::future::join_all(runs).await;
}
let mut results = Vec::with_capacity(runs.len());
let mut ordered: FuturesOrdered<F> = runs.into_iter().collect();
while let Some(result) = ordered.next().await {
results.push(result);
}
results
}
/// Mirrors `futures_util::future::join_all::SMALL`.
const JOIN_ALL_SMALL: usize = 30;
/// Steps run inside one future, not as tasks, as Node runs them on one event loop. The first error fails the chunk after the rest end.
/// Panics on `Some(0)`, as Node's `pLimit(0)` throws.
pub fn per_element<T, U>(
step: ProcessingStep<T, U>,
max_concurrency: Option<usize>,
) -> ChunkProcessingStep<T, U>
where
T: Send + 'static,
U: Send + 'static,
{
let limit = max_concurrency.map(FifoLimiter::new);
let name = step.name().clone();
ChunkProcessingStep::new(&name, move |values: Vec<T>| {
let step = step.clone();
let limit = limit.clone();
let first_error: Arc<Mutex<Option<anyhow::Error>>> = Arc::default();
// Every slot is taken here, in input order, before any step starts.
let runs: Vec<_> = values
.into_iter()
.map(|value| {
let permit = limit.as_ref().map(|limit| limit.acquire());
let step = step.clone();
let first_error = first_error.clone();
async move {
let _permit = match permit {
Some(permit) => Some(permit.await),
None => None,
};
let result = step.call(value).await;
match result {
Ok(result) => Some(result),
Err(error) => {
first_error.lock().unwrap().get_or_insert(error);
None
}
}
}
})
.collect();
async move {
let results = join_all_presized(runs).await;
if let Some(error) = first_error.lock().unwrap().take() {
return Err(error);
}
Ok(results
.into_iter()
.map(|result| result.expect("no error was recorded"))
.collect())
}
})
}