:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! App layer: one person chunk's `stream-scan → eval → paced enqueue → mark → await → confirm`
//! pipeline. Depends on `clickhouse`, `kafka`, `store`, `domain`, and its `app` siblings.
//!
//! The scan is interleaved, never buffered: each row is evaluated and its seed enqueued before the
//! next row is read, so a chunk's memory stays constant. At stream end the chunk CASes
//! `scanning`→`produced` with the emitted count, then the remaining acks drain and the terminal
//! confirm runs — the behavioral post-mark machinery reused verbatim.
use std::sync::Arc;
use metrics::counter;
use tokio_util::sync::CancellationToken;
use cohort_core::hogvm::VmErrorClass;
use clickhouse::query::RowCursor;
use crate::clickhouse::person_scanner::{PersonRow, PersonScanError, PersonScanner};
use crate::clickhouse::person_sql::PersonScanSpec;
use crate::clickhouse::scan_volume::{self, ScanKind};
use crate::domain::{
CancelCause, ClaimedChunk, Halted, PersonChunkSpecError, PersonEmissionPolicy, PersonEvaluator,
PersonRowOutcome, PersonSeedContext, PinnedPersonRun, RecordStats, RetryBackoffPolicy,
StreamedChunk,
};
use crate::kafka::pacing::TilePacer;
use crate::kafka::producer::SeedTileProducer;
use crate::observability::metrics::{
MetricTimer, PERSONS_SCANNED, PERSON_CHUNK_SCAN_DURATION_SECONDS, PERSON_CONDITIONS_SHORTCUT,
PERSON_HOGVM_ERRORS, PERSON_NONMATCHERS_SKIPPED, PERSON_ROWS_PRUNED, PERSON_ROWS_SKIPPED,
PERSON_SCAN_FILTER, PERSON_SEEDS_PRODUCED, PRUNE_REASON_IRRELEVANT,
};
use crate::store::chunks::PgChunkStore;
use crate::store::lease::LeaseHandle;
use super::deliver::{self, InFlight, ProduceError, ProduceStop};
use super::execute::{mark_produced_halt, resolve_halt, ChunkOutcome, ConfirmedDetail};
use super::settings::ProducerSettings;
/// The owned inputs to one person chunk's processing task.
pub(super) struct PersonChunkTaskContext {
pub(super) chunk: ClaimedChunk,
pub(super) lease: LeaseHandle,
pub(super) run: Arc<PinnedPersonRun>,
pub(super) store: PgChunkStore,
pub(super) scanner: PersonScanner,
pub(super) producer: SeedTileProducer,
pub(super) pacer: TilePacer,
pub(super) producer_settings: ProducerSettings,
pub(super) emission: PersonEmissionPolicy,
pub(super) retry_backoff: RetryBackoffPolicy,
}
/// What one person chunk's fold came to, rendered on the `chunk confirmed` line so a run's shape is
/// legible without joining four counters.
#[derive(Debug, Default, Clone, Copy)]
pub(super) struct PersonChunkStats {
pub(super) persons_scanned: u64,
pub(super) nonmatchers: u64,
pub(super) pruned: u64,
pub(super) shortcut_evaluations: u64,
}
pub(super) async fn execute_person_chunk(
ctx: PersonChunkTaskContext,
shutdown: CancellationToken,
) -> ChunkOutcome {
let PersonChunkTaskContext {
chunk,
lease,
run,
store,
scanner,
producer,
pacer,
producer_settings,
emission,
retry_backoff,
} = ctx;
let lease_cancel = lease.cancellation_token();
// PreMark: stream the range scan, evaluating and enqueueing one seed per emitted person.
let (streamed, inflight, stats) = match stream_chunk(
chunk,
&run,
&scanner,
&producer,
&pacer,
producer_settings,
emission,
&lease_cancel,
&shutdown,
)
.await
{
Ok(triple) => triple,
Err(halt) => return resolve_halt(&store, halt, &shutdown, retry_backoff).await,
};
// PreMark: CAS `scanning`→`produced` — the row is still `scanning` on failure.
let enqueued = match store.mark_produced(streamed).await {
Ok(enqueued) => enqueued,
Err(halt) => {
return resolve_halt(&store, mark_produced_halt(halt), &shutdown, retry_backoff).await
}
};
// PostMark: drain the remaining delivery acks, then the terminal confirm.
let produced = match deliver::await_deliveries(enqueued, inflight, &lease_cancel).await {
Ok(produced) => produced,
Err(halt) => return resolve_halt(&store, halt, &shutdown, retry_backoff).await,
};
let lease = produced.spec().lease;
let seeds_produced = produced.tiles_produced();
match store.confirm(produced).await {
Ok(_) => ChunkOutcome::Confirmed {
lease,
tiles_produced: seeds_produced,
detail: ConfirmedDetail::Person(stats),
},
Err(halt) => resolve_halt(&store, halt, &shutdown, retry_backoff).await,
}
}
#[allow(clippy::too_many_arguments)]
async fn stream_chunk(
chunk: ClaimedChunk,
run: &Arc<PinnedPersonRun>,
scanner: &PersonScanner,
producer: &SeedTileProducer,
pacer: &TilePacer,
settings: ProducerSettings,
emission: PersonEmissionPolicy,
lease_cancel: &CancellationToken,
shutdown: &CancellationToken,
) -> Result<(StreamedChunk, InFlight, PersonChunkStats), Halted<ClaimedChunk, PersonChunkError>> {
let spec = match run.chunk_spec(&chunk.spec()) {
Ok(spec) => spec,
Err(error) => return Err(Halted::failed(chunk, PersonChunkError::Spec(error))),
};
let scan_spec = PersonScanSpec::new(
spec.team_id,
run.scan_since,
spec.range,
run.scan_key_filter(emission),
);
counter!(PERSON_SCAN_FILTER, "outcome" => scan_spec.filter_outcome().as_str()).increment(1);
let mut cursor = match scanner.scan_rows(&scan_spec, chunk.spec()) {
Ok(cursor) => cursor,
Err(error) => return Err(Halted::failed(chunk, PersonChunkError::Scan(error))),
};
let mut evaluator = PersonEvaluator::new(run, emission);
let seed_ctx = PersonSeedContext {
scanned_at: spec.scanned_at,
run_id: spec.lease.run_id(),
claim_epoch: spec.lease.epoch(),
};
let mut inflight = InFlight::new(settings.max_inflight.get());
let mut stats = PersonChunkStats::default();
let mut seeds_produced: u64 = 0;
// Started after setup so the spec/cursor early returns don't seed the histogram with
// near-zero samples.
let _timer = MetricTimer::start(PERSON_CHUNK_SCAN_DURATION_SECONDS);
// Every way out of the fold funnels back here, so the volume is metered once whether the scan
// finished, was cancelled, or failed mid-stream.
let folded = fold_person_rows(
&mut cursor,
&mut evaluator,
&seed_ctx,
producer,
&mut inflight,
pacer,
settings,
&mut stats,
&mut seeds_produced,
lease_cancel,
shutdown,
)
.await;
let volume = scan_volume::observe(ScanKind::Person, &cursor);
match folded {
Ok(()) => Ok((chunk.into_streamed(seeds_produced, volume), inflight, stats)),
Err(stop) => Err(stop.into_halt(chunk)),
}
}
/// Drive the person cursor into the evaluator and the paced producer until it is exhausted,
/// cancelled, or fails. Returns rather than metering, so the caller owns the single recording site.
#[allow(clippy::too_many_arguments)]
async fn fold_person_rows(
cursor: &mut RowCursor<PersonRow>,
evaluator: &mut PersonEvaluator,
seed_ctx: &PersonSeedContext,
producer: &SeedTileProducer,
inflight: &mut InFlight,
pacer: &TilePacer,
settings: ProducerSettings,
stats: &mut PersonChunkStats,
seeds_produced: &mut u64,
lease_cancel: &CancellationToken,
shutdown: &CancellationToken,
) -> Result<(), PersonStreamStop> {
loop {
let row = tokio::select! {
biased;
_ = shutdown.cancelled() => {
return Err(PersonStreamStop::Cancelled(CancelCause::Shutdown));
}
_ = lease_cancel.cancelled() => {
return Err(PersonStreamStop::Cancelled(CancelCause::LeaseLost));
}
row = cursor.next() => match row {
Ok(row) => row,
Err(error) => {
return Err(PersonStreamStop::Failed(PersonChunkError::Cursor(error)));
}
},
};
let Some(row) = row else {
return Ok(());
};
counter!(PERSONS_SCANNED).increment(1);
stats.persons_scanned += 1;
let (outcome, row_stats) = evaluator.evaluate_row(&row.id, &row.properties, seed_ctx);
stats.shortcut_evaluations += u64::from(row_stats.shortcut_evaluations);
record_row_stats(row_stats);
match outcome {
PersonRowOutcome::Seed(seed) => {
if let Err(stop) = deliver::enqueue_one(
|| producer.enqueue_person(&seed),
inflight,
pacer,
settings,
lease_cancel,
shutdown,
)
.await
{
return Err(stop.into());
}
*seeds_produced += 1;
counter!(PERSON_SEEDS_PRODUCED).increment(1);
}
PersonRowOutcome::NonMatcher => {
stats.nonmatchers += 1;
counter!(PERSON_NONMATCHERS_SKIPPED).increment(1);
}
PersonRowOutcome::Irrelevant => {
stats.pruned += 1;
counter!(PERSON_ROWS_PRUNED, "reason" => PRUNE_REASON_IRRELEVANT).increment(1);
}
PersonRowOutcome::Skipped(skip) => {
counter!(PERSON_ROWS_SKIPPED, "reason" => skip.as_str()).increment(1);
}
}
}
}
/// Why the person fold stopped, carried without the chunk so the caller can meter the cursor before
/// giving the chunk up to [`Halted`].
enum PersonStreamStop {
Cancelled(CancelCause),
Failed(PersonChunkError),
}
impl PersonStreamStop {
fn into_halt(self, chunk: ClaimedChunk) -> Halted<ClaimedChunk, PersonChunkError> {
match self {
Self::Cancelled(cause) => Halted::cancelled(chunk, cause),
Self::Failed(error) => Halted::failed(chunk, error),
}
}
}
impl From<ProduceStop> for PersonStreamStop {
fn from(stop: ProduceStop) -> Self {
match stop {
ProduceStop::Cancelled(cause) => Self::Cancelled(cause),
ProduceStop::Failed(error) => Self::Failed(PersonChunkError::Produce(error)),
}
}
}
fn record_row_stats(stats: RecordStats) {
if stats.shortcut_evaluations > 0 {
counter!(PERSON_CONDITIONS_SHORTCUT).increment(u64::from(stats.shortcut_evaluations));
}
if stats.unknown_functions > 0 {
counter!(PERSON_HOGVM_ERRORS, "class" => VmErrorClass::UnknownFunction.as_str())
.increment(u64::from(stats.unknown_functions));
}
for (class, count) in stats.vm_failures.iter().filter(|(_, count)| *count > 0) {
counter!(PERSON_HOGVM_ERRORS, "class" => class.as_str()).increment(u64::from(count));
}
}
#[derive(Debug, thiserror::Error)]
enum PersonChunkError {
#[error("resolving the person chunk spec")]
Spec(#[from] PersonChunkSpecError),
#[error(transparent)]
Scan(#[from] PersonScanError),
#[error("streaming ClickHouse person scan cursor")]
Cursor(#[source] clickhouse::error::Error),
#[error(transparent)]
Produce(ProduceError),
}
#[cfg(test)]
mod tests {
use super::*;
use crate::clickhouse::ResourceError;
/// `Scan` is transparent, so its source chain skips `PersonScanError` and must still reach the
/// ClickHouse error, or the person breaker never sees a resource error.
#[test]
fn person_scan_resource_errors_reach_the_breaker() {
let memory = || {
clickhouse::error::Error::BadResponse(
"Code: 241. DB::Exception: Memory limit (for query) exceeded. (MEMORY_LIMIT_EXCEEDED)"
.to_owned(),
)
};
for error in [
PersonChunkError::Scan(PersonScanError::Cursor(memory())),
PersonChunkError::Cursor(memory()),
] {
assert_eq!(
ResourceError::classify(&error),
Some(ResourceError::MemoryLimitExceeded),
"{error:?}"
);
}
}
}