Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai-research-v4): log the session of dropped v4 data (#128184)

Nicholas Waltzd4764c4adcb9
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.test.ts
new file mode 100644
index 000000000000..9e8f02cac034
--- /dev/null
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.test.ts
@@ -0,0 +1,35 @@
+import { Message } from 'node-rdkafka'
+
+import { logger } from '~/common/utils/logger'
+
+import { INDEX_HEADER, WRITE_DAY_HEADER, parseIndexMessage } from './index-message'
+
+describe('parseIndexMessage', () => {
+ it.each([
+ [
+ 'names the team and session of a rejected row',
+ { team_id: 2, session_id: 'session-1' },
+ { teamId: 2, sessionId: 'session-1' },
+ ],
+ ['leaves out an oversized session ID', { team_id: 2, session_id: 's'.repeat(71) }, { teamId: 2 }],
+ ['leaves out identifiers of the wrong type', { team_id: '2', session_id: 1 }, {}],
+ ])('%s', (_, row, identity) => {
+ const warn = jest.spyOn(logger, 'warn').mockImplementation()
+ const message = {
+ partition: 0,
+ offset: 1,
+ value: Buffer.from(JSON.stringify(row)),
+ headers: [{ [INDEX_HEADER]: Buffer.from('blocks') }, { [WRITE_DAY_HEADER]: Buffer.from('2026-01-01') }],
+ } as unknown as Message
+
+ expect(parseIndexMessage(message)).toBeNull()
+
+ const logged = warn.mock.calls[0][2] as Record<string, unknown>
+ expect({ teamId: logged.teamId, sessionId: logged.sessionId }).toEqual({
+ teamId: undefined,
+ sessionId: undefined,
+ ...identity,
+ })
+ warn.mockRestore()
+ })
+})
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts
index 78553a6accd4..6545a0deda32 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-message.ts
@@ -13,6 +13,7 @@ import { V4ParquetSinkMetrics, V4RowRejection } from './metrics'
export const INDEX_HEADER = 'ml-index'
/** The UTC day of the objects the row points at, the `dt=` partition of its file. */
export const WRITE_DAY_HEADER = 'ml-write-day'
+const MAX_LOGGED_SESSION_ID_LENGTH = 70
export interface V4IndexMessage {
index: V4Index
@@ -43,13 +44,22 @@ function isWriteDay(value: string | undefined): value is string {
return !Number.isNaN(day.getTime()) && day.toISOString().slice(0, 10) === value
}
-function reject(reason: V4RowRejection, message: Message, invalidColumn?: string): null {
+function rowIdentity(value: unknown): { teamId?: number; sessionId?: string } {
+ const { team_id: teamId, session_id: sessionId } = (value ?? {}) as Record<string, unknown>
+ return {
+ ...(typeof teamId === 'number' && { teamId }),
+ ...(typeof sessionId === 'string' && sessionId.length <= MAX_LOGGED_SESSION_ID_LENGTH && { sessionId }),
+ }
+}
+
+function reject(reason: V4RowRejection, message: Message, invalidColumn?: string, value?: unknown): null {
// Skip a bad row rather than wedge the partition, as the v3 sink does.
logger.warn('🪶', 'ai_research_v4_parquet_row_rejected', {
reason,
partition: message.partition,
offset: message.offset,
...(invalidColumn && { column: invalidColumn }),
+ ...rowIdentity(value),
})
V4ParquetSinkMetrics.incRowsRejected(reason)
return null
@@ -74,7 +84,7 @@ export function parseIndexMessage(message: Message): V4IndexMessage | null {
const selected = selectIndexRow(index, value)
if (!('row' in selected)) {
V4ParquetSinkMetrics.incInvalidColumn(index, selected.invalidColumn)
- return reject('invalid_row', message, selected.invalidColumn)
+ return reject('invalid_row', message, selected.invalidColumn, value)
}
V4ParquetSinkMetrics.incRowsParsed(index)
return { index, writeDay, row: selected.row }
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index 02816142c108..d37789d69bc5 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -15,7 +15,7 @@ use replay_core_v4::inspect::stats::BlockStats;
use replay_core_v4::scan::cv::{Expansion, MESSAGE_DECODE_BUDGET};
use replay_core_v4::scan::message::{Event, Message};
-use crate::process::{check_read, read_within};
+use crate::process::{check_read, read_within, Skip};
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct SessionKey {
@@ -355,7 +355,7 @@ impl Inspectors {
let ((session, inspector, partitions, session_blocks), last_seen) = match result {
Ok(inspected) => inspected,
Err(error) => {
- tracing::error!(%error, team_id = key.team_id, "inspecting a session panicked: its block is dropped");
+ tracing::error!(%error, team_id = key.team_id, session_id = %key.session_id, "inspecting a session panicked: its block is dropped");
counter!("ai_research_v4_recorder_messages_total", "outcome" => "panicked")
.increment(messages as u64);
continue;
@@ -443,6 +443,7 @@ fn inspect_session(
) -> (SessionKey, SessionInspector, Vec<Tp>, Vec<Block>) {
let mut decoded = Vec::with_capacity(parts.len());
let mut session_spent = 0;
+ let mut skipped: BTreeMap<&'static str, u64> = BTreeMap::new();
for (origin, raws) in parts {
let captured_at_ms = raws.iter().filter_map(|raw| raw.captured_at_ms).min();
let mut messages = Vec::with_capacity(raws.len());
@@ -459,6 +460,7 @@ fn inspect_session(
) {
counter!("ai_research_v4_recorder_messages_total", "outcome" => "over_budget")
.increment(1);
+ *skipped.entry(Skip::OverBudget.as_str()).or_default() += 1;
continue;
}
// Decoded once here, for the inspector and the block writer both.
@@ -485,6 +487,9 @@ fn inspect_session(
Err(skip) => {
counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
.increment(1);
+ if !matches!(skip, Skip::NotWeb) {
+ *skipped.entry(skip.as_str()).or_default() += 1;
+ }
}
}
}
@@ -498,6 +503,9 @@ fn inspect_session(
.increment(messages.len() as u64);
decoded.push((origin, messages, captured_at_ms));
}
+ if !skipped.is_empty() {
+ tracing::warn!(team_id = session.team_id, session_id = %session.session_id, ?skipped, "messages of a session were skipped");
+ }
// A session can have messages on the main and the overflow topic in one flush. The inspector takes them in replay
// order, so that a mutation never comes before the nodes it changes. Each still counts toward its partition's block.
let mut inspections: Vec<BlockInspection> = decoded
diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rs
index 3777ef271205..991aeba15af8 100644
--- a/rust/ai-research/replay-recorder-v4/src/flush.rs
+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs
@@ -202,12 +202,13 @@ pub async fn write_flush(
month.to_owned(),
work.clone(),
);
- Some(seal_blocking(&writer.seal_permits, move || {
+ let seal = seal_blocking(&writer.seal_permits, move || {
let compressed = work.compressed(Arc::as_ptr(&block) as usize, || {
compress_block(&block, &key, &team_key, &month)
})?;
- seal_compressed(&compressed, &object_key).map(|sealed| (index, sealed))
- }))
+ seal_compressed(&compressed, &object_key)
+ });
+ Some(async move { (index, seal.await) })
})
.collect();
let mut done = Vec::with_capacity(seals.len());
@@ -217,12 +218,13 @@ pub async fn write_flush(
.collect::<Vec<_>>(),
mark_fresh_urls(writer, &urls, &day),
);
- for result in results {
+ for (index, result) in results {
match result {
- Ok(sealed) => done.push(sealed),
+ Ok(sealed) => done.push((index, sealed)),
// A panic in one block loses that block only. Its messages are committed with the rest of the flush.
Err(WriteError::Panicked(error)) => {
- tracing::error!(%error, "sealing a block panicked: the block is dropped");
+ let session = &blocks[index].session;
+ tracing::error!(%error, team_id = session.team_id, session_id = %session.session_id, "sealing a block panicked: the block is dropped");
counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => "panicked")
.increment(1);
}
@@ -230,10 +232,12 @@ pub async fn write_flush(
}
}
// Fail closed: a block that still holds a declined URL, a credential or a private host, is not stored.
- done.retain(|(_, block)| {
+ done.retain(|(index, block)| {
let Some(outcome) = block.counts.dropped else {
return true;
};
+ let session = &blocks[*index].session;
+ tracing::warn!(team_id = session.team_id, session_id = %session.session_id, outcome, "a block is not stored");
counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => outcome).increment(1);
false
});
diff --git a/rust/ai-research/replay-recorder-v4/src/limits.rs b/rust/ai-research/replay-recorder-v4/src/limits.rs
index 91bf67c0324b..b1026ced4239 100644
--- a/rust/ai-research/replay-recorder-v4/src/limits.rs
+++ b/rust/ai-research/replay-recorder-v4/src/limits.rs
@@ -284,6 +284,7 @@ impl SessionLimiter {
let outcome = if blocking { "blocked" } else { "counted" };
counter!("ai_research_v4_recorder_rate_limited_sessions_total", "outcome" => outcome)
.increment(1);
+ tracing::info!(team_id = session.team_id, session_id = %session.session_id, outcome, "the session limiter limited a session");
}
let prefix = if allowed || !blocking {
self.seen.insert(session.clone(), ());
diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rs
index b9a2556edbb0..b0824218165f 100644
--- a/rust/ai-research/replay-recorder-v4/src/write.rs
+++ b/rust/ai-research/replay-recorder-v4/src/write.rs
@@ -258,6 +258,7 @@ pub fn write_block<W: Write>(
if let Some(key) = &report.kept_in {
tracing::info!(
team_id = block.session.team_id,
+ session_id = %block.session.session_id,
key,
"a declined URL stayed inside a longer token"
);