:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai-research): record surfacing features per block in the v4 index (#125806)
Robbie22c3951a1bab
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.tsindex ea23f1a5109b..c68e6b861bf7 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts@@ -25,7 +25,7 @@ function blockRow(sessionId: string): Record<string, unknown> {const row: Record<string, unknown> = {}for (const column of BLOCK_INDEX_COLUMNS) {row[column.name] =- 'repeated' in column || 'list' in column+ 'repeated' in column || 'list' in column || 'elements' in column? []: column.optional? null@@ -214,6 +214,16 @@ describe('IndexBatcher', () => {'invalid_row',message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), window_ids: 'w1' }),],+ [+ 'click target id that is not an integer',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), surfacing_click_target_ids: [1.5] }),+ ],+ [+ 'visited url fingerprints that are not a list',+ 'invalid_row',+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), surfacing_visited_url_fingerprints: 'x' }),+ ],['missing urls_truncated flag','invalid_row',diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.tsindex a988a0e37395..facd1232a1d5 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts@@ -30,7 +30,13 @@ export interface V4ListColumn {list: ParquetType}-export type V4Column = V4LeafColumn | V4RepeatedColumn | V4ListColumn+/** A nullable list of non-null values, in the 3-level LIST layout. */+export interface V4NullableListColumn {+ name: string+ elements: ParquetType+}++export type V4Column = V4LeafColumn | V4RepeatedColumn | V4ListColumn | V4NullableListColumnconst required = (type: ParquetType, ...names: string[]): V4LeafColumn[] => names.map((name) => ({ name, type }))const optional = (type: ParquetType, ...names: string[]): V4LeafColumn[] =>@@ -108,6 +114,75 @@ export const BLOCK_INDEX_COLUMNS: readonly V4Column[] = [{ name: 'window_ids', list: 'UTF8' },{ name: 'urls', type: 'UTF8', optional: true, opaque: true },...required('BOOLEAN', 'urls_truncated'),+ // A recorder from before the surfacing columns sends rows without them, so they are nullable.+ ...optional(+ 'INT32',+ 'surfacing_event_count',+ 'surfacing_click_count',+ 'surfacing_keypress_count',+ 'surfacing_mouse_activity_count',+ 'surfacing_rage_click_count',+ 'surfacing_dead_click_count',+ 'surfacing_quick_back_count',+ 'surfacing_page_visit_count',+ 'surfacing_text_selection_count',+ 'surfacing_scroll_event_count',+ 'surfacing_console_error_count',+ 'surfacing_console_error_after_click_count',+ 'surfacing_console_warn_count',+ 'surfacing_network_request_count',+ 'surfacing_network_failed_request_count',+ 'surfacing_network_4xx_count',+ 'surfacing_network_5xx_count',+ 'surfacing_network_request_duration_count',+ 'surfacing_mouse_position_count',+ 'surfacing_mouse_direction_change_count',+ 'surfacing_mouse_velocity_count',+ 'surfacing_scroll_direction_reversal_count',+ 'surfacing_rapid_scroll_reversal_count',+ 'surfacing_scroll_to_top_count',+ 'surfacing_inter_action_gap_count',+ 'surfacing_long_idle_gap_count',+ 'surfacing_backspace_count',+ 'surfacing_mutation_count',+ 'surfacing_viewport_resize_count',+ 'surfacing_touch_event_count',+ 'surfacing_selection_copy_count',+ 'surfacing_login_path_visit_count',+ 'surfacing_signup_path_visit_count',+ 'surfacing_checkout_path_visit_count',+ 'surfacing_cart_path_visit_count',+ 'surfacing_billing_path_visit_count',+ 'surfacing_settings_path_visit_count',+ 'surfacing_account_path_visit_count',+ 'surfacing_error_path_visit_count',+ 'surfacing_not_found_path_visit_count',+ 'surfacing_admin_path_visit_count',+ 'surfacing_dashboard_path_visit_count',+ 'surfacing_onboarding_path_visit_count',+ 'surfacing_cancel_path_visit_count',+ 'surfacing_refund_path_visit_count'+ ),+ ...optional(+ 'INT64',+ 'surfacing_mouse_sum_x_millipx',+ 'surfacing_mouse_sum_x_squared_px2',+ 'surfacing_mouse_sum_y_millipx',+ 'surfacing_mouse_sum_y_squared_px2',+ 'surfacing_mouse_distance_traveled_millipx',+ 'surfacing_mouse_velocity_sum_px_per_s',+ 'surfacing_mouse_velocity_sum_of_squares_px2_per_s2',+ 'surfacing_total_scroll_magnitude_millipx',+ 'surfacing_inter_action_gap_sum_ms',+ 'surfacing_inter_action_gap_sum_of_squares_ms2',+ 'surfacing_network_request_duration_sum_us',+ 'surfacing_network_request_duration_sum_of_squares_ms2',+ 'surfacing_max_scroll_y_millipx',+ 'surfacing_max_idle_gap_ms'+ ),+ { name: 'surfacing_visited_url_fingerprints', elements: 'UTF8' },+ { name: 'surfacing_click_target_ids', elements: 'INT64' },+ { name: 'surfacing_form_field_ids', elements: 'INT64' },]export const INDEX_COLUMNS: Record<V4Index, readonly V4Column[]> = {@@ -164,6 +239,14 @@ function selectColumns(value: unknown, columns: readonly V4Column[], prefix = ''return invalid}row[column.name] = field+ } else if ('elements' in column) {+ if (field === null || field === undefined) {+ row[column.name] = null+ } else if (Array.isArray(field) && field.every((item) => isEncodableValue(column.elements, item))) {+ row[column.name] = field+ } else {+ return invalid+ }} else if (field === null || field === undefined) {if (!column.optional) {return invaliddiff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.tsindex 17c2535e1337..f04e04931009 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts@@ -129,6 +129,74 @@ describe('indexRowsToParquet', () => {'REPEATED BYTE_ARRAY window_ids UTF8','OPTIONAL BYTE_ARRAY urls UTF8','REQUIRED BOOLEAN urls_truncated',+ 'OPTIONAL INT32 surfacing_event_count',+ 'OPTIONAL INT32 surfacing_click_count',+ 'OPTIONAL INT32 surfacing_keypress_count',+ 'OPTIONAL INT32 surfacing_mouse_activity_count',+ 'OPTIONAL INT32 surfacing_rage_click_count',+ 'OPTIONAL INT32 surfacing_dead_click_count',+ 'OPTIONAL INT32 surfacing_quick_back_count',+ 'OPTIONAL INT32 surfacing_page_visit_count',+ 'OPTIONAL INT32 surfacing_text_selection_count',+ 'OPTIONAL INT32 surfacing_scroll_event_count',+ 'OPTIONAL INT32 surfacing_console_error_count',+ 'OPTIONAL INT32 surfacing_console_error_after_click_count',+ 'OPTIONAL INT32 surfacing_console_warn_count',+ 'OPTIONAL INT32 surfacing_network_request_count',+ 'OPTIONAL INT32 surfacing_network_failed_request_count',+ 'OPTIONAL INT32 surfacing_network_4xx_count',+ 'OPTIONAL INT32 surfacing_network_5xx_count',+ 'OPTIONAL INT32 surfacing_network_request_duration_count',+ 'OPTIONAL INT32 surfacing_mouse_position_count',+ 'OPTIONAL INT32 surfacing_mouse_direction_change_count',+ 'OPTIONAL INT32 surfacing_mouse_velocity_count',+ 'OPTIONAL INT32 surfacing_scroll_direction_reversal_count',+ 'OPTIONAL INT32 surfacing_rapid_scroll_reversal_count',+ 'OPTIONAL INT32 surfacing_scroll_to_top_count',+ 'OPTIONAL INT32 surfacing_inter_action_gap_count',+ 'OPTIONAL INT32 surfacing_long_idle_gap_count',+ 'OPTIONAL INT32 surfacing_backspace_count',+ 'OPTIONAL INT32 surfacing_mutation_count',+ 'OPTIONAL INT32 surfacing_viewport_resize_count',+ 'OPTIONAL INT32 surfacing_touch_event_count',+ 'OPTIONAL INT32 surfacing_selection_copy_count',+ 'OPTIONAL INT32 surfacing_login_path_visit_count',+ 'OPTIONAL INT32 surfacing_signup_path_visit_count',+ 'OPTIONAL INT32 surfacing_checkout_path_visit_count',+ 'OPTIONAL INT32 surfacing_cart_path_visit_count',+ 'OPTIONAL INT32 surfacing_billing_path_visit_count',+ 'OPTIONAL INT32 surfacing_settings_path_visit_count',+ 'OPTIONAL INT32 surfacing_account_path_visit_count',+ 'OPTIONAL INT32 surfacing_error_path_visit_count',+ 'OPTIONAL INT32 surfacing_not_found_path_visit_count',+ 'OPTIONAL INT32 surfacing_admin_path_visit_count',+ 'OPTIONAL INT32 surfacing_dashboard_path_visit_count',+ 'OPTIONAL INT32 surfacing_onboarding_path_visit_count',+ 'OPTIONAL INT32 surfacing_cancel_path_visit_count',+ 'OPTIONAL INT32 surfacing_refund_path_visit_count',+ 'OPTIONAL INT64 surfacing_mouse_sum_x_millipx',+ 'OPTIONAL INT64 surfacing_mouse_sum_x_squared_px2',+ 'OPTIONAL INT64 surfacing_mouse_sum_y_millipx',+ 'OPTIONAL INT64 surfacing_mouse_sum_y_squared_px2',+ 'OPTIONAL INT64 surfacing_mouse_distance_traveled_millipx',+ 'OPTIONAL INT64 surfacing_mouse_velocity_sum_px_per_s',+ 'OPTIONAL INT64 surfacing_mouse_velocity_sum_of_squares_px2_per_s2',+ 'OPTIONAL INT64 surfacing_total_scroll_magnitude_millipx',+ 'OPTIONAL INT64 surfacing_inter_action_gap_sum_ms',+ 'OPTIONAL INT64 surfacing_inter_action_gap_sum_of_squares_ms2',+ 'OPTIONAL INT64 surfacing_network_request_duration_sum_us',+ 'OPTIONAL INT64 surfacing_network_request_duration_sum_of_squares_ms2',+ 'OPTIONAL INT64 surfacing_max_scroll_y_millipx',+ 'OPTIONAL INT64 surfacing_max_idle_gap_ms',+ 'OPTIONAL group surfacing_visited_url_fingerprints LIST',+ 'REPEATED group surfacing_visited_url_fingerprints.list',+ 'REQUIRED BYTE_ARRAY surfacing_visited_url_fingerprints.list.element UTF8',+ 'OPTIONAL group surfacing_click_target_ids LIST',+ 'REPEATED group surfacing_click_target_ids.list',+ 'REQUIRED INT64 surfacing_click_target_ids.list.element',+ 'OPTIONAL group surfacing_form_field_ids LIST',+ 'REPEATED group surfacing_form_field_ids.list',+ 'REQUIRED INT64 surfacing_form_field_ids.list.element',],}@@ -164,7 +232,7 @@ describe('indexRowsToParquet', () => {? moments: 'list' in column? windowIds- : column.optional+ : 'elements' in column || column.optional? null: column.type === 'UTF8'? 'x'@@ -207,6 +275,41 @@ describe('indexRowsToParquet', () => {])})+ it('reads the surfacing lists back as 3-level lists, and a row without them as null', async () => {+ const surfacing = {+ surfacing_click_count: 3,+ surfacing_mouse_sum_x_millipx: -(2 ** 53 - 1),+ surfacing_visited_url_fingerprints: ['0f1e2d3c4b5a6978', 'b08517cee3ff5ecc'],+ surfacing_click_target_ids: [-1, 2 ** 53 - 1],+ surfacing_form_field_ids: [],+ }+ const { rows } = await describeFile(+ await indexRowsToParquet('blocks', [{ ...block([], []), ...surfacing }, block([], [])], 1_000)+ )+ const read = (rows as Record<string, unknown>[]).map((row) =>+ Object.fromEntries(Object.keys(surfacing).map((name) => [name, row[name] ?? null]))+ )+ expect(read).toEqual([+ {+ surfacing_click_count: 3,+ surfacing_mouse_sum_x_millipx: -(2n ** 53n - 1n),+ surfacing_visited_url_fingerprints: {+ list: surfacing.surfacing_visited_url_fingerprints.map((element) => ({ element })),+ },+ surfacing_click_target_ids: { list: [{ element: -1n }, { element: 2n ** 53n - 1n }] },+ // parquetjs reads an empty list as a null `list`; Arrow readers see an empty list.+ surfacing_form_field_ids: { list: null },+ },+ {+ surfacing_click_count: null,+ surfacing_mouse_sum_x_millipx: null,+ surfacing_visited_url_fingerprints: null,+ surfacing_click_target_ids: null,+ surfacing_form_field_ids: null,+ },+ ])+ })+it('reads moments back as a 3-level list of structs and the string lists as lists', async () => {const moments = [{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 - 1 },diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.tsindex b20557ff1542..77057e6588a4 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts@@ -29,6 +29,14 @@ function schemaDef(columns: readonly V4Column[]): SchemaDef {type: 'LIST',fields: { list: { repeated: true, fields: { element: { fields: schemaDef(column.fields) } } } },}+ } else if ('elements' in column) {+ def[column.name] = {+ type: 'LIST',+ optional: true,+ fields: {+ list: { repeated: true, fields: { element: { type: column.elements, compression: 'ZSTD' } } },+ },+ }} else if ('list' in column) {def[column.name] = { type: column.list, repeated: true, compression: 'ZSTD' }} else {@@ -80,6 +88,11 @@ function toRecord(row: V4IndexRow, columns: readonly V4Column[]): Record<string,}} else if ('list' in column) {record[column.name] = (value as unknown[]).map((item) => toValue(column.list, item))+ } else if ('elements' in column) {+ record[column.name] =+ value === null || value === undefined+ ? null+ : { list: (value as unknown[]).map((item) => ({ element: toValue(column.elements, item) })) }} else if (value === null || value === undefined) {record[column.name] = null} else {diff --git a/products/ai_training/docs/replay-data.md b/products/ai_training/docs/replay-data.mdindex 2f5a04a69ac9..93a477ddb78f 100644--- a/products/ai_training/docs/replay-data.md+++ b/products/ai_training/docs/replay-data.md@@ -498,7 +498,7 @@ The recorder does not write the two index prefixes itself, as in v3:- After a flush stores its packs and blocks, the recorder sends one JSON record per index row to `ai_research_v4_index`. The record's keys are the Parquet column names.- The recorder commits without waiting for delivery. A row that is not delivered leaves its block or image stored but not in the index.- The v4 Parquet sink reads the topic, groups the rows by index and write day, and writes one file per group and flush as `{ms}-{random}.parquet`. A flush runs every 10 minutes, or at 128 MiB of rows or 1,000,000 rows. The sink commits its offsets only after the files are stored.-- Index files use four column types: `INT32`, `INT64`, `UTF8` and `BOOLEAN`. Integers are signed. The recorder caps an `INT32` value at the type's range and an `INT64` value at ±(2^53 − 1), so every value is a safe JavaScript integer. It ignores an event timestamp outside the JavaScript date range. Byte counts, offsets and times are `INT64`, the other counts `INT32`. `moments` is a 3-level `LIST` (`moments`, then the repeated group `list`, then the struct `element`), so Athena, Arrow readers and DuckDB see a non-null list of structs. Files written before the sink used that layout hold `moments` as a bare repeated group whose item is named `moments`. Arrow readers and DuckDB read those as the same list of structs, but Athena reads the column as null, because Trino reads a bare repeated group of more than one field as a missing column. Those files expire with the bucket's lifecycle. `url_fingerprints` and `window_ids` are lists of `UTF8` with no null items. Row groups hold 131072 rows, and every column is zstd-compressed at level 9. Older files can still be snappy-compressed and in arrival order until they expire; Parquet records the codec per column chunk, so a reader handles both. The sink writes no statistics for `urls`, because a sealed value has no useful order.+- Index files use four column types: `INT32`, `INT64`, `UTF8` and `BOOLEAN`. Integers are signed. The recorder caps an `INT32` value at the type's range and an `INT64` value at ±(2^53 − 1), so every value is a safe JavaScript integer. It ignores an event timestamp outside the JavaScript date range. Byte counts, offsets and times are `INT64`, the other counts `INT32`. `moments` is a 3-level `LIST` (`moments`, then the repeated group `list`, then the struct `element`), so Athena, Arrow readers and DuckDB see a non-null list of structs. Files written before the sink used that layout hold `moments` as a bare repeated group whose item is named `moments`. Arrow readers and DuckDB read those as the same list of structs, but Athena reads the column as null, because Trino reads a bare repeated group of more than one field as a missing column. Those files expire with the bucket's lifecycle. `url_fingerprints` and `window_ids` are lists of `UTF8` with no null items. The three [surfacing](#surfacing-features) sets are nullable 3-level `LIST`s of `UTF8` or `INT64` with no null items. Row groups hold 131072 rows, and every column is zstd-compressed at level 9. Older files can still be snappy-compressed and in arrival order until they expire; Parquet records the codec per column chunk, so a reader handles both. The sink writes no statistics for `urls`, because a sealed value has no useful order.- The sink can write a row twice after a restart. The copies have the same `flush_id`, so the rule below does not remove them: readers first drop rows that repeat within one `flush_id`.### Block index@@ -531,6 +531,7 @@ A session row is a fold over its block rows: sort them by `first_ts_ms`, then meCounts sum, times take the minimum or maximum, flags take the logical or, `host` takes the first value and the viewport takes the one with the highest `viewport_ts_ms`.The DOM sizes `dom_nodes` and `dom_text_chars` and `max_written_event_bytes` take the maximum, because they describe a state or a single event, not an amount.`url_fingerprints` and `window_ids` take the ordered union, capped at 10 and 32 entries. A session at the cap has that many or more.+The [surfacing](#surfacing-features) sums add, its two maxima take the maximum, and its three sets take the union with no cap.The flush columns describe a row, not a session, so the fold does not carry them.`events_before_first_full_snapshot` sums, because the recorder stops counting it once the session has a decoded full snapshot.`max_gap_ms` also counts the gap between one block's last event and the next block's first event. When blocks overlap, for example the main and overflow topics, it is an upper bound.@@ -553,6 +554,7 @@ This holds for the blocks that one recorder process inspected. After a rebalance| Images | `images_extracted`, `image_bytes`, `images_inline`, `image_urls` || Moments | `moments`: a list of `{kind, window_id, ts_ms, event_index}` for full snapshots, navigations, clicks and large DOM changes, and `moments_truncated` || Sealed | `urls`: the visited URLs as a sealed index value, `urls_truncated`, and `url_fingerprints` |+| Surfacing | The `surfacing_` columns: what ingestion's surfacing feature recorder records for the block as one batch. See [Surfacing features](#surfacing-features) |- `full_snapshots` counts only snapshots that decoded.- `events_before_first_full_snapshot` counts the session's events before its first decoded full snapshot.@@ -568,6 +570,29 @@ This holds for the blocks that one recorder process inspected. After a rebalance- `url_fingerprints` holds up to 10 distinct page URLs of the block as the first 16 hexadecimal characters of an HMAC-SHA256 of the URL. The key is derived from the session key with HKDF-SHA256, empty salt and info `ml-url-fingerprint-key`, so fingerprints can be compared only within one session. They count distinct URLs, where `navigations` counts URL changes.- `cold_start` is true when the recorder first sees a session after it started, for example after a partition rebalance, or after it dropped the session's DOM state to stay within its memory budget. Its playability columns then describe only the part of the session the recorder saw.+#### Surfacing features++The `surfacing_` columns let a sampler score whole sessions with the surfacing booster without opening a block.+Each block row holds what ingestion's `SessionFeatureRecorder` (`session-feature-recorder.ts`) records for one batch, and the recorder computes it with the same definitions.+Its state starts again at each block, as ingestion's does at each batch: a block is one session's events from one Kafka partition in one recorder flush, as a batch is.+After the prefix, each column has the name of the `session_replay_features` column it matches, with a unit for the sums that are not counts.+A session's values are the fold of its block rows, as `_AGGREGATED_STATS_FRAGMENT` (`posthog/temporal/session_replay/surfacing_scoring_sweep/sql.py`) folds the batch rows, and `fold_session` in `replay-core-v4` implements it.++| Kind | Columns |+| ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |+| Counts | `INT32`: `surfacing_{name}_count` for the 31 counters of `session_replay_features`, from `event_count` to `selection_copy_count`, and `surfacing_{token}_path_visit_count` for the 14 path tokens, where `404` is `not_found` |+| Sums | `INT64`: `surfacing_mouse_sum_x_millipx`, `surfacing_mouse_sum_y_millipx`, `surfacing_mouse_sum_x_squared_px2`, `surfacing_mouse_sum_y_squared_px2`, `surfacing_mouse_distance_traveled_millipx`, `surfacing_mouse_velocity_sum_px_per_s`, `surfacing_mouse_velocity_sum_of_squares_px2_per_s2`, `surfacing_total_scroll_magnitude_millipx`, `surfacing_inter_action_gap_sum_ms`, `surfacing_inter_action_gap_sum_of_squares_ms2`, `surfacing_network_request_duration_sum_us`, `surfacing_network_request_duration_sum_of_squares_ms2` |+| Maxima | `INT64`: `surfacing_max_scroll_y_millipx`, `surfacing_max_idle_gap_ms` |+| Sets | `surfacing_visited_url_fingerprints` (`UTF8`), `surfacing_click_target_ids` and `surfacing_form_field_ids` (`INT64`) |++- Ingestion stores a float sum as a double. The index stores it as an integer in the unit at the end of its name, rounded once per block, so that a fold is exact in any order. `millipx` is 1/1000 of a pixel and `us` is a microsecond. Ingestion's mouse velocity is in pixels per millisecond, so `surfacing_mouse_velocity_sum_px_per_s` is 1000 times its sum and `surfacing_mouse_velocity_sum_of_squares_px2_per_s2` 1000000 times its sum of squares.+- The sets hold distinct values, sorted. The click targets and form fields are rrweb node IDs. The visited pages are fingerprints in the format of `url_fingerprints`, under the same key: ingestion stores an md5 of each URL instead. Each set keeps the first 1000 values of a block, as ingestion keeps 1000 per batch. The sweep counts a set with `uniqCombinedMerge(12)`, and the length of the folded set is the exact count.+- A page URL is the trimmed `data.href`, or else `data.payload.href`, of any event, as `hrefFrom` in `rrweb-types.ts` reads it. The path token counts compare the segments of its path with a fixed list of words, so they hold no other part of a URL.+- The recorder skips an event without a positive timestamp, which ingestion's parse step drops before its recorder. The booster's rates divide by the session duration, which a sampler takes from `first_ts_ms` and `last_ts_ms`. Those also count events with a timestamp of zero or less.+- The columns are null in a row from a recorder that predates them, and absent from files written before them. A sampler falls back to the session's live score for such a session.+- Ingestion corrects the event times of a message by the sender's clock skew, rounded to 5 minutes, and the recorder does not. A block whose messages got different corrections can therefore differ in its time-based values.+- A field of an unexpected JSON type counts as missing, where JavaScript can coerce it or produce NaN, for example a mouse position without coordinates. Ingestion's parse step lets such fields through.+#### Watermark filesAfter a flush, the sink writes one JSON file per index and Kafka partition of `ai_research_v4_index`, so a reader can learn from the bucket how far the index is complete:diff --git a/rust/ai-research/replay-core-v4/src/bytescan.rs b/rust/ai-research/replay-core-v4/src/bytescan.rsindex 5731f63fb5a2..2479dbd14411 100644--- a/rust/ai-research/replay-core-v4/src/bytescan.rs+++ b/rust/ai-research/replay-core-v4/src/bytescan.rs@@ -351,6 +351,108 @@ impl<'b> Iterator for ArrayItems<'b> {}}+/// The top-level `keys` of each object item of the array that starts at `start`, found in the same pass that finds+/// where the item ends, so that an array of small objects is scanned once. Items that are not objects are skipped. As+/// in a parse, the last of a repeated key wins, and an escaped key is not one of `keys`.+pub fn array_object_fields<'b, const N: usize>(+ bytes: &'b [u8],+ start: usize,+ keys: [&'b [u8]; N],+) -> Result<ArrayObjectFields<'b, N>> {+ if bytes.get(start) != Some(&b'[') {+ return Err(ScanError("not an array"));+ }+ Ok(ArrayObjectFields {+ bytes,+ keys,+ pos: start + 1,+ done: false,+ first: true,+ })+}++pub struct ArrayObjectFields<'b, const N: usize> {+ bytes: &'b [u8],+ keys: [&'b [u8]; N],+ pos: usize,+ done: bool,+ first: bool,+}++impl<const N: usize> ArrayObjectFields<'_, N> {+ /// Reads the object at `pos`, and returns its end and its fields.+ fn object(&self, mut pos: usize) -> Result<(usize, [Option<Span>; N])> {+ let bytes = self.bytes;+ let mut found = [None; N];+ pos = skip_ws(bytes, pos + 1);+ if bytes.get(pos) == Some(&b'}') {+ return Ok((pos + 1, found));+ }+ loop {+ if bytes.get(pos) != Some(&b'"') {+ return Err(ScanError("expected object key"));+ }+ let key_end = skip_string(bytes, pos)?;+ let key = &bytes[pos + 1..key_end - 1];+ pos = skip_ws(bytes, key_end);+ if bytes.get(pos) != Some(&b':') {+ return Err(ScanError("expected ':' after object key"));+ }+ let value = locate_value(bytes, pos + 1)?;+ if let Some(index) = self.keys.iter().position(|&k| k == key) {+ found[index] = Some(value);+ }+ pos = skip_ws(bytes, value.1);+ match bytes.get(pos) {+ Some(b',') => pos = skip_ws(bytes, pos + 1),+ Some(b'}') => return Ok((pos + 1, found)),+ _ => return Err(ScanError("expected ',' between object entries")),+ }+ }+ }+}++impl<const N: usize> Iterator for ArrayObjectFields<'_, N> {+ type Item = Result<[Option<Span>; N]>;++ fn next(&mut self) -> Option<Self::Item> {+ while !self.done {+ let bytes = self.bytes;+ let mut pos = skip_ws(bytes, self.pos);+ if bytes.get(pos) == Some(&b']') {+ self.done = true;+ return None;+ }+ if !self.first {+ if bytes.get(pos) != Some(&b',') {+ self.done = true;+ return Some(Err(ScanError("expected ',' between array items")));+ }+ pos = skip_ws(bytes, pos + 1);+ }+ self.first = false;+ let item = if bytes.get(pos) == Some(&b'{') {+ self.object(pos).map(|(end, found)| (end, Some(found)))+ } else {+ locate_value(bytes, pos).map(|span| (span.1, None))+ };+ match item {+ Ok((end, found)) => {+ self.pos = end;+ if let Some(found) = found {+ return Some(Ok(found));+ }+ }+ Err(error) => {+ self.done = true;+ return Some(Err(error));+ }+ }+ }+ None+ }+}+/// Parse a scanned scalar span as a JSON number, mirroring what `JSON.parse` would produce (an f64)./// Returns `None` for non-numbers. Accepts the full JSON number grammar (fractions, exponents); the/// charset pre-check keeps Rust-isms like `inf`/`NaN` out.@@ -360,6 +462,19 @@ pub fn parse_number(bytes: &[u8], span: Span) -> Option<f64> {if first != b'-' && !first.is_ascii_digit() {return None;}+ // Most numbers in a recording are small integers, which an f64 holds exactly, so they skip the float parser.+ let digits = raw.strip_prefix(b"-").unwrap_or(raw);+ if !digits.is_empty() && digits.len() <= 15 && digits.iter().all(u8::is_ascii_digit) {+ let value = digits+ .iter()+ .fold(0i64, |value, digit| value * 10 + i64::from(digit - b'0'))+ as f64;+ return Some(if digits.len() < raw.len() {+ -value+ } else {+ value+ });+ }if !raw.iter().all(|b| b.is_ascii_digit() || matches!(b, b'-' | b'+' | b'.' | b'e' | b'E'))diff --git a/rust/ai-research/replay-core-v4/src/inspect/clicks.rs b/rust/ai-research/replay-core-v4/src/inspect/clicks.rsindex 1c87f5f5cab3..c01649a42811 100644--- a/rust/ai-research/replay-core-v4/src/inspect/clicks.rs+++ b/rust/ai-research/replay-core-v4/src/inspect/clicks.rs@@ -41,7 +41,9 @@ impl ClickTracker {self.run = 1;return ClickKind::Plain;};- if ts - last_ts < RUN_WINDOW_MS && (x - last_x).hypot(y - last_y) < RUN_DISTANCE_PX {+ let (dx, dy) = (x - last_x, y - last_y);+ // `Math.sqrt(dx * dx + dy * dy)`, which can round differently from `hypot`.+ if ts - last_ts < RUN_WINDOW_MS && (dx * dx + dy * dy).sqrt() < RUN_DISTANCE_PX {self.run += 1;return if self.run >= RAGE_RUN_LENGTH {ClickKind::Ragediff --git a/rust/ai-research/replay-core-v4/src/inspect/events.rs b/rust/ai-research/replay-core-v4/src/inspect/events.rsindex b5066041ea16..120996c19b38 100644--- a/rust/ai-research/replay-core-v4/src/inspect/events.rs+++ b/rust/ai-research/replay-core-v4/src/inspect/events.rs@@ -1,10 +1,12 @@//! Event-level counts for one block: activity, interactions, navigation, console and network.+use std::borrow::Cow;use std::collections::BTreeSet;use std::ops::Bound;use crate::bytescan::{self as scan, Span};+use super::features::js_trim;use super::rrweb::*;use super::stats::BlockStats;use crate::scan::message::Event;@@ -14,7 +16,8 @@ const ACTIVITY_THRESHOLD_MS: f64 = 5000.0;/// The distinct times an open inactive segment keeps. Past this the oldest go, so a late active event older than all/// that remain changes nothing. Out-of-order input then costs a log-time insert, and the state stays small.const MAX_INACTIVE_TIMES: usize = 1024;-/// The status field of a `posthog/network@1` payload, as `session-feature-recorder.ts` reads it.+/// The duration and status fields of a `posthog/network@1` payload, as `session-feature-recorder.ts` reads them.+const POSTHOG_NETWORK_DURATION_KEY: &[u8] = b"39";const POSTHOG_NETWORK_STATUS_KEY: &[u8] = b"21";/// The longest DNS name. A longer host resolves nowhere, so it names no site.const MAX_HOST_BYTES: usize = 253;@@ -73,11 +76,129 @@ pub(crate) fn number(bytes: &[u8], span: Option<Span>) -> Option<f64> {span.and_then(|span| scan::parse_number(bytes, span))}-pub(crate) fn text<'b>(bytes: &'b [u8], span: Option<Span>) -> Option<std::borrow::Cow<'b, str>> {+pub(crate) fn text<'b>(bytes: &'b [u8], span: Option<Span>) -> Option<Cow<'b, str>> {span.filter(|&span| scan::is_string(bytes, span)).and_then(|span| scan::unescape(bytes, span).ok())}+/// One request of an `rrweb/network@1` event, as `session-feature-recorder.ts` reads it.+#[derive(Debug, Clone, Copy)]+pub struct NetworkRequest {+ pub duration: Option<f64>,+ /// `req.status ?? req.responseStatus`+ pub status: Option<f64>,+}++/// The fields of an event's `data`, and of `data.payload`, that the counters and the feature recorder read. Each object+/// is read once per event, for all of them.+#[derive(Debug, Clone, Copy, Default)]+pub struct DataFields {+ /// `type`: the kind of a mouse interaction.+ pub interaction: Option<Span>,+ pub x: Option<Span>,+ pub y: Option<Span>,+ pub id: Option<Span>,+ pub text: Option<Span>,+ pub plugin: Option<Span>,+ pub payload: Option<Span>,+ pub href: Option<Span>,+ pub tag: Option<Span>,+ pub width: Option<Span>,+ pub height: Option<Span>,+ pub payload_href: Option<Span>,+ pub level: Option<Span>,+ pub network_duration: Option<Span>,+ pub network_status: Option<Span>,+ /// The requests of an `rrweb/network@1` event, which [`DataFields::for_each_network_request`] reads.+ network_requests: Option<Span>,+}++impl DataFields {+ /// Reads the fields of the events that use them. A snapshot, a mutation or a mouse move's positions are large and+ /// use none of them, so their `data` is read only when it has a key where `hrefFrom` looks for a page.+ pub fn read(inner: &[u8], event: &Event) -> Self {+ let Some(data) = event.data else {+ return Self::default();+ };+ let used = match event.kind {+ Some(META | CUSTOM | PLUGIN) => true,+ Some(INCREMENTAL_SNAPSHOT) => matches!(+ event.source,+ Some(MOUSE_INTERACTION | SCROLL | VIEWPORT_RESIZE | INPUT)+ ),+ _ => false,+ };+ if !used && !event.may_carry_page_url {+ return Self::default();+ }+ let [interaction, x, y, id, text, plugin, payload, href, tag, width, height] = fields(+ inner,+ data,+ [+ b"type", b"x", b"y", b"id", b"text", b"plugin", b"payload", b"href", b"tag",+ b"width", b"height",+ ],+ );+ let [payload_href, level, network_requests, network_duration, network_status] = payload+ .map(|payload| {+ fields(+ inner,+ payload,+ [+ b"href",+ b"level",+ b"requests",+ POSTHOG_NETWORK_DURATION_KEY,+ POSTHOG_NETWORK_STATUS_KEY,+ ],+ )+ })+ .unwrap_or_default();+ let network_plugin = event.kind == Some(PLUGIN)+ && self::text(inner, plugin).as_deref() == Some("rrweb/network@1");+ Self {+ interaction,+ x,+ y,+ id,+ text,+ plugin,+ payload,+ href,+ tag,+ width,+ height,+ payload_href,+ level,+ network_duration,+ network_status,+ network_requests: network_requests.filter(|_| network_plugin),+ }+ }++ /// Calls `each` with every request of an `rrweb/network@1` event, in order, as one pass reads them. The requests+ /// are not kept, because an event of many tiny requests would take far more memory as a list than in its payload.+ pub fn for_each_network_request(&self, inner: &[u8], mut each: impl FnMut(NetworkRequest)) {+ let Some(items) = self+ .network_requests+ .and_then(|r| scan::array_items(inner, r).ok())+ else {+ return;+ };+ for request in items.filter_map(Result::ok) {+ let [duration, status, response_status] =+ fields(inner, request, [b"duration", b"status", b"responseStatus"]);+ let status = status+ .filter(|&s| !scan::is_null(inner, s))+ .or(response_status);+ each(NetworkRequest {+ duration: number(inner, duration),+ status: number(inner, status),+ });+ }+ }+}+/// Counts what one block's events did. Navigation and active time need the session's earlier events, so the caller/// keeps that state.#[derive(Default)]@@ -93,6 +214,7 @@ impl EventCounter {&mut self,inner: &[u8],event: &Event,+ data: &DataFields,last_href: &mut Option<String>,activity: &mut Activity,) -> bool {@@ -110,18 +232,15 @@ impl EventCounter {stats.first_ts_ms = Some(stats.first_ts_ms.map_or(ts, |first| first.min(ts)));stats.last_ts_ms = Some(stats.last_ts_ms.map_or(ts, |last| last.max(ts)));}- let Some(data) = event.data else { return false };+ if event.data.is_none() {+ return false;+ }match (event.kind, event.source) {(Some(META | CUSTOM), _) => {- let [href, payload, tag, width, height] = fields(- inner,- data,- [b"href", b"payload", b"tag", b"width", b"height"],- );if event.kind == Some(META) {- set_viewport(stats, inner, event, width, height);+ set_viewport(stats, inner, event, data.width, data.height);}- let Some(href) = page_url(inner, event.kind, href, payload, tag) else {+ let Some(href) = href_from(inner, event.kind, data) else {return false;};if stats.host.is_none() {@@ -129,13 +248,13 @@ impl EventCounter {}if last_href.as_deref() != Some(&*href) {stats.navigations += 1;- *last_href = Some(href);+ *last_href = Some(href.into_owned());}return true;}(Some(INCREMENTAL_SNAPSHOT), Some(MOUSE_INTERACTION)) => {stats.mouse_interactions += 1;- let interaction = number(inner, field(inner, data, b"type")).map(|t| t as u8);+ let interaction = number(inner, data.interaction).map(|t| t as u8);if interaction.is_some_and(is_click) {stats.clicks += 1;}@@ -153,8 +272,7 @@ impl EventCounter {(Some(INCREMENTAL_SNAPSHOT), Some(SCROLL)) => stats.scrolls += 1,(Some(INCREMENTAL_SNAPSHOT), Some(VIEWPORT_RESIZE)) => {stats.viewport_resizes += 1;- let [width, height] = fields(inner, data, [b"width", b"height"]);- set_viewport(stats, inner, event, width, height);+ set_viewport(stats, inner, event, data.width, data.height);}(Some(INCREMENTAL_SNAPSHOT), Some(INPUT)) => stats.keypresses += 1,(Some(INCREMENTAL_SNAPSHOT), Some(MEDIA_INTERACTION)) => stats.media_events += 1,@@ -165,41 +283,23 @@ impl EventCounter {false}- fn observe_plugin(&mut self, inner: &[u8], data: Span) {+ /// Counts one request of an `rrweb/network@1` event. The inspector reads an event's requests in one pass for this+ /// and the feature recorder.+ pub fn count_network_request(&mut self, status: Option<f64>) {+ count_request(&mut self.stats, status);+ }++ fn observe_plugin(&mut self, inner: &[u8], data: &DataFields) {let stats = &mut self.stats;- let payload = field(inner, data, b"payload");- match text(inner, field(inner, data, b"plugin")).as_deref() {- Some("rrweb/console@1") => match payload- .and_then(|p| text(inner, field(inner, p, b"level")))- .as_deref()- {+ match text(inner, data.plugin).as_deref() {+ Some("rrweb/console@1") => match text(inner, data.level).as_deref() {Some("error") => stats.console_errors += 1,Some("warn") => stats.console_warnings += 1,_ => {}},- Some("rrweb/network@1") => {- let Some(requests) = payload.and_then(|p| field(inner, p, b"requests")) else {- return;- };- let Ok(items) = scan::array_items(inner, requests) else {- return;- };- for request in items.filter_map(Result::ok) {- // `req.status ?? req.responseStatus`, as `session-feature-recorder.ts` reads it.- let [status, response_status] =- fields(inner, request, [b"status", b"responseStatus"]);- let status = status- .filter(|&s| !scan::is_null(inner, s))- .or(response_status);- count_request(stats, number(inner, status));- }- }Some("posthog/network@1") => {- if let Some(payload) = payload.filter(|&p| scan::is_object(inner, p)) {- count_request(- stats,- number(inner, field(inner, payload, POSTHOG_NETWORK_STATUS_KEY)),- );+ if data.payload.is_some_and(|p| scan::is_object(inner, p)) {+ count_request(stats, number(inner, data.network_status));}}_ => {}@@ -240,32 +340,28 @@ fn set_viewport();}-/// The page URL of a Meta or custom event, by the rules of [`page_url`].+/// The page URL of a Meta or custom event, by the rules of [`href_from`].pub fn event_page_url(inner: &[u8], event: &Event) -> Option<String> {if !matches!(event.kind, Some(META | CUSTOM)) {return None;}- let [href, payload, tag] = fields(inner, event.data?, [b"href", b"payload", b"tag"]);- page_url(inner, event.kind, href, payload, tag)+ href_from(inner, event.kind, &DataFields::read(inner, event)).map(Cow::into_owned)}-/// The page URL an event carries, as `hrefFrom` in `rrweb-types.ts` reads it: a Meta event's `href`, or the-/// `payload.href` of a custom event such as posthog-js's `$url_changed`. JSON-LD custom events carry no page.-fn page_url(- inner: &[u8],- kind: Option<u8>,- href: Option<Span>,- payload: Option<Span>,- tag: Option<Span>,-) -> Option<String> {- if kind == Some(CUSTOM) && text(inner, tag).as_deref() == Some("$json_ld") {+/// The page URL an event carries, as `hrefFrom` in `rrweb-types.ts` reads it: `data.href`, such as a Meta event's, or+/// `data.payload.href`, such as that of posthog-js's `$url_changed` custom event, trimmed as JavaScript trims.+/// JSON-LD custom events carry no page.+pub fn href_from<'b>(inner: &'b [u8], kind: Option<u8>, data: &DataFields) -> Option<Cow<'b, str>> {+ if kind == Some(CUSTOM) && text(inner, data.tag).as_deref() == Some("$json_ld") {return None;}- let payload_href = payload.and_then(|p| field(inner, p, b"href"));- [href, payload_href]+ [data.href, data.payload_href].into_iter().filter_map(|span| text(inner, span))- .map(|href| href.trim().to_owned())+ .map(|href| match href {+ Cow::Borrowed(href) => Cow::Borrowed(js_trim(href)),+ Cow::Owned(href) => Cow::Owned(js_trim(&href).to_owned()),+ }).find(|href| !href.is_empty())}@@ -521,7 +617,11 @@ mod tests {let mut counter = EventCounter::default();let (mut last_href, mut activity) = (None, Activity::default());for event in &message.events {- counter.observe(&message.inner, event, &mut last_href, &mut activity);+ let data = DataFields::read(&message.inner, event);+ counter.observe(&message.inner, event, &data, &mut last_href, &mut activity);+ data.for_each_network_request(&message.inner, |request| {+ counter.count_network_request(request.status);+ });}let stats = counter.finish();diff --git a/rust/ai-research/replay-core-v4/src/inspect/features.rs b/rust/ai-research/replay-core-v4/src/inspect/features.rsnew file mode 100644index 000000000000..c8d72da07e48--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/inspect/features.rs@@ -0,0 +1,1174 @@+//! The values of ingestion's surfacing feature recorder (`session-feature-recorder.ts`), one block at a time.+//!+//! Ingestion records them per batch and the surfacing sweep merges a session's batch rows. The recorder starts fresh at+//! each block, as ingestion's does at each batch, so a block row holds what one batch row of `session_replay_features`+//! holds, and a session's values are the fold of its block rows: [`SurfacingFeatures::merge`].++use std::borrow::Cow;+use std::collections::{HashMap, HashSet};++use crate::bytescan::{self as scan, Span};+use crate::scan::message::Event;++use super::clicks::{ClickKind, ClickTracker};+use super::events::{href_from, number, text, DataFields, NetworkRequest};+use super::rrweb::*;++/// `MAX_UNIQUE_VALUES`: the values each set of a block keeps, and the input fields it remembers for backspaces.+pub const MAX_UNIQUE_VALUES: usize = 1000;++/// `PATH_TOKEN_ALLOWLIST` in its order, each with the name its count has in `session_replay_features`.+pub const PATH_TOKENS: [(&str, &str); 14] = [+ ("login", "login"),+ ("signup", "signup"),+ ("checkout", "checkout"),+ ("cart", "cart"),+ ("billing", "billing"),+ ("settings", "settings"),+ ("account", "account"),+ ("error", "error"),+ ("404", "not_found"),+ ("admin", "admin"),+ ("dashboard", "dashboard"),+ ("onboarding", "onboarding"),+ ("cancel", "cancel"),+ ("refund", "refund"),+];++const LONG_IDLE_GAP_MS: f64 = 30_000.0;+const SELECTION_COPY_WINDOW_MS: f64 = 2_000.0;+const QUICK_BACK_WINDOW_MS: f64 = 2_000.0;+const RAPID_SCROLL_REVERSAL_MS: f64 = 500.0;+const ERROR_AFTER_ACTION_MS: f64 = 5_000.0;++/// The largest node ID a set keeps. Past it a JavaScript number does not hold every integer, and the index refuses it.+const MAX_NODE_ID: f64 = 9_007_199_254_740_991.0;++/// One block's values as `SessionFeatureRecorder.end()` returns them for a batch. Float sums keep the exact values the+/// TypeScript computes, in the same order of operations.+#[derive(Debug, Clone, Default, PartialEq)]+pub struct BlockFeatures {+ pub event_count: u64,+ pub mouse_position_count: u64,+ pub mouse_sum_x: f64,+ pub mouse_sum_x_squared: f64,+ pub mouse_sum_y: f64,+ pub mouse_sum_y_squared: f64,+ pub mouse_distance_traveled: f64,+ pub mouse_direction_change_count: u64,+ pub mouse_velocity_sum: f64,+ pub mouse_velocity_sum_of_squares: f64,+ pub mouse_velocity_count: u64,+ pub scroll_event_count: u64,+ pub total_scroll_magnitude: f64,+ pub scroll_direction_reversal_count: u64,+ pub rapid_scroll_reversal_count: u64,+ pub scroll_to_top_count: u64,+ pub click_count: u64,+ pub keypress_count: u64,+ pub mouse_activity_count: u64,+ pub rage_click_count: u64,+ pub dead_click_count: u64,+ pub backspace_count: u64,+ pub inter_action_gap_count: u64,+ pub inter_action_gap_sum_ms: f64,+ pub inter_action_gap_sum_of_squares_ms: f64,+ pub max_idle_gap_ms: f64,+ pub long_idle_gap_count: u64,+ pub quick_back_count: u64,+ pub page_visit_count: u64,+ /// The distinct page URLs in plaintext, sorted. The writer stores them as fingerprints under the session key, where+ /// ingestion stores their md5.+ pub visited_urls: Vec<String>,+ pub path_token_counts: [u64; PATH_TOKENS.len()],+ pub console_error_count: u64,+ pub console_error_after_click_count: u64,+ pub console_warn_count: u64,+ pub network_request_count: u64,+ pub network_failed_request_count: u64,+ pub network_4xx_count: u64,+ pub network_5xx_count: u64,+ pub network_request_duration_sum: f64,+ pub network_request_duration_sum_of_squares: f64,+ pub network_request_duration_count: u64,+ pub mutation_count: u64,+ pub viewport_resize_count: u64,+ pub touch_event_count: u64,+ pub max_scroll_y: f64,+ /// Sorted.+ pub click_target_ids: Vec<i64>,+ /// Sorted.+ pub form_field_ids: Vec<i64>,+ pub text_selection_count: u64,+ pub selection_copy_count: u64,+}++/// What the recorder knows about the block so far. It never outlives the block.+#[derive(Default)]+pub struct FeatureRecorder {+ values: BlockFeatures,+ visited_urls: HashSet<String>,+ click_target_ids: HashSet<i64>,+ form_field_ids: HashSet<i64>,+ /// The last text length of each input, by the bits of its ID, for the backspace count.+ input_text_lengths: HashMap<u64, usize>,+ last_mouse: Option<MousePosition>,+ last_mouse_delta: Option<(f64, f64)>,+ scroll: ScrollState,+ clicks: ClickTracker,+ /// The last click or keypress.+ last_action_ts: Option<f64>,+ last_navigation_ts: Option<f64>,+ /// Valid while `last_navigation_ts` is set. Reused, so that a navigation allocates only for a new page.+ last_navigation_url: String,+ /// The path tokens of `last_navigation_url`, one bit per token, so that the same page is parsed once.+ last_navigation_tokens: u16,+ last_selection_ts: Option<f64>,+}++#[derive(Clone, Copy)]+struct MousePosition {+ x: f64,+ y: f64,+ ts: f64,+}++#[derive(Default)]+struct ScrollState {+ last_up: Option<bool>,+ last_ts: Option<f64>,+ last_y: Option<f64>,+ /// `lastScrollId`: JavaScript `null` or a node ID.+ last_id: Option<f64>,+}++impl FeatureRecorder {+ /// Records one event of the block, apart from the requests of an `rrweb/network@1` event, which reach+ /// [`FeatureRecorder::track_network_request`]. Returns whether the event counts: ingestion's parse step drops an+ /// event without a positive timestamp before its recorder sees it, so such an event changes nothing here.+ pub fn observe(&mut self, inner: &[u8], event: &Event, data: &DataFields) -> bool {+ let Some(ts) = event.timestamp_ms.filter(|&ts| ts > 0.0) else {+ return false;+ };+ self.values.event_count += 1;+ let source = event+ .source+ .filter(|_| event.kind == Some(INCREMENTAL_SNAPSHOT));+ let interaction = number(inner, data.interaction);+ let is_click = source == Some(MOUSE_INTERACTION)+ && interaction.is_some_and(|t| {+ [CLICK, DOUBLE_CLICK, TOUCH_END, CONTEXT_MENU]+ .map(f64::from)+ .contains(&t)+ });+ let is_keypress = source == Some(INPUT);++ // The order of `aggregateFeatures`: a click reads the navigation state before a navigation on the same event+ // sets it.+ match source {+ Some(MOUSE_MOVE) => self.track_mouse_positions(inner, ts, event.positions_start),+ Some(SCROLL) => self.track_scroll(inner, ts, data),+ _ => {}+ }+ if is_click {+ self.track_click(inner, ts, data);+ }+ if is_keypress {+ self.track_keypress(inner, ts, data);+ }+ if is_click || is_keypress {+ self.track_inter_action_gap(ts);+ }+ if let Some(href) = href_from(inner, event.kind, data) {+ self.track_navigation(ts, &href);+ }+ if event.kind == Some(PLUGIN) {+ self.track_plugin(inner, ts, data);+ }+ let values = &mut self.values;+ match source {+ Some(SELECTION) => {+ values.text_selection_count += 1;+ self.last_selection_ts = Some(ts);+ }+ Some(MUTATION) => values.mutation_count += 1,+ Some(VIEWPORT_RESIZE) => values.viewport_resize_count += 1,+ _ => {}+ }+ let touch_interaction =+ interaction.is_some_and(|t| [TOUCH_START, TOUCH_END].map(f64::from).contains(&t));+ if source == Some(TOUCH_MOVE) || (source == Some(MOUSE_INTERACTION) && touch_interaction) {+ values.touch_event_count += 1;+ }+ if matches!(source, Some(MOUSE_INTERACTION | MOUSE_MOVE | TOUCH_MOVE)) {+ values.mouse_activity_count += 1;+ }+ true+ }++ fn track_mouse_positions(&mut self, inner: &[u8], ts: f64, positions: Option<u32>) {+ let Some(items) = positions.and_then(|start| {+ scan::array_object_fields(inner, start as usize, [b"x", b"y", b"timeOffset"]).ok()+ }) else {+ return;+ };+ for [x, y, offset] in items.map_while(Result::ok) {+ // The TypeScript skips a falsy item, and turns its sums into NaN on an item without coordinates.+ let (Some(x), Some(y)) = (number(inner, x), number(inner, y)) else {+ continue;+ };+ let ts = ts + number(inner, offset).unwrap_or(0.0);+ let values = &mut self.values;+ values.mouse_position_count += 1;+ values.mouse_sum_x += x;+ values.mouse_sum_x_squared += x * x;+ values.mouse_sum_y += y;+ values.mouse_sum_y_squared += y * y;+ if let Some(last) = self.last_mouse {+ let (dx, dy) = (x - last.x, y - last.y);+ let distance = (dx * dx + dy * dy).sqrt();+ values.mouse_distance_traveled += distance;+ if let Some((last_dx, last_dy)) = self.last_mouse_delta {+ if dx * last_dx + dy * last_dy < 0.0 {+ values.mouse_direction_change_count += 1;+ }+ }+ self.last_mouse_delta = Some((dx, dy));+ let dt = ts - last.ts;+ if dt > 0.0 {+ let velocity = distance / dt;+ values.mouse_velocity_sum += velocity;+ values.mouse_velocity_sum_of_squares += velocity * velocity;+ values.mouse_velocity_count += 1;+ }+ }+ self.last_mouse = Some(MousePosition { x, y, ts });+ }+ }++ fn track_scroll(&mut self, inner: &[u8], ts: f64, data: &DataFields) {+ let values = &mut self.values;+ values.scroll_event_count += 1;+ let Some(y) = number(inner, data.y) else {+ return;+ };+ if y > values.max_scroll_y {+ values.max_scroll_y = y;+ }+ // `scrollId !== this.lastScrollId`: a missing ID never equals the last one, so it resets every time.+ let id = match data.id {+ None => None,+ Some(id) if scan::is_null(inner, id) => Some(None),+ Some(id) => number(inner, Some(id)).map(Some),+ };+ let scroll = &mut self.scroll;+ if id.is_none_or(|id| id != scroll.last_id) {+ scroll.last_y = None;+ scroll.last_up = None;+ scroll.last_id = id.flatten();+ }+ let Some(last_y) = scroll.last_y else {+ scroll.last_y = Some(y);+ scroll.last_ts = Some(ts);+ return;+ };+ let delta = y - last_y;+ values.total_scroll_magnitude += delta.abs();+ if last_y > 0.0 && y == 0.0 {+ values.scroll_to_top_count += 1;+ }+ if delta != 0.0 {+ let up = delta < 0.0;+ if scroll.last_up.is_some_and(|last_up| last_up != up) {+ values.scroll_direction_reversal_count += 1;+ if scroll+ .last_ts+ .is_some_and(|last_ts| ts - last_ts < RAPID_SCROLL_REVERSAL_MS)+ {+ values.rapid_scroll_reversal_count += 1;+ }+ }+ scroll.last_up = Some(up);+ }+ scroll.last_y = Some(y);+ scroll.last_ts = Some(ts);+ }++ fn track_click(&mut self, inner: &[u8], ts: f64, data: &DataFields) {+ self.values.click_count += 1;+ if let Some(id) = node_id(inner, data.id) {+ if self.click_target_ids.len() < MAX_UNIQUE_VALUES {+ self.click_target_ids.insert(id);+ }+ }+ let position = number(inner, data.x).zip(number(inner, data.y));+ match self.clicks.click(ts, position) {+ ClickKind::Rage => self.values.rage_click_count += 1,+ ClickKind::AfterDead => self.values.dead_click_count += 1,+ ClickKind::Plain => {}+ }+ }++ fn track_keypress(&mut self, inner: &[u8], ts: f64, data: &DataFields) {+ self.values.keypress_count += 1;+ if let Some(id) = number(inner, data.id) {+ if let Some(id) = node_id(inner, data.id) {+ if self.form_field_ids.len() < MAX_UNIQUE_VALUES {+ self.form_field_ids.insert(id);+ }+ }+ if let Some(length) = data.text.and_then(|text| utf16_length(inner, text)) {+ // A JavaScript `Map` holds 0 and -0 as one key.+ let key = (id + 0.0).to_bits();+ let previous = self.input_text_lengths.get(&key).copied();+ if let Some(previous) = previous.filter(|&previous| length < previous) {+ self.values.backspace_count += (previous - length) as u64;+ }+ if previous.is_some() || self.input_text_lengths.len() < MAX_UNIQUE_VALUES {+ self.input_text_lengths.insert(key, length);+ }+ }+ }+ if self+ .last_selection_ts+ .is_some_and(|selected| ts - selected <= SELECTION_COPY_WINDOW_MS)+ {+ self.values.selection_copy_count += 1;+ self.last_selection_ts = None;+ }+ }++ fn track_inter_action_gap(&mut self, ts: f64) {+ if let Some(last) = self.last_action_ts {+ let gap = ts - last;+ if gap > 0.0 {+ let values = &mut self.values;+ values.inter_action_gap_count += 1;+ values.inter_action_gap_sum_ms += gap;+ values.inter_action_gap_sum_of_squares_ms += gap * gap;+ if gap > values.max_idle_gap_ms {+ values.max_idle_gap_ms = gap;+ }+ if gap > LONG_IDLE_GAP_MS {+ values.long_idle_gap_count += 1;+ }+ }+ }+ self.last_action_ts = Some(ts);+ }++ fn track_navigation(&mut self, ts: f64, url: &str) {+ self.values.page_visit_count += 1;+ if self.visited_urls.len() < MAX_UNIQUE_VALUES && !self.visited_urls.contains(url) {+ self.visited_urls.insert(url.to_owned());+ }+ self.clicks.navigated();+ let same_page = self.last_navigation_ts.is_some() && self.last_navigation_url == url;+ let tokens = if same_page {+ self.last_navigation_tokens+ } else {+ path_tokens(url)+ };+ for (index, count) in self.values.path_token_counts.iter_mut().enumerate() {+ if tokens & (1 << index) != 0 {+ *count += 1;+ }+ }+ if self+ .last_navigation_ts+ .is_some_and(|last| ts - last < QUICK_BACK_WINDOW_MS)+ && !same_page+ {+ self.values.quick_back_count += 1;+ }+ if !same_page {+ self.last_navigation_url.clear();+ self.last_navigation_url.push_str(url);+ self.last_navigation_tokens = tokens;+ }+ self.last_navigation_ts = Some(ts);+ }++ fn track_plugin(&mut self, inner: &[u8], ts: f64, data: &DataFields) {+ let Some(plugin) = text(inner, data.plugin) else {+ return;+ };+ match &*plugin {+ "rrweb/console@1" => match text(inner, data.level).as_deref() {+ Some("error") => {+ self.values.console_error_count += 1;+ if self+ .last_action_ts+ .is_some_and(|last| (0.0..ERROR_AFTER_ACTION_MS).contains(&(ts - last)))+ {+ self.values.console_error_after_click_count += 1;+ }+ }+ Some("warn") => self.values.console_warn_count += 1,+ _ => {}+ },+ // `if (!payload) return`, then `payload[39]` and `payload[21]`.+ "posthog/network@1" => {+ if data.payload.is_some_and(|p| js_truthy(inner, p)) {+ self.track_network_request(NetworkRequest {+ duration: number(inner, data.network_duration),+ status: number(inner, data.network_status),+ });+ }+ }+ _ => {}+ }+ }++ /// Records one request of an `rrweb/network@1` event. The inspector reads an event's requests in one pass for this+ /// and the event counter.+ pub fn track_network_request(&mut self, request: NetworkRequest) {+ let NetworkRequest { duration, status } = request;+ let values = &mut self.values;+ values.network_request_count += 1;+ if let Some(status) = status {+ if (400.0..500.0).contains(&status) {+ values.network_4xx_count += 1;+ values.network_failed_request_count += 1;+ } else if (500.0..600.0).contains(&status) {+ values.network_5xx_count += 1;+ values.network_failed_request_count += 1;+ } else if status >= 600.0 {+ values.network_failed_request_count += 1;+ }+ }+ if let Some(duration) = duration.filter(|&d| d > 0.0) {+ values.network_request_duration_sum += duration;+ values.network_request_duration_sum_of_squares += duration * duration;+ values.network_request_duration_count += 1;+ }+ }++ pub fn finish(self) -> BlockFeatures {+ fn sorted<T: Ord>(values: HashSet<T>) -> Vec<T> {+ let mut values: Vec<T> = values.into_iter().collect();+ values.sort_unstable();+ values+ }+ BlockFeatures {+ visited_urls: sorted(self.visited_urls),+ click_target_ids: sorted(self.click_target_ids),+ form_field_ids: sorted(self.form_field_ids),+ ..self.values+ }+ }+}++/// The surfacing values of a block as the index stores them, or of a session as [`SurfacingFeatures::merge`] folds+/// them. Each float sum of [`BlockFeatures`] is an integer in the unit its name gives, rounded once per block, so that+/// the fold is exact and does not depend on the order of the blocks.+#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord)]+pub struct SurfacingFeatures {+ pub event_count: u64,+ pub click_count: u64,+ pub keypress_count: u64,+ pub mouse_activity_count: u64,+ pub rage_click_count: u64,+ pub dead_click_count: u64,+ pub quick_back_count: u64,+ pub page_visit_count: u64,+ pub text_selection_count: u64,+ pub scroll_event_count: u64,+ pub console_error_count: u64,+ pub console_error_after_click_count: u64,+ pub console_warn_count: u64,+ pub network_request_count: u64,+ pub network_failed_request_count: u64,+ pub network_4xx_count: u64,+ pub network_5xx_count: u64,+ pub network_request_duration_count: u64,+ pub mouse_position_count: u64,+ pub mouse_direction_change_count: u64,+ pub mouse_velocity_count: u64,+ pub scroll_direction_reversal_count: u64,+ pub rapid_scroll_reversal_count: u64,+ pub scroll_to_top_count: u64,+ pub inter_action_gap_count: u64,+ pub long_idle_gap_count: u64,+ pub backspace_count: u64,+ pub mutation_count: u64,+ pub viewport_resize_count: u64,+ pub touch_event_count: u64,+ pub selection_copy_count: u64,+ /// In [`PATH_TOKENS`] order.+ pub path_visit_counts: [u64; PATH_TOKENS.len()],++ pub mouse_sum_x_millipx: i64,+ pub mouse_sum_x_squared_px2: i64,+ pub mouse_sum_y_millipx: i64,+ pub mouse_sum_y_squared_px2: i64,+ pub mouse_distance_traveled_millipx: i64,+ pub mouse_velocity_sum_px_per_s: i64,+ pub mouse_velocity_sum_of_squares_px2_per_s2: i64,+ pub total_scroll_magnitude_millipx: i64,+ pub inter_action_gap_sum_ms: i64,+ pub inter_action_gap_sum_of_squares_ms2: i64,+ pub network_request_duration_sum_us: i64,+ pub network_request_duration_sum_of_squares_ms2: i64,++ pub max_scroll_y_millipx: i64,+ pub max_idle_gap_ms: i64,++ /// The distinct page URLs as [`crate::refs::url_fingerprint`]s under the session's fingerprint key, sorted. The+ /// inspector leaves this empty: the writer fills it, because only the writer holds the session key.+ pub visited_url_fingerprints: Vec<String>,+ /// Sorted.+ pub click_target_ids: Vec<i64>,+ /// Sorted.+ pub form_field_ids: Vec<i64>,+}++impl SurfacingFeatures {+ /// Folds another block of the session into this one: counts and sums add, maxima take the larger, and sets take+ /// the union with no cap. The result does not depend on the order or grouping of the blocks.+ pub fn merge(&mut self, other: &SurfacingFeatures) {+ macro_rules! add {+ ($($field:ident),*) => { $( self.$field = self.$field.saturating_add(other.$field); )* };+ }+ add!(+ event_count,+ click_count,+ keypress_count,+ mouse_activity_count,+ rage_click_count,+ dead_click_count,+ quick_back_count,+ page_visit_count,+ text_selection_count,+ scroll_event_count,+ console_error_count,+ console_error_after_click_count,+ console_warn_count,+ network_request_count,+ network_failed_request_count,+ network_4xx_count,+ network_5xx_count,+ network_request_duration_count,+ mouse_position_count,+ mouse_direction_change_count,+ mouse_velocity_count,+ scroll_direction_reversal_count,+ rapid_scroll_reversal_count,+ scroll_to_top_count,+ inter_action_gap_count,+ long_idle_gap_count,+ backspace_count,+ mutation_count,+ viewport_resize_count,+ touch_event_count,+ selection_copy_count,+ mouse_sum_x_millipx,+ mouse_sum_x_squared_px2,+ mouse_sum_y_millipx,+ mouse_sum_y_squared_px2,+ mouse_distance_traveled_millipx,+ mouse_velocity_sum_px_per_s,+ mouse_velocity_sum_of_squares_px2_per_s2,+ total_scroll_magnitude_millipx,+ inter_action_gap_sum_ms,+ inter_action_gap_sum_of_squares_ms2,+ network_request_duration_sum_us,+ network_request_duration_sum_of_squares_ms2+ );+ for (count, other) in self+ .path_visit_counts+ .iter_mut()+ .zip(other.path_visit_counts)+ {+ *count = count.saturating_add(other);+ }+ self.max_scroll_y_millipx = self.max_scroll_y_millipx.max(other.max_scroll_y_millipx);+ self.max_idle_gap_ms = self.max_idle_gap_ms.max(other.max_idle_gap_ms);+ union(+ &mut self.visited_url_fingerprints,+ &other.visited_url_fingerprints,+ );+ union(&mut self.click_target_ids, &other.click_target_ids);+ union(&mut self.form_field_ids, &other.form_field_ids);+ }+}++impl From<&BlockFeatures> for SurfacingFeatures {+ fn from(block: &BlockFeatures) -> Self {+ SurfacingFeatures {+ event_count: block.event_count,+ click_count: block.click_count,+ keypress_count: block.keypress_count,+ mouse_activity_count: block.mouse_activity_count,+ rage_click_count: block.rage_click_count,+ dead_click_count: block.dead_click_count,+ quick_back_count: block.quick_back_count,+ page_visit_count: block.page_visit_count,+ text_selection_count: block.text_selection_count,+ scroll_event_count: block.scroll_event_count,+ console_error_count: block.console_error_count,+ console_error_after_click_count: block.console_error_after_click_count,+ console_warn_count: block.console_warn_count,+ network_request_count: block.network_request_count,+ network_failed_request_count: block.network_failed_request_count,+ network_4xx_count: block.network_4xx_count,+ network_5xx_count: block.network_5xx_count,+ network_request_duration_count: block.network_request_duration_count,+ mouse_position_count: block.mouse_position_count,+ mouse_direction_change_count: block.mouse_direction_change_count,+ mouse_velocity_count: block.mouse_velocity_count,+ scroll_direction_reversal_count: block.scroll_direction_reversal_count,+ rapid_scroll_reversal_count: block.rapid_scroll_reversal_count,+ scroll_to_top_count: block.scroll_to_top_count,+ inter_action_gap_count: block.inter_action_gap_count,+ long_idle_gap_count: block.long_idle_gap_count,+ backspace_count: block.backspace_count,+ mutation_count: block.mutation_count,+ viewport_resize_count: block.viewport_resize_count,+ touch_event_count: block.touch_event_count,+ selection_copy_count: block.selection_copy_count,+ path_visit_counts: block.path_token_counts,+ mouse_sum_x_millipx: fixed(block.mouse_sum_x, 1e3),+ mouse_sum_x_squared_px2: fixed(block.mouse_sum_x_squared, 1.0),+ mouse_sum_y_millipx: fixed(block.mouse_sum_y, 1e3),+ mouse_sum_y_squared_px2: fixed(block.mouse_sum_y_squared, 1.0),+ mouse_distance_traveled_millipx: fixed(block.mouse_distance_traveled, 1e3),+ // The TypeScript's velocity is in pixels per millisecond.+ mouse_velocity_sum_px_per_s: fixed(block.mouse_velocity_sum, 1e3),+ mouse_velocity_sum_of_squares_px2_per_s2: fixed(+ block.mouse_velocity_sum_of_squares,+ 1e6,+ ),+ total_scroll_magnitude_millipx: fixed(block.total_scroll_magnitude, 1e3),+ inter_action_gap_sum_ms: fixed(block.inter_action_gap_sum_ms, 1.0),+ inter_action_gap_sum_of_squares_ms2: fixed(+ block.inter_action_gap_sum_of_squares_ms,+ 1.0,+ ),+ network_request_duration_sum_us: fixed(block.network_request_duration_sum, 1e3),+ network_request_duration_sum_of_squares_ms2: fixed(+ block.network_request_duration_sum_of_squares,+ 1.0,+ ),+ max_scroll_y_millipx: fixed(block.max_scroll_y, 1e3),+ max_idle_gap_ms: fixed(block.max_idle_gap_ms, 1.0),+ visited_url_fingerprints: Vec::new(),+ click_target_ids: block.click_target_ids.clone(),+ form_field_ids: block.form_field_ids.clone(),+ }+ }+}++/// `value` in units `scale` times smaller, rounded half away from zero. A value past the `i64` range saturates.+fn fixed(value: f64, scale: f64) -> i64 {+ (value * scale).round() as i64+}++/// The sorted union of two sorted sets.+fn union<T: Ord + Clone>(values: &mut Vec<T>, other: &[T]) {+ if other.is_empty() {+ return;+ }+ values.extend_from_slice(other);+ values.sort_unstable();+ values.dedup();+}++/// A node ID as a set keeps it. The TypeScript keeps any value, but rrweb's node IDs are integers.+fn node_id(inner: &[u8], span: Option<Span>) -> Option<i64> {+ number(inner, span)+ .filter(|id| id.fract() == 0.0 && id.abs() <= MAX_NODE_ID)+ .map(|id| id as i64)+}++/// `text.length` in JavaScript: the UTF-16 code units of a JSON string, counted on its escaped bytes so that nothing+/// is decoded or allocated. An escape is one unit, and so is each half of an escaped surrogate pair.+fn utf16_length(bytes: &[u8], span: Span) -> Option<usize> {+ if !scan::is_string(bytes, span) || span.1 < span.0 + 2 {+ return None;+ }+ let raw = &bytes[span.0 + 1..span.1 - 1];+ let (mut at, mut units) = (0, 0);+ while let Some(&byte) = raw.get(at) {+ let (width, length) = match byte {+ b'\\' if raw.get(at + 1) == Some(&b'u') => (6, 1),+ b'\\' => (2, 1),+ 0x00..=0x7f => (1, 1),+ 0xc0..=0xdf => (2, 1),+ 0xe0..=0xef => (3, 1),+ 0xf0..=0xff => (4, 2),+ // A continuation byte, which a valid sequence never starts with.+ _ => (1, 0),+ };+ at += width;+ units += length;+ }+ Some(units)+}++/// JavaScript truthiness of a JSON value.+fn js_truthy(bytes: &[u8], span: Span) -> bool {+ match bytes.get(span.0) {+ Some(b'n' | b'f') => false,+ Some(b'"') => span.1 - span.0 > 2,+ Some(b'-' | b'0'..=b'9') => scan::parse_number(bytes, span).is_some_and(|n| n != 0.0),+ _ => true,+ }+}++/// The [`PATH_TOKENS`] among the segments of the URL's path, one bit per token in their order. A token counts once per+/// URL. `new URL(url).pathname`, or for a URL that does not parse, a relative path up to its query or fragment.+fn path_tokens(url: &str) -> u16 {+ let parsed = url::Url::parse(url);+ let path = match &parsed {+ Ok(parsed) => parsed.path(),+ Err(_) if url.starts_with('/') => url.split(['?', '#']).next().unwrap_or_default(),+ Err(_) => return 0,+ };+ let mut tokens = 0;+ for segment in path.split('/').filter(|s| !s.is_empty()) {+ // A parsed path is ASCII. A relative one can hold a character that lowercases to ASCII, such as the Kelvin sign.+ let lowered = if segment.is_ascii() {+ Cow::Borrowed(segment)+ } else {+ Cow::Owned(segment.to_lowercase())+ };+ for (index, (token, _)) in PATH_TOKENS.iter().enumerate() {+ if lowered.eq_ignore_ascii_case(token) {+ tokens |= 1 << index;+ }+ }+ }+ tokens+}++/// JavaScript's `String.prototype.trim`: its white space and line terminators, which differ from Rust's+/// [`str::trim`] in U+FEFF and U+0085.+pub(crate) fn js_trim(text: &str) -> &str {+ text.trim_matches(|c| {+ matches!(+ c,+ '\u{9}'..='\u{d}'+ | ' '+ | '\u{a0}'+ | '\u{1680}'+ | '\u{2000}'..='\u{200a}'+ | '\u{2028}'+ | '\u{2029}'+ | '\u{202f}'+ | '\u{205f}'+ | '\u{3000}'+ | '\u{feff}'+ )+ })+}++#[cfg(test)]+mod tests {+ use serde_json::{json, Value};++ use super::*;+ use crate::inspect::session::{BlockInspection, SessionInspector};+ use crate::scan::message::read_message;++ // Expected values traced through `session-feature-recorder.ts` over the same events.++ fn record_blocks(blocks: &[Vec<Value>]) -> Vec<BlockFeatures> {+ let mut session = SessionInspector::default();+ blocks+ .iter()+ .map(|events| {+ let inner = json!({"event": "$snapshot_items", "properties": {+ "$session_id": "s", "$window_id": "w", "$snapshot_source": "web", "$snapshot_items": events,+ }});+ let payload = json!({"distinct_id": "d", "data": inner.to_string()});+ let message = read_message(payload.to_string().into_bytes()).unwrap();+ let mut block = BlockInspection::default();+ session.observe_message(&mut block, &message);+ block.finish().features+ })+ .collect()+ }++ fn record(events: Vec<Value>) -> BlockFeatures {+ record_blocks(&[events]).remove(0)+ }++ fn incremental(ts: i64, data: Value) -> Value {+ json!({"type": 3, "timestamp": ts, "data": data})+ }++ fn click(ts: i64, x: f64, y: f64, id: i64, kind: Value) -> Value {+ incremental(+ ts,+ json!({"source": 2, "type": kind, "x": x, "y": y, "id": id}),+ )+ }++ fn scroll(ts: i64, mut target: Value) -> Value {+ target["source"] = json!(3);+ incremental(ts, target)+ }++ fn input(ts: i64, id: Option<i64>, text: Value) -> Value {+ let mut data = json!({"source": 5, "text": text});+ if let Some(id) = id {+ data["id"] = json!(id);+ }+ incremental(ts, data)+ }++ fn page(ts: i64, data: Value) -> Value {+ json!({"type": 4, "timestamp": ts, "data": data})+ }++ fn plugin(ts: i64, name: &str, payload: Value) -> Value {+ json!({"type": 6, "timestamp": ts, "data": {"plugin": name, "payload": payload}})+ }++ #[test]+ fn mouse_positions() {+ let f = record(vec![incremental(+ 1000,+ json!({"source": 1, "positions": [+ {"x": 0, "y": 0, "timeOffset": 0}, null, {"x": 3, "y": 4, "timeOffset": 10},+ {"x": 0, "y": 0, "timeOffset": 20}, {"x": 0.5, "y": 0},+ ]}),+ )]);+ assert_eq!(+ (+ f.mouse_position_count,+ f.mouse_sum_x,+ f.mouse_sum_x_squared,+ f.mouse_sum_y,+ f.mouse_sum_y_squared+ ),+ (4, 3.5, 9.25, 4.0, 16.0)+ );+ assert_eq!(+ (+ f.mouse_distance_traveled,+ f.mouse_direction_change_count,+ f.mouse_velocity_sum+ ),+ (10.5, 2, 1.0)+ );+ assert_eq!(+ (f.mouse_velocity_sum_of_squares, f.mouse_velocity_count),+ (0.5, 2)+ );+ }++ #[test]+ fn scrolls() {+ // A scroll without a target starts again every time; a null target continues after it.+ let f = record(vec![+ scroll(1000, json!({"id": 1, "y": 0})),+ scroll(1100, json!({"id": 1, "y": 300})),+ scroll(1200, json!({"id": 1, "y": 200})),+ scroll(2200, json!({"id": 1, "y": 900})),+ scroll(2300, json!({"id": 1, "y": 0})),+ scroll(2400, json!({"y": 100})),+ scroll(2500, json!({"y": 400})),+ scroll(2600, json!({"id": null, "y": 100})),+ scroll(2700, json!({"id": 1})),+ ]);+ assert_eq!(+ (+ f.scroll_event_count,+ f.total_scroll_magnitude,+ f.scroll_direction_reversal_count,+ f.rapid_scroll_reversal_count,+ f.scroll_to_top_count,+ f.max_scroll_y+ ),+ (9, 2300.0, 3, 2, 1, 900.0)+ );+ }++ #[test]+ fn clicks_rage_dead_and_touches() {+ let f = record(vec![+ click(1000, 100.0, 100.0, 5, json!(2)),+ click(1200, 110.0, 105.0, 5, json!(2)),+ click(1400, 112.0, 100.0, 6, json!(2)),+ click(1500, 112.0, 101.0, 6, json!(4)),+ click(4000, 500.0, 500.0, 7, json!(2)),+ // The lone click at 4000 did nothing before this context menu, so it was dead.+ click(7000, 900.0, 900.0, 8, json!(3)),+ page(7500, json!({"href": "https://shop.example.com/a"})),+ click(10000, 100.0, 100.0, 9, json!(9)),+ incremental(13000, json!({"source": 2, "type": 2, "id": 10})),+ click(13100, 1.0, 1.0, 11, json!(1)),+ click(13200, 1.0, 1.0, 12, json!("2")),+ click(13300, 1.0, 1.0, 13, json!(7)),+ incremental(13400, json!({"source": 6, "positions": []})),+ ]);+ assert_eq!(+ (+ f.click_count,+ f.rage_click_count,+ f.dead_click_count,+ f.touch_event_count,+ f.mouse_activity_count+ ),+ (8, 2, 1, 3, 12)+ );+ assert_eq!(f.click_target_ids, [5, 6, 7, 8, 9, 10]);+ assert_eq!(+ (+ f.inter_action_gap_count,+ f.inter_action_gap_sum_ms,+ f.inter_action_gap_sum_of_squares_ms,+ f.max_idle_gap_ms+ ),+ (7, 12_000.0, 33_340_000.0, 3000.0)+ );+ }++ #[test]+ fn typing_backspaces_in_utf16_units_and_copies() {+ let selection = |ts: i64| incremental(ts, json!({"source": 14, "ranges": []}));+ let f = record(vec![+ input(1000, Some(20), json!("hello")),+ input(1100, Some(20), json!("hel")),+ input(1200, Some(21), json!("\u{1f600}ab")),+ input(1300, Some(21), json!("a")),+ input(1400, Some(22), json!("a\n\"b\" \u{1}")),+ input(1500, Some(22), json!("")),+ input(1600, None, json!("x")),+ input(1700, Some(23), json!(4)),+ selection(2000),+ input(3500, Some(20), json!("he")),+ input(3600, Some(20), json!("h")),+ selection(5000),+ input(7001, Some(20), json!("")),+ ]);+ assert_eq!(+ (+ f.keypress_count,+ f.backspace_count,+ f.text_selection_count,+ f.selection_copy_count+ ),+ (11, 15, 2, 1)+ );+ assert_eq!(f.form_field_ids, [20, 21, 22, 23]);+ }++ #[test]+ fn gaps_and_console_errors_after_an_action() {+ let console = |ts: i64, level: &str| plugin(ts, "rrweb/console@1", json!({"level": level}));+ let f = record(vec![+ console(500, "error"),+ click(1000, 1.0, 1.0, 1, json!(2)),+ click(1000, 1.0, 1.0, 1, json!(2)),+ input(41000, Some(2), json!("a")),+ // Out of order: no gap counts, but the next starts from here.+ click(2000, 1.0, 1.0, 1, json!(2)),+ console(6999, "error"),+ console(7000, "error"),+ console(1500, "error"),+ console(7100, "warn"),+ console(7200, "info"),+ ]);+ assert_eq!(+ (+ f.inter_action_gap_count,+ f.inter_action_gap_sum_ms,+ f.max_idle_gap_ms,+ f.long_idle_gap_count+ ),+ (1, 40_000.0, 40_000.0, 1)+ );+ assert_eq!(+ (+ f.console_error_count,+ f.console_error_after_click_count,+ f.console_warn_count+ ),+ (4, 1, 1)+ );+ }++ #[test]+ fn navigations_from_any_event_with_javascript_trim_and_path_tokens() {+ let changed = |ts: i64, tag: &str, href: &str| json!({"type": 5, "timestamp": ts, "data": {"tag": tag, "payload": {"href": href}}});+ let f = record(vec![+ page(+ 1000,+ json!({"href": "https://shop.example.com/Account/Settings?tab=billing#checkout"}),+ ),+ changed(1500, "$url_changed", "https://shop.example.com/cart"),+ page(1800, json!({"href": "https://shop.example.com/cart"})),+ changed(1900, "$json_ld", "https://shop.example.com/login"),+ page(+ 5000,+ json!({"href": " ", "payload": {"href": "\u{feff}https://shop.example.com/login/LOGIN\u{a0}"}}),+ ),+ page(9000, json!({"href": "/settings/404?next=/admin"})),+ page(12000, json!({"href": "/chec\u{212a}out"})),+ page(15000, json!({"href": "dashboard"})),+ page(+ 18000,+ json!({"href": "https://shop.example.com/onboarding/../refund"}),+ ),+ incremental(+ 40000,+ json!({"source": 2, "type": 2, "x": 1, "y": 1, "id": 1, "href": "https://shop.example.com/b"}),+ ),+ plugin(+ 40500,+ "rrweb/console@1",+ json!({"level": "log", "href": "https://shop.example.com/c"}),+ ),+ incremental(+ 41000,+ json!({"source": 0, "adds": [], "removes": [], "texts": [], "attributes": [],+ "payload": {"href": "https://shop.example.com/d"}}),+ ),+ ]);+ assert_eq!(+ (f.page_visit_count, f.quick_back_count, f.visited_urls.len()),+ (11, 3, 10)+ );+ assert!(f+ .visited_urls+ .contains(&"https://shop.example.com/login/LOGIN".to_owned()));+ // login, signup, checkout, cart, billing, settings, account, error, 404, admin, dashboard, onboarding, cancel, refund+ assert_eq!(+ f.path_token_counts,+ [1, 0, 1, 2, 0, 2, 1, 0, 1, 0, 0, 0, 0, 1]+ );+ }++ #[test]+ fn network_requests_from_both_plugins() {+ let f = record(vec![+ plugin(+ 1000,+ "rrweb/network@1",+ json!({"requests": [+ {"status": 200, "duration": 12.5}, {"status": 404, "duration": 0},+ {"status": null, "responseStatus": 503, "duration": 0.25}, {"responseStatus": 600}, {"status": "500"}, 7,+ ]}),+ ),+ plugin(1100, "posthog/network@1", json!({"21": 500, "39": 100})),+ plugin(1200, "posthog/network@1", json!("")),+ plugin(1300, "posthog/network@1", json!("x")),+ plugin(1400, "posthog/network@1", json!(0)),+ plugin(1500, "other/plugin@1", json!({"requests": [{}]})),+ ]);+ assert_eq!(+ (+ f.network_request_count,+ f.network_failed_request_count,+ f.network_4xx_count,+ f.network_5xx_count+ ),+ (8, 4, 1, 2)+ );+ assert_eq!(+ (+ f.network_request_duration_sum,+ f.network_request_duration_sum_of_squares,+ f.network_request_duration_count+ ),+ (112.75, 10_156.312_5, 3)+ );+ }++ #[test]+ fn skips_the_events_ingestion_drops_and_starts_again_at_each_block() {+ let tap = |ts: Value, id: i64| json!({"type": 3, "timestamp": ts, "data": {"source": 2, "type": 2, "id": id}});+ let f = record(vec![+ json!({"type": 3, "data": {"source": 2, "type": 2, "id": 1}}),+ tap(json!(0), 2),+ tap(json!(-5), 3),+ tap(json!("1000"), 4),+ tap(json!(9e15), 5),+ tap(json!(1000), 6),+ json!({"type": 3, "timestamp": 1001}),+ ]);+ assert_eq!(+ (f.event_count, f.click_count, f.click_target_ids),+ (2, 1, vec![6])+ );++ // Three quick clicks are a rage click in one batch, and none across two.+ let at = |ts: i64| click(ts, 10.0, 10.0, 1, json!(2));+ let blocks = record_blocks(&[vec![at(1000), at(1100)], vec![at(1200)]]);+ assert_eq!(blocks.iter().map(|b| b.rage_click_count).sum::<u64>(), 0);+ assert_eq!(+ record(vec![at(1000), at(1100), at(1200)]).rage_click_count,+ 1+ );+ }++ #[test]+ fn keeps_the_first_thousand_values_of_each_set_and_input() {+ let mut events: Vec<Value> = (1..=1001)+ .map(|id| incremental(1000 + id, json!({"source": 2, "type": 2, "id": id, "href": format!("https://a.example/{id}")})))+ .collect();+ events.extend((1..=1001).map(|id| input(5000 + id, Some(id), json!("ab"))));+ // The text of the 1001st input is not remembered, so only the first shrink counts.+ events.extend([+ input(7000, Some(1001), json!("a")),+ input(7001, Some(1), json!("a")),+ ]);+ let f = record(events);+ let mut pages: Vec<String> = (1..=1000)+ .map(|id| format!("https://a.example/{id}"))+ .collect();+ pages.sort_unstable();+ assert_eq!(f.visited_urls, pages);+ assert_eq!(f.click_target_ids, (1..=1000).collect::<Vec<_>>());+ assert_eq!(f.form_field_ids, (1..=1000).collect::<Vec<_>>());+ assert_eq!(f.backspace_count, 1);+ }++ #[test]+ fn stores_each_float_sum_in_the_unit_its_column_names() {+ let block = BlockFeatures {+ mouse_sum_x: 1182.25,+ mouse_sum_y_squared: 647_657.25,+ mouse_distance_traveled: -1.0625,+ mouse_velocity_sum: 0.5,+ mouse_velocity_sum_of_squares: 0.25,+ inter_action_gap_sum_of_squares_ms: 1e300,+ network_request_duration_sum: 1170.35,+ network_request_duration_sum_of_squares: 2.5,+ max_scroll_y: 12_000.25,+ max_idle_gap_ms: 40_000.0,+ ..BlockFeatures::default()+ };+ let stored = SurfacingFeatures::from(&block);+ assert_eq!(+ [+ stored.mouse_sum_x_millipx,+ stored.mouse_sum_y_squared_px2,+ stored.mouse_distance_traveled_millipx,+ stored.mouse_velocity_sum_px_per_s,+ stored.mouse_velocity_sum_of_squares_px2_per_s2,+ stored.inter_action_gap_sum_of_squares_ms2,+ stored.network_request_duration_sum_us,+ stored.network_request_duration_sum_of_squares_ms2,+ stored.max_scroll_y_millipx,+ stored.max_idle_gap_ms,+ ],+ // Halves round away from zero, and a value past the range saturates.+ [+ 1_182_250,+ 647_657,+ -1_063,+ 500,+ 250_000,+ i64::MAX,+ 1_170_350,+ 3,+ 12_000_250,+ 40_000+ ]+ );+ }+}diff --git a/rust/ai-research/replay-core-v4/src/inspect/mod.rs b/rust/ai-research/replay-core-v4/src/inspect/mod.rsindex 8836d7d3ff28..2cdb26f1e535 100644--- a/rust/ai-research/replay-core-v4/src/inspect/mod.rs+++ b/rust/ai-research/replay-core-v4/src/inspect/mod.rs@@ -4,6 +4,7 @@ pub mod clicks;pub mod css;pub mod dom;pub mod events;+pub mod features;pub mod rrweb;pub mod session;pub mod stats;diff --git a/rust/ai-research/replay-core-v4/src/inspect/session.rs b/rust/ai-research/replay-core-v4/src/inspect/session.rsindex 436326e4c446..9b009decdf6d 100644--- a/rust/ai-research/replay-core-v4/src/inspect/session.rs+++ b/rust/ai-research/replay-core-v4/src/inspect/session.rs@@ -4,7 +4,8 @@ use std::sync::Arc;use super::clicks::{ClickKind, ClickTracker};use super::dom::{DomInspector, RemoteUrls, Srcset, UrlLimit};-use super::events::{fields, number, Activity, EventCounter};+use super::events::{number, Activity, DataFields, EventCounter};+use super::features::{BlockFeatures, FeatureRecorder, SurfacingFeatures};use super::rrweb::{is_click, CUSTOM, FULL_SNAPSHOT, INCREMENTAL_SNAPSHOT, META, MOUSE_INTERACTION, MUTATION,};@@ -46,6 +47,7 @@ pub struct SessionInspector {#[derive(Default)]pub struct BlockInspection {counter: EventCounter,+ features: FeatureRecorder,moments: Vec<Moment>,image_urls: Vec<String>,remote_urls: RemoteUrls,@@ -55,6 +57,8 @@ pub struct BlockInspection {/// What the inspector learned about one block.pub struct Inspected {pub stats: BlockStats,+ /// The surfacing values as the TypeScript recorder computes them, which `stats.surfacing` stores in index units.+ pub features: BlockFeatures,pub moments: Vec<Moment>,/// Remote image URLs the block's DOM referenced, for the image fetcher.pub image_urls: Vec<String>,@@ -68,8 +72,12 @@ pub struct Inspected {impl BlockInspection {pub fn finish(mut self) -> Inspected {+ let features = self.features.finish();+ let mut stats = self.counter.finish();+ stats.surfacing = SurfacingFeatures::from(&features);Inspected {- stats: self.counter.finish(),+ stats,+ features,moments: self.moments,image_urls: self.image_urls,url_limit: self.remote_urls.limit,@@ -139,10 +147,18 @@ impl SessionInspector {let index = block.events;block.events += 1;let navigations = block.counter.stats.navigations;+ let data = DataFields::read(inner, event);let carried_href =block.counter- .observe(inner, event, &mut self.last_href, &mut self.activity);+ .observe(inner, event, &data, &mut self.last_href, &mut self.activity);+ let recorded = block.features.observe(inner, event, &data);+ data.for_each_network_request(inner, |request| {+ block.counter.count_network_request(request.status);+ if recorded {+ block.features.track_network_request(request);+ }+ });let navigated = block.counter.stats.navigations > navigations;let waiting = block.counter.stats.events_before_first_full_snapshot;@@ -182,19 +198,17 @@ impl SessionInspector {}_ => None,};- if event.kind == Some(INCREMENTAL_SNAPSHOT) && event.source == Some(MOUSE_INTERACTION) {- if let Some(data) = event.data {- let [interaction, x, y] = fields(inner, data, [b"type", b"x", b"y"]);- if number(inner, interaction).is_some_and(|t| is_click(t as u8)) {- let position = number(inner, x).zip(number(inner, y));- match self.clicks.click(ts, position) {- ClickKind::Rage => stats.rage_clicks += 1,- ClickKind::AfterDead => stats.dead_clicks += 1,- ClickKind::Plain => {}- }- moment = Some("click");- }+ if event.kind == Some(INCREMENTAL_SNAPSHOT)+ && event.source == Some(MOUSE_INTERACTION)+ && number(inner, data.interaction).is_some_and(|t| is_click(t as u8))+ {+ let position = number(inner, data.x).zip(number(inner, data.y));+ match self.clicks.click(ts, position) {+ ClickKind::Rage => stats.rage_clicks += 1,+ ClickKind::AfterDead => stats.dead_clicks += 1,+ ClickKind::Plain => {}}+ moment = Some("click");}if let Some(kind) = moment {if block.moments.len() < MAX_MOMENTS_PER_BLOCK {diff --git a/rust/ai-research/replay-core-v4/src/inspect/stats.rs b/rust/ai-research/replay-core-v4/src/inspect/stats.rsindex aef9dcbdacdb..f9d626aabb30 100644--- a/rust/ai-research/replay-core-v4/src/inspect/stats.rs+++ b/rust/ai-research/replay-core-v4/src/inspect/stats.rs@@ -1,6 +1,8 @@//! One row of the v4 block index. Every value covers its block only, so a session row is a fold of its block rows://! [`fold_session`].+use super::features::SurfacingFeatures;+/// Fingerprints a block or session row keeps. A session row with this many has visited this many pages or more.pub const MAX_URL_FINGERPRINTS: usize = 10;/// Window IDs a block or session row keeps. A session row with this many has this many windows or more.@@ -86,6 +88,9 @@ pub struct BlockStats {pub written_jsonl_bytes: u64,/// The longest of those lines.pub max_written_event_bytes: u64,++ /// What ingestion's surfacing feature recorder records for the block's events as one batch.+ pub surfacing: SurfacingFeatures,}impl BlockStats {@@ -124,6 +129,7 @@ impl BlockStats {MAX_URL_FINGERPRINTS,);union(&mut self.window_ids, &later.window_ids, MAX_WINDOW_IDS);+ self.surfacing.merge(&later.surfacing);self.touch_input |= later.touch_input;self.set_viewport(later.viewport_ts_ms,@@ -207,6 +213,8 @@ fn min_option(a: Option<i64>, b: Option<i64>) -> Option<i64> {#[cfg(test)]mod tests {+ use std::ops::Range;+use super::*;fn block(first: i64, last: i64, host: &str, full_snapshots: u32, waiting: u32) -> BlockStats {@@ -281,6 +289,77 @@ mod tests {);}+ #[test]+ fn folds_the_surfacing_features_the_same_in_any_order_and_grouping() {+ let page = |n: usize| format!("{n:016x}");+ let block =+ |first: i64, clicks: u64, distance: i64, max_gap: i64, ids: Range<i64>| BlockStats {+ first_ts_ms: Some(first),+ surfacing: SurfacingFeatures {+ click_count: clicks,+ path_visit_counts: std::array::from_fn(|token| clicks * token as u64),+ mouse_distance_traveled_millipx: distance,+ mouse_sum_x_millipx: -distance,+ max_idle_gap_ms: max_gap,+ visited_url_fingerprints: ids.clone().map(|id| page(id as usize % 7)).collect(),+ click_target_ids: ids.collect(),+ ..SurfacingFeatures::default()+ },+ ..BlockStats::default()+ };+ let blocks = [+ block(1_000, 3, 1_500, 40_000, 0..1_000),+ block(2_000, 5, 2_500, 100, 500..1_500),+ block(64_000, 1, 0, 70_000, 1_400..2_000),+ BlockStats::default(),+ ];+ let whole = fold_session(blocks.to_vec()).surfacing;+ let mut pages: Vec<String> = (0..7).map(page).collect();+ pages.sort_unstable();+ assert_eq!(+ (+ whole.click_count,+ whole.path_visit_counts[2],+ whole.mouse_distance_traveled_millipx,+ whole.mouse_sum_x_millipx,+ whole.max_idle_gap_ms,+ whole.visited_url_fingerprints.clone(),+ whole.click_target_ids.clone(),+ ),+ // The sets keep every distinct value, past the cap of one block.+ (9, 18, 4_000, -4_000, 70_000, pages, (0..2_000).collect())+ );+ // Every order, each cut into two groups that fold first, as a reader that folds day by day does.+ for order in permutations(blocks.len()) {+ let ordered: Vec<BlockStats> = order.iter().map(|&i| blocks[i].clone()).collect();+ for cut in 0..=ordered.len() {+ let (earlier, later) = ordered.split_at(cut);+ let parts = vec![fold_session(earlier.to_vec()), fold_session(later.to_vec())];+ assert_eq!(+ fold_session(parts).surfacing,+ whole,+ "{order:?} cut at {cut}"+ );+ }+ }+ }++ fn permutations(n: usize) -> Vec<Vec<usize>> {+ if n == 0 {+ return vec![Vec::new()];+ }+ permutations(n - 1)+ .into_iter()+ .flat_map(|rest| {+ (0..n).map(move |at| {+ let mut order = rest.clone();+ order.insert(at, n - 1);+ order+ })+ })+ .collect()+ }+#[test]fn counts_the_waiting_events_of_a_block_that_overlaps_the_snapshots_block() {let main = block(1_000, 5_000, "a.example.com", 1, 2);diff --git a/rust/ai-research/replay-core-v4/src/scan/message.rs b/rust/ai-research/replay-core-v4/src/scan/message.rsindex 52dfb4d99e42..892935f1e483 100644--- a/rust/ai-research/replay-core-v4/src/scan/message.rs+++ b/rust/ai-research/replay-core-v4/src/scan/message.rs@@ -46,6 +46,12 @@ pub struct Event {pub source: Option<u8>,/// The event carries `cv` [`CV_VERSION`]: parts of its data are gzip or zstd stored as latin-1 codepoints.pub compressed: bool,+ /// `data` has a top-level `href` or `payload`, where `hrefFrom` in `rrweb-types.ts` looks for a page URL on any+ /// event, so that an inspector reads a large `data` again only when it can hold one.+ pub may_carry_page_url: bool,+ /// Where `data.positions` starts in [`Message::inner`]. The reader passes it on its way to `source`, so that an+ /// inspector reads a mouse move's positions without scanning its `data` again.+ pub positions_start: Option<u32>,/// The compressed strings decoded, once per event by [`Message::decode_compressed`]. `None` until then.pub expansion: Option<Expansion>,}@@ -219,6 +225,8 @@ fn read_event(inner: &[u8], span: Span) -> Result<Event, ReadError> {data: None,source: None,compressed: false,+ may_carry_page_url: false,+ positions_start: None,expansion: None,};for entry in scan::object_entries(inner, span)? {@@ -243,8 +251,14 @@ fn read_event(inner: &[u8], span: Span) -> Result<Event, ReadError> {if let Some(data) = event.data.filter(|&d| scan::is_object(inner, d)) {for entry in scan::object_entries(inner, data)? {let entry = entry?;- if !entry.key_escaped && &inner[entry.key.0..entry.key.1] == b"source" {- event.source = small_uint(inner, entry.value);+ if entry.key_escaped {+ continue;+ }+ match &inner[entry.key.0..entry.key.1] {+ b"source" => event.source = small_uint(inner, entry.value),+ b"href" | b"payload" => event.may_carry_page_url = true,+ b"positions" => event.positions_start = u32::try_from(entry.value.0).ok(),+ _ => {}}}}diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsindex dab9ac95a210..5dbdd24d6abb 100644--- a/rust/ai-research/replay-recorder-v4/src/batch.rs+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -98,6 +98,8 @@ pub struct Block {pub url_limit: Option<UrlLimit>,/// The image-slot srcset values of the block, which the writer rebuilds from the candidates it keeps.pub srcsets: Vec<Srcset>,+ /// The distinct pages of the block's surfacing features, which the writer stores as fingerprints.+ pub visited_urls: Vec<String>,/// The earliest Kafka timestamp of the block's messages, for the capture watermark of the index topic.pub captured_at_ms: Option<i64>,}@@ -511,6 +513,7 @@ fn inspect_session(remote_urls: inspected.remote_urls,url_limit: inspected.url_limit,srcsets: inspected.srcsets,+ visited_urls: inspected.features.visited_urls,captured_at_ms,}})diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 2c93ed8d8736..8fa8e2234e60 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -842,6 +842,7 @@ mod tests {remote_urls: Vec::new(),url_limit: None,srcsets: Vec::new(),+ visited_urls: Vec::new(),captured_at_ms: None,})};diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rsindex f930b118644e..535dc2826de0 100644--- a/rust/ai-research/replay-recorder-v4/src/flush.rs+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs@@ -226,6 +226,8 @@ pub async fn write_flush(stats[index].written_jsonl_bytes += block.counts.jsonl_bytes;stats[index].max_written_event_bytes = block.counts.max_line_bytes;stats[index].url_fingerprints = std::mem::take(&mut block.url_fingerprints);+ stats[index].surfacing.visited_url_fingerprints =+ std::mem::take(&mut block.visited_url_fingerprints);}// Before the index is written: building the jobs counts each block's image URLs into its stats. A dropped block's// images are not stored, so the fetcher does not fetch them.diff --git a/rust/ai-research/replay-recorder-v4/src/index.rs b/rust/ai-research/replay-recorder-v4/src/index.rsindex 675742df79a3..28b1c201e866 100644--- a/rust/ai-research/replay-recorder-v4/src/index.rs+++ b/rust/ai-research/replay-recorder-v4/src/index.rs@@ -16,6 +16,7 @@//! flush is whole when that many distinct `flush_row_index` values are present. Check this before any filter on//! sessions or teams.+use replay_core_v4::inspect::features::{SurfacingFeatures, PATH_TOKENS};use replay_core_v4::inspect::session::Moment;use replay_core_v4::inspect::stats::BlockStats;use serde::ser::{SerializeMap, Serializer};@@ -40,6 +41,10 @@ fn int32(value: u32) -> i32 {i32::try_from(value).unwrap_or(i32::MAX)}+fn int32_count(value: u64) -> i32 {+ i32::try_from(value).unwrap_or(i32::MAX)+}+fn serialize_int64<S: Serializer>(value: &u64, serializer: S) -> Result<S::Ok, S::Error> {serializer.serialize_i64(int64(*value))}@@ -102,6 +107,7 @@ fn serialize_stats<S: Serializer>(stats: &&BlockStats, serializer: S) -> Result<touch_moves,url_fingerprints,window_ids,+ surfacing,} = *stats;let mut map = serializer.serialize_map(None)?;macro_rules! columns {@@ -158,9 +164,134 @@ fn serialize_stats<S: Serializer>(stats: &&BlockStats, serializer: S) -> Result<image_urls);map.serialize_entry("host", host)?;+ serialize_surfacing(&mut map, surfacing)?;map.end()}+/// The surfacing feature columns, each named for its field after the `surfacing_` prefix.+fn serialize_surfacing<M: SerializeMap>(+ map: &mut M,+ surfacing: &SurfacingFeatures,+) -> Result<(), M::Error> {+ // Destructured in full, so a new field does not compile until it has a column here.+ let SurfacingFeatures {+ event_count,+ click_count,+ keypress_count,+ mouse_activity_count,+ rage_click_count,+ dead_click_count,+ quick_back_count,+ page_visit_count,+ text_selection_count,+ scroll_event_count,+ console_error_count,+ console_error_after_click_count,+ console_warn_count,+ network_request_count,+ network_failed_request_count,+ network_4xx_count,+ network_5xx_count,+ network_request_duration_count,+ mouse_position_count,+ mouse_direction_change_count,+ mouse_velocity_count,+ scroll_direction_reversal_count,+ rapid_scroll_reversal_count,+ scroll_to_top_count,+ inter_action_gap_count,+ long_idle_gap_count,+ backspace_count,+ mutation_count,+ viewport_resize_count,+ touch_event_count,+ selection_copy_count,+ path_visit_counts,+ mouse_sum_x_millipx,+ mouse_sum_x_squared_px2,+ mouse_sum_y_millipx,+ mouse_sum_y_squared_px2,+ mouse_distance_traveled_millipx,+ mouse_velocity_sum_px_per_s,+ mouse_velocity_sum_of_squares_px2_per_s2,+ total_scroll_magnitude_millipx,+ inter_action_gap_sum_ms,+ inter_action_gap_sum_of_squares_ms2,+ network_request_duration_sum_us,+ network_request_duration_sum_of_squares_ms2,+ max_scroll_y_millipx,+ max_idle_gap_ms,+ visited_url_fingerprints,+ click_target_ids,+ form_field_ids,+ } = surfacing;+ macro_rules! columns {+ ($convert:expr => $($field:ident),*) => {+ $( map.serialize_entry(concat!("surfacing_", stringify!($field)), &$convert($field))?; )*+ };+ }+ columns!(+ |v: &u64| int32_count(*v) =>+ event_count,+ click_count,+ keypress_count,+ mouse_activity_count,+ rage_click_count,+ dead_click_count,+ quick_back_count,+ page_visit_count,+ text_selection_count,+ scroll_event_count,+ console_error_count,+ console_error_after_click_count,+ console_warn_count,+ network_request_count,+ network_failed_request_count,+ network_4xx_count,+ network_5xx_count,+ network_request_duration_count,+ mouse_position_count,+ mouse_direction_change_count,+ mouse_velocity_count,+ scroll_direction_reversal_count,+ rapid_scroll_reversal_count,+ scroll_to_top_count,+ inter_action_gap_count,+ long_idle_gap_count,+ backspace_count,+ mutation_count,+ viewport_resize_count,+ touch_event_count,+ selection_copy_count+ );+ for ((_, name), count) in PATH_TOKENS.iter().zip(path_visit_counts) {+ map.serialize_entry(+ &format!("surfacing_{name}_path_visit_count"),+ &int32_count(*count),+ )?;+ }+ columns!(+ |v: &i64| safe(*v) =>+ mouse_sum_x_millipx,+ mouse_sum_x_squared_px2,+ mouse_sum_y_millipx,+ mouse_sum_y_squared_px2,+ mouse_distance_traveled_millipx,+ mouse_velocity_sum_px_per_s,+ mouse_velocity_sum_of_squares_px2_per_s2,+ total_scroll_magnitude_millipx,+ inter_action_gap_sum_ms,+ inter_action_gap_sum_of_squares_ms2,+ network_request_duration_sum_us,+ network_request_duration_sum_of_squares_ms2,+ max_scroll_y_millipx,+ max_idle_gap_ms+ );+ columns!(|v: &Vec<String>| v.clone() => visited_url_fingerprints);+ columns!(|v: &Vec<i64>| v.iter().copied().map(safe).collect::<Vec<_>>() => click_target_ids, form_field_ids);+ Ok(())+}+fn serialize_moments<S: Serializer>(moments: &&[Moment], serializer: S) -> Result<S::Ok, S::Error> {#[derive(Serialize)]struct Entry<'a> {@@ -367,6 +498,80 @@ mod tests {("window_ids", "LIST<UTF8>", false),("urls", "UTF8", true),("urls_truncated", "BOOLEAN", false),+ ("surfacing_event_count", "INT32", true),+ ("surfacing_click_count", "INT32", true),+ ("surfacing_keypress_count", "INT32", true),+ ("surfacing_mouse_activity_count", "INT32", true),+ ("surfacing_rage_click_count", "INT32", true),+ ("surfacing_dead_click_count", "INT32", true),+ ("surfacing_quick_back_count", "INT32", true),+ ("surfacing_page_visit_count", "INT32", true),+ ("surfacing_text_selection_count", "INT32", true),+ ("surfacing_scroll_event_count", "INT32", true),+ ("surfacing_console_error_count", "INT32", true),+ ("surfacing_console_error_after_click_count", "INT32", true),+ ("surfacing_console_warn_count", "INT32", true),+ ("surfacing_network_request_count", "INT32", true),+ ("surfacing_network_failed_request_count", "INT32", true),+ ("surfacing_network_4xx_count", "INT32", true),+ ("surfacing_network_5xx_count", "INT32", true),+ ("surfacing_network_request_duration_count", "INT32", true),+ ("surfacing_mouse_position_count", "INT32", true),+ ("surfacing_mouse_direction_change_count", "INT32", true),+ ("surfacing_mouse_velocity_count", "INT32", true),+ ("surfacing_scroll_direction_reversal_count", "INT32", true),+ ("surfacing_rapid_scroll_reversal_count", "INT32", true),+ ("surfacing_scroll_to_top_count", "INT32", true),+ ("surfacing_inter_action_gap_count", "INT32", true),+ ("surfacing_long_idle_gap_count", "INT32", true),+ ("surfacing_backspace_count", "INT32", true),+ ("surfacing_mutation_count", "INT32", true),+ ("surfacing_viewport_resize_count", "INT32", true),+ ("surfacing_touch_event_count", "INT32", true),+ ("surfacing_selection_copy_count", "INT32", true),+ ("surfacing_login_path_visit_count", "INT32", true),+ ("surfacing_signup_path_visit_count", "INT32", true),+ ("surfacing_checkout_path_visit_count", "INT32", true),+ ("surfacing_cart_path_visit_count", "INT32", true),+ ("surfacing_billing_path_visit_count", "INT32", true),+ ("surfacing_settings_path_visit_count", "INT32", true),+ ("surfacing_account_path_visit_count", "INT32", true),+ ("surfacing_error_path_visit_count", "INT32", true),+ ("surfacing_not_found_path_visit_count", "INT32", true),+ ("surfacing_admin_path_visit_count", "INT32", true),+ ("surfacing_dashboard_path_visit_count", "INT32", true),+ ("surfacing_onboarding_path_visit_count", "INT32", true),+ ("surfacing_cancel_path_visit_count", "INT32", true),+ ("surfacing_refund_path_visit_count", "INT32", true),+ ("surfacing_mouse_sum_x_millipx", "INT64", true),+ ("surfacing_mouse_sum_x_squared_px2", "INT64", true),+ ("surfacing_mouse_sum_y_millipx", "INT64", true),+ ("surfacing_mouse_sum_y_squared_px2", "INT64", true),+ ("surfacing_mouse_distance_traveled_millipx", "INT64", true),+ ("surfacing_mouse_velocity_sum_px_per_s", "INT64", true),+ (+ "surfacing_mouse_velocity_sum_of_squares_px2_per_s2",+ "INT64",+ true,+ ),+ ("surfacing_total_scroll_magnitude_millipx", "INT64", true),+ ("surfacing_inter_action_gap_sum_ms", "INT64", true),+ (+ "surfacing_inter_action_gap_sum_of_squares_ms2",+ "INT64",+ true,+ ),+ ("surfacing_network_request_duration_sum_us", "INT64", true),+ (+ "surfacing_network_request_duration_sum_of_squares_ms2",+ "INT64",+ true,+ ),+ ("surfacing_max_scroll_y_millipx", "INT64", true),+ ("surfacing_max_idle_gap_ms", "INT64", true),+ ("surfacing_visited_url_fingerprints", "LIST<UTF8>", true),+ ("surfacing_click_target_ids", "LIST<INT64>", true),+ ("surfacing_form_field_ids", "LIST<INT64>", true),];const IMAGE_COLUMNS: &[(&str, &str, bool)] = &[@@ -389,6 +594,9 @@ mod tests {"LIST<UTF8>" => value.as_array().is_some_and(|items| items.iter().all(|item| fits(item, "UTF8"))),+ "LIST<INT64>" => value+ .as_array()+ .is_some_and(|items| items.iter().all(|item| fits(item, "INT64"))),"MOMENTS" => value.as_array().is_some_and(|moments| {moments.iter().all(|moment| {let moment = moment.as_object().unwrap();@@ -434,6 +642,15 @@ mod tests {written_jsonl_bytes: u64::MAX,url_fingerprints: vec!["b08517cee3ff5ecc".to_owned()],window_ids: vec!["w1".to_owned(), "w2".to_owned()],+ surfacing: SurfacingFeatures {+ event_count: u64::MAX,+ path_visit_counts: std::array::from_fn(|token| token as u64),+ mouse_sum_x_millipx: i64::MIN,+ max_idle_gap_ms: 30_001,+ visited_url_fingerprints: vec!["0f1e2d3c4b5a6978".to_owned()],+ click_target_ids: vec![-1, 7],+ ..SurfacingFeatures::default()+ },..BlockStats::default()};let moments = [Moment {@@ -500,6 +717,30 @@ mod tests {&json!(["w1", "w2"]),]);+ assert_eq!(+ [+ &value["surfacing_event_count"],+ &value["surfacing_login_path_visit_count"],+ &value["surfacing_not_found_path_visit_count"],+ &value["surfacing_refund_path_visit_count"],+ &value["surfacing_mouse_sum_x_millipx"],+ &value["surfacing_max_idle_gap_ms"],+ &value["surfacing_visited_url_fingerprints"],+ &value["surfacing_click_target_ids"],+ &value["surfacing_form_field_ids"],+ ],+ [+ &json!(i32::MAX),+ &json!(0),+ &json!(8),+ &json!(13),+ &json!(-MAX_SAFE_INTEGER),+ &json!(30_001),+ &json!(["0f1e2d3c4b5a6978"]),+ &json!([-1, 7]),+ &json!([]),+ ]+ );assert_eq!(value["moments"],json!([{"kind": "full_snapshot", "window_id": "w1", "ts_ms": MAX_SAFE_INTEGER, "event_index": i32::MAX}])diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rsindex d014a5a7ba10..00475a835a1d 100644--- a/rust/ai-research/replay-recorder-v4/src/urls.rs+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs@@ -208,6 +208,7 @@ mod tests {remote_urls: Vec::new(),url_limit: None,srcsets: Vec::new(),+ visited_urls: Vec::new(),captured_at_ms: None,};let mut stats = BlockStats::default();diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rsindex 748af3dce228..9d8e9168dcf9 100644--- a/rust/ai-research/replay-recorder-v4/src/write.rs+++ b/rust/ai-research/replay-recorder-v4/src/write.rs@@ -84,6 +84,7 @@ pub struct CompressedBlock {pub images: Vec<Arc<Image>>,pub counts: WriterCounts,pub url_fingerprints: Vec<String>,+ pub visited_url_fingerprints: Vec<String>,}/// A block sealed for one object, with the images it moved out.@@ -94,6 +95,7 @@ pub struct SealedBlock {pub images: Vec<Arc<Image>>,pub counts: WriterCounts,pub url_fingerprints: Vec<String>,+ pub visited_url_fingerprints: Vec<String>,}fn unique_suffix(at: DateTime<Utc>) -> String {@@ -320,6 +322,7 @@ pub fn compress_block(images: images.into_iter().map(Arc::new).collect(),counts,url_fingerprints: url_fingerprints(&key.key, &block_urls(block)),+ visited_url_fingerprints: visited_url_fingerprints(&key.key, &block.visited_urls),})}@@ -342,6 +345,7 @@ pub fn seal_compressed(images: block.images.clone(),counts: block.counts,url_fingerprints: block.url_fingerprints.clone(),+ visited_url_fingerprints: block.visited_url_fingerprints.clone(),})}@@ -585,6 +589,16 @@ pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String>fingerprints}+/// The block's distinct visited pages for its surfacing features, as fingerprints under the session's fingerprint key,+/// sorted.+pub fn visited_url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String> {+ let key = url_fingerprint_key(session_key);+ let mut fingerprints: Vec<String> = urls.iter().map(|url| url_fingerprint(&key, url)).collect();+ fingerprints.sort_unstable();+ fingerprints.dedup();+ fingerprints+}+/// Numbers the rows of each partition in order and gives each row the count of its partition.fn number_rows(rows: &mut [BlockIndexRow<'_>]) {let mut counts: HashMap<(&str, i32), u32> = HashMap::new();@@ -771,6 +785,7 @@ mod tests {remote_urls: Vec::new(),url_limit: None,srcsets: Vec::new(),+ visited_urls: Vec::new(),captured_at_ms: None,}}@@ -1222,6 +1237,9 @@ mod tests {{"type": 4, "timestamp": 4, "data": {"href": "https://app.example.com/a"}},]);let mut blocks = [block("kept", events.clone()), block("again", events)];+ blocks[0].visited_urls = ["/b", "/a"]+ .map(|path| format!("https://app.example.com{path}"))+ .to_vec();let mut keys = FlushKeys::default();for session in ["kept", "again"] {let (id, key) = session_key(session);@@ -1265,6 +1283,9 @@ mod tests {sealed[0].url_fingerprints,[fingerprint("/a"), fingerprint("/b")]);+ let mut visited = [fingerprint("/a"), fingerprint("/b")];+ visited.sort_unstable();+ assert_eq!(sealed[0].visited_url_fingerprints, visited);let session_frames = session_keyed_frames(&sealed);let memory = Arc::new(MemoryStore::default());