Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai-research): record surfacing features per block in the v4 index (#125806)

Robbie22c3951a1bab
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts
index ea23f1a5109b..c68e6b861bf7 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-batcher.test.ts
@@ -25,7 +25,7 @@ function blockRow(sessionId: string): Record<string, unknown> {
const row: Record<string, unknown> = {}
for (const column of BLOCK_INDEX_COLUMNS) {
row[column.name] =
- 'repeated' in column || 'list' in column
+ 'repeated' in column || 'list' in column || 'elements' in column
? []
: column.optional
? null
@@ -214,6 +214,16 @@ describe('IndexBatcher', () => {
'invalid_row',
message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), window_ids: 'w1' }),
],
+ [
+ 'click target id that is not an integer',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), surfacing_click_target_ids: [1.5] }),
+ ],
+ [
+ 'visited url fingerprints that are not a list',
+ 'invalid_row',
+ message(1, 'blocks', '2026-10-15', { ...blockRow('s1'), surfacing_visited_url_fingerprints: 'x' }),
+ ],
[
'missing urls_truncated flag',
'invalid_row',
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts
index a988a0e37395..facd1232a1d5 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/index-columns.ts
@@ -30,7 +30,13 @@ export interface V4ListColumn {
list: ParquetType
}
-export type V4Column = V4LeafColumn | V4RepeatedColumn | V4ListColumn
+/** A nullable list of non-null values, in the 3-level LIST layout. */
+export interface V4NullableListColumn {
+ name: string
+ elements: ParquetType
+}
+
+export type V4Column = V4LeafColumn | V4RepeatedColumn | V4ListColumn | V4NullableListColumn
const required = (type: ParquetType, ...names: string[]): V4LeafColumn[] => names.map((name) => ({ name, type }))
const optional = (type: ParquetType, ...names: string[]): V4LeafColumn[] =>
@@ -108,6 +114,75 @@ export const BLOCK_INDEX_COLUMNS: readonly V4Column[] = [
{ name: 'window_ids', list: 'UTF8' },
{ name: 'urls', type: 'UTF8', optional: true, opaque: true },
...required('BOOLEAN', 'urls_truncated'),
+ // A recorder from before the surfacing columns sends rows without them, so they are nullable.
+ ...optional(
+ 'INT32',
+ 'surfacing_event_count',
+ 'surfacing_click_count',
+ 'surfacing_keypress_count',
+ 'surfacing_mouse_activity_count',
+ 'surfacing_rage_click_count',
+ 'surfacing_dead_click_count',
+ 'surfacing_quick_back_count',
+ 'surfacing_page_visit_count',
+ 'surfacing_text_selection_count',
+ 'surfacing_scroll_event_count',
+ 'surfacing_console_error_count',
+ 'surfacing_console_error_after_click_count',
+ 'surfacing_console_warn_count',
+ 'surfacing_network_request_count',
+ 'surfacing_network_failed_request_count',
+ 'surfacing_network_4xx_count',
+ 'surfacing_network_5xx_count',
+ 'surfacing_network_request_duration_count',
+ 'surfacing_mouse_position_count',
+ 'surfacing_mouse_direction_change_count',
+ 'surfacing_mouse_velocity_count',
+ 'surfacing_scroll_direction_reversal_count',
+ 'surfacing_rapid_scroll_reversal_count',
+ 'surfacing_scroll_to_top_count',
+ 'surfacing_inter_action_gap_count',
+ 'surfacing_long_idle_gap_count',
+ 'surfacing_backspace_count',
+ 'surfacing_mutation_count',
+ 'surfacing_viewport_resize_count',
+ 'surfacing_touch_event_count',
+ 'surfacing_selection_copy_count',
+ 'surfacing_login_path_visit_count',
+ 'surfacing_signup_path_visit_count',
+ 'surfacing_checkout_path_visit_count',
+ 'surfacing_cart_path_visit_count',
+ 'surfacing_billing_path_visit_count',
+ 'surfacing_settings_path_visit_count',
+ 'surfacing_account_path_visit_count',
+ 'surfacing_error_path_visit_count',
+ 'surfacing_not_found_path_visit_count',
+ 'surfacing_admin_path_visit_count',
+ 'surfacing_dashboard_path_visit_count',
+ 'surfacing_onboarding_path_visit_count',
+ 'surfacing_cancel_path_visit_count',
+ 'surfacing_refund_path_visit_count'
+ ),
+ ...optional(
+ 'INT64',
+ 'surfacing_mouse_sum_x_millipx',
+ 'surfacing_mouse_sum_x_squared_px2',
+ 'surfacing_mouse_sum_y_millipx',
+ 'surfacing_mouse_sum_y_squared_px2',
+ 'surfacing_mouse_distance_traveled_millipx',
+ 'surfacing_mouse_velocity_sum_px_per_s',
+ 'surfacing_mouse_velocity_sum_of_squares_px2_per_s2',
+ 'surfacing_total_scroll_magnitude_millipx',
+ 'surfacing_inter_action_gap_sum_ms',
+ 'surfacing_inter_action_gap_sum_of_squares_ms2',
+ 'surfacing_network_request_duration_sum_us',
+ 'surfacing_network_request_duration_sum_of_squares_ms2',
+ 'surfacing_max_scroll_y_millipx',
+ 'surfacing_max_idle_gap_ms'
+ ),
+ { name: 'surfacing_visited_url_fingerprints', elements: 'UTF8' },
+ { name: 'surfacing_click_target_ids', elements: 'INT64' },
+ { name: 'surfacing_form_field_ids', elements: 'INT64' },
]
export const INDEX_COLUMNS: Record<V4Index, readonly V4Column[]> = {
@@ -164,6 +239,14 @@ function selectColumns(value: unknown, columns: readonly V4Column[], prefix = ''
return invalid
}
row[column.name] = field
+ } else if ('elements' in column) {
+ if (field === null || field === undefined) {
+ row[column.name] = null
+ } else if (Array.isArray(field) && field.every((item) => isEncodableValue(column.elements, item))) {
+ row[column.name] = field
+ } else {
+ return invalid
+ }
} else if (field === null || field === undefined) {
if (!column.optional) {
return invalid
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts
index 17c2535e1337..f04e04931009 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.test.ts
@@ -129,6 +129,74 @@ describe('indexRowsToParquet', () => {
'REPEATED BYTE_ARRAY window_ids UTF8',
'OPTIONAL BYTE_ARRAY urls UTF8',
'REQUIRED BOOLEAN urls_truncated',
+ 'OPTIONAL INT32 surfacing_event_count',
+ 'OPTIONAL INT32 surfacing_click_count',
+ 'OPTIONAL INT32 surfacing_keypress_count',
+ 'OPTIONAL INT32 surfacing_mouse_activity_count',
+ 'OPTIONAL INT32 surfacing_rage_click_count',
+ 'OPTIONAL INT32 surfacing_dead_click_count',
+ 'OPTIONAL INT32 surfacing_quick_back_count',
+ 'OPTIONAL INT32 surfacing_page_visit_count',
+ 'OPTIONAL INT32 surfacing_text_selection_count',
+ 'OPTIONAL INT32 surfacing_scroll_event_count',
+ 'OPTIONAL INT32 surfacing_console_error_count',
+ 'OPTIONAL INT32 surfacing_console_error_after_click_count',
+ 'OPTIONAL INT32 surfacing_console_warn_count',
+ 'OPTIONAL INT32 surfacing_network_request_count',
+ 'OPTIONAL INT32 surfacing_network_failed_request_count',
+ 'OPTIONAL INT32 surfacing_network_4xx_count',
+ 'OPTIONAL INT32 surfacing_network_5xx_count',
+ 'OPTIONAL INT32 surfacing_network_request_duration_count',
+ 'OPTIONAL INT32 surfacing_mouse_position_count',
+ 'OPTIONAL INT32 surfacing_mouse_direction_change_count',
+ 'OPTIONAL INT32 surfacing_mouse_velocity_count',
+ 'OPTIONAL INT32 surfacing_scroll_direction_reversal_count',
+ 'OPTIONAL INT32 surfacing_rapid_scroll_reversal_count',
+ 'OPTIONAL INT32 surfacing_scroll_to_top_count',
+ 'OPTIONAL INT32 surfacing_inter_action_gap_count',
+ 'OPTIONAL INT32 surfacing_long_idle_gap_count',
+ 'OPTIONAL INT32 surfacing_backspace_count',
+ 'OPTIONAL INT32 surfacing_mutation_count',
+ 'OPTIONAL INT32 surfacing_viewport_resize_count',
+ 'OPTIONAL INT32 surfacing_touch_event_count',
+ 'OPTIONAL INT32 surfacing_selection_copy_count',
+ 'OPTIONAL INT32 surfacing_login_path_visit_count',
+ 'OPTIONAL INT32 surfacing_signup_path_visit_count',
+ 'OPTIONAL INT32 surfacing_checkout_path_visit_count',
+ 'OPTIONAL INT32 surfacing_cart_path_visit_count',
+ 'OPTIONAL INT32 surfacing_billing_path_visit_count',
+ 'OPTIONAL INT32 surfacing_settings_path_visit_count',
+ 'OPTIONAL INT32 surfacing_account_path_visit_count',
+ 'OPTIONAL INT32 surfacing_error_path_visit_count',
+ 'OPTIONAL INT32 surfacing_not_found_path_visit_count',
+ 'OPTIONAL INT32 surfacing_admin_path_visit_count',
+ 'OPTIONAL INT32 surfacing_dashboard_path_visit_count',
+ 'OPTIONAL INT32 surfacing_onboarding_path_visit_count',
+ 'OPTIONAL INT32 surfacing_cancel_path_visit_count',
+ 'OPTIONAL INT32 surfacing_refund_path_visit_count',
+ 'OPTIONAL INT64 surfacing_mouse_sum_x_millipx',
+ 'OPTIONAL INT64 surfacing_mouse_sum_x_squared_px2',
+ 'OPTIONAL INT64 surfacing_mouse_sum_y_millipx',
+ 'OPTIONAL INT64 surfacing_mouse_sum_y_squared_px2',
+ 'OPTIONAL INT64 surfacing_mouse_distance_traveled_millipx',
+ 'OPTIONAL INT64 surfacing_mouse_velocity_sum_px_per_s',
+ 'OPTIONAL INT64 surfacing_mouse_velocity_sum_of_squares_px2_per_s2',
+ 'OPTIONAL INT64 surfacing_total_scroll_magnitude_millipx',
+ 'OPTIONAL INT64 surfacing_inter_action_gap_sum_ms',
+ 'OPTIONAL INT64 surfacing_inter_action_gap_sum_of_squares_ms2',
+ 'OPTIONAL INT64 surfacing_network_request_duration_sum_us',
+ 'OPTIONAL INT64 surfacing_network_request_duration_sum_of_squares_ms2',
+ 'OPTIONAL INT64 surfacing_max_scroll_y_millipx',
+ 'OPTIONAL INT64 surfacing_max_idle_gap_ms',
+ 'OPTIONAL group surfacing_visited_url_fingerprints LIST',
+ 'REPEATED group surfacing_visited_url_fingerprints.list',
+ 'REQUIRED BYTE_ARRAY surfacing_visited_url_fingerprints.list.element UTF8',
+ 'OPTIONAL group surfacing_click_target_ids LIST',
+ 'REPEATED group surfacing_click_target_ids.list',
+ 'REQUIRED INT64 surfacing_click_target_ids.list.element',
+ 'OPTIONAL group surfacing_form_field_ids LIST',
+ 'REPEATED group surfacing_form_field_ids.list',
+ 'REQUIRED INT64 surfacing_form_field_ids.list.element',
],
}
@@ -164,7 +232,7 @@ describe('indexRowsToParquet', () => {
? moments
: 'list' in column
? windowIds
- : column.optional
+ : 'elements' in column || column.optional
? null
: column.type === 'UTF8'
? 'x'
@@ -207,6 +275,41 @@ describe('indexRowsToParquet', () => {
])
})
+ it('reads the surfacing lists back as 3-level lists, and a row without them as null', async () => {
+ const surfacing = {
+ surfacing_click_count: 3,
+ surfacing_mouse_sum_x_millipx: -(2 ** 53 - 1),
+ surfacing_visited_url_fingerprints: ['0f1e2d3c4b5a6978', 'b08517cee3ff5ecc'],
+ surfacing_click_target_ids: [-1, 2 ** 53 - 1],
+ surfacing_form_field_ids: [],
+ }
+ const { rows } = await describeFile(
+ await indexRowsToParquet('blocks', [{ ...block([], []), ...surfacing }, block([], [])], 1_000)
+ )
+ const read = (rows as Record<string, unknown>[]).map((row) =>
+ Object.fromEntries(Object.keys(surfacing).map((name) => [name, row[name] ?? null]))
+ )
+ expect(read).toEqual([
+ {
+ surfacing_click_count: 3,
+ surfacing_mouse_sum_x_millipx: -(2n ** 53n - 1n),
+ surfacing_visited_url_fingerprints: {
+ list: surfacing.surfacing_visited_url_fingerprints.map((element) => ({ element })),
+ },
+ surfacing_click_target_ids: { list: [{ element: -1n }, { element: 2n ** 53n - 1n }] },
+ // parquetjs reads an empty list as a null `list`; Arrow readers see an empty list.
+ surfacing_form_field_ids: { list: null },
+ },
+ {
+ surfacing_click_count: null,
+ surfacing_mouse_sum_x_millipx: null,
+ surfacing_visited_url_fingerprints: null,
+ surfacing_click_target_ids: null,
+ surfacing_form_field_ids: null,
+ },
+ ])
+ })
+
it('reads moments back as a 3-level list of structs and the string lists as lists', async () => {
const moments = [
{ kind: 'click', window_id: 'w1', ts_ms: 1_000, event_index: 2 ** 31 - 1 },
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts
index b20557ff1542..77057e6588a4 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/parquet-sink-v4/parquet-writer.ts
@@ -29,6 +29,14 @@ function schemaDef(columns: readonly V4Column[]): SchemaDef {
type: 'LIST',
fields: { list: { repeated: true, fields: { element: { fields: schemaDef(column.fields) } } } },
}
+ } else if ('elements' in column) {
+ def[column.name] = {
+ type: 'LIST',
+ optional: true,
+ fields: {
+ list: { repeated: true, fields: { element: { type: column.elements, compression: 'ZSTD' } } },
+ },
+ }
} else if ('list' in column) {
def[column.name] = { type: column.list, repeated: true, compression: 'ZSTD' }
} else {
@@ -80,6 +88,11 @@ function toRecord(row: V4IndexRow, columns: readonly V4Column[]): Record<string,
}
} else if ('list' in column) {
record[column.name] = (value as unknown[]).map((item) => toValue(column.list, item))
+ } else if ('elements' in column) {
+ record[column.name] =
+ value === null || value === undefined
+ ? null
+ : { list: (value as unknown[]).map((item) => ({ element: toValue(column.elements, item) })) }
} else if (value === null || value === undefined) {
record[column.name] = null
} else {
diff --git a/products/ai_training/docs/replay-data.md b/products/ai_training/docs/replay-data.md
index 2f5a04a69ac9..93a477ddb78f 100644
--- a/products/ai_training/docs/replay-data.md
+++ b/products/ai_training/docs/replay-data.md
@@ -498,7 +498,7 @@ The recorder does not write the two index prefixes itself, as in v3:
- After a flush stores its packs and blocks, the recorder sends one JSON record per index row to `ai_research_v4_index`. The record's keys are the Parquet column names.
- The recorder commits without waiting for delivery. A row that is not delivered leaves its block or image stored but not in the index.
- The v4 Parquet sink reads the topic, groups the rows by index and write day, and writes one file per group and flush as `{ms}-{random}.parquet`. A flush runs every 10 minutes, or at 128 MiB of rows or 1,000,000 rows. The sink commits its offsets only after the files are stored.
-- Index files use four column types: `INT32`, `INT64`, `UTF8` and `BOOLEAN`. Integers are signed. The recorder caps an `INT32` value at the type's range and an `INT64` value at ±(2^53 − 1), so every value is a safe JavaScript integer. It ignores an event timestamp outside the JavaScript date range. Byte counts, offsets and times are `INT64`, the other counts `INT32`. `moments` is a 3-level `LIST` (`moments`, then the repeated group `list`, then the struct `element`), so Athena, Arrow readers and DuckDB see a non-null list of structs. Files written before the sink used that layout hold `moments` as a bare repeated group whose item is named `moments`. Arrow readers and DuckDB read those as the same list of structs, but Athena reads the column as null, because Trino reads a bare repeated group of more than one field as a missing column. Those files expire with the bucket's lifecycle. `url_fingerprints` and `window_ids` are lists of `UTF8` with no null items. Row groups hold 131072 rows, and every column is zstd-compressed at level 9. Older files can still be snappy-compressed and in arrival order until they expire; Parquet records the codec per column chunk, so a reader handles both. The sink writes no statistics for `urls`, because a sealed value has no useful order.
+- Index files use four column types: `INT32`, `INT64`, `UTF8` and `BOOLEAN`. Integers are signed. The recorder caps an `INT32` value at the type's range and an `INT64` value at ±(2^53 − 1), so every value is a safe JavaScript integer. It ignores an event timestamp outside the JavaScript date range. Byte counts, offsets and times are `INT64`, the other counts `INT32`. `moments` is a 3-level `LIST` (`moments`, then the repeated group `list`, then the struct `element`), so Athena, Arrow readers and DuckDB see a non-null list of structs. Files written before the sink used that layout hold `moments` as a bare repeated group whose item is named `moments`. Arrow readers and DuckDB read those as the same list of structs, but Athena reads the column as null, because Trino reads a bare repeated group of more than one field as a missing column. Those files expire with the bucket's lifecycle. `url_fingerprints` and `window_ids` are lists of `UTF8` with no null items. The three [surfacing](#surfacing-features) sets are nullable 3-level `LIST`s of `UTF8` or `INT64` with no null items. Row groups hold 131072 rows, and every column is zstd-compressed at level 9. Older files can still be snappy-compressed and in arrival order until they expire; Parquet records the codec per column chunk, so a reader handles both. The sink writes no statistics for `urls`, because a sealed value has no useful order.
- The sink can write a row twice after a restart. The copies have the same `flush_id`, so the rule below does not remove them: readers first drop rows that repeat within one `flush_id`.
### Block index
@@ -531,6 +531,7 @@ A session row is a fold over its block rows: sort them by `first_ts_ms`, then me
Counts sum, times take the minimum or maximum, flags take the logical or, `host` takes the first value and the viewport takes the one with the highest `viewport_ts_ms`.
The DOM sizes `dom_nodes` and `dom_text_chars` and `max_written_event_bytes` take the maximum, because they describe a state or a single event, not an amount.
`url_fingerprints` and `window_ids` take the ordered union, capped at 10 and 32 entries. A session at the cap has that many or more.
+The [surfacing](#surfacing-features) sums add, its two maxima take the maximum, and its three sets take the union with no cap.
The flush columns describe a row, not a session, so the fold does not carry them.
`events_before_first_full_snapshot` sums, because the recorder stops counting it once the session has a decoded full snapshot.
`max_gap_ms` also counts the gap between one block's last event and the next block's first event. When blocks overlap, for example the main and overflow topics, it is an upper bound.
@@ -553,6 +554,7 @@ This holds for the blocks that one recorder process inspected. After a rebalance
| Images | `images_extracted`, `image_bytes`, `images_inline`, `image_urls` |
| Moments | `moments`: a list of `{kind, window_id, ts_ms, event_index}` for full snapshots, navigations, clicks and large DOM changes, and `moments_truncated` |
| Sealed | `urls`: the visited URLs as a sealed index value, `urls_truncated`, and `url_fingerprints` |
+| Surfacing | The `surfacing_` columns: what ingestion's surfacing feature recorder records for the block as one batch. See [Surfacing features](#surfacing-features) |
- `full_snapshots` counts only snapshots that decoded.
- `events_before_first_full_snapshot` counts the session's events before its first decoded full snapshot.
@@ -568,6 +570,29 @@ This holds for the blocks that one recorder process inspected. After a rebalance
- `url_fingerprints` holds up to 10 distinct page URLs of the block as the first 16 hexadecimal characters of an HMAC-SHA256 of the URL. The key is derived from the session key with HKDF-SHA256, empty salt and info `ml-url-fingerprint-key`, so fingerprints can be compared only within one session. They count distinct URLs, where `navigations` counts URL changes.
- `cold_start` is true when the recorder first sees a session after it started, for example after a partition rebalance, or after it dropped the session's DOM state to stay within its memory budget. Its playability columns then describe only the part of the session the recorder saw.
+#### Surfacing features
+
+The `surfacing_` columns let a sampler score whole sessions with the surfacing booster without opening a block.
+Each block row holds what ingestion's `SessionFeatureRecorder` (`session-feature-recorder.ts`) records for one batch, and the recorder computes it with the same definitions.
+Its state starts again at each block, as ingestion's does at each batch: a block is one session's events from one Kafka partition in one recorder flush, as a batch is.
+After the prefix, each column has the name of the `session_replay_features` column it matches, with a unit for the sums that are not counts.
+A session's values are the fold of its block rows, as `_AGGREGATED_STATS_FRAGMENT` (`posthog/temporal/session_replay/surfacing_scoring_sweep/sql.py`) folds the batch rows, and `fold_session` in `replay-core-v4` implements it.
+
+| Kind | Columns |
+| ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Counts | `INT32`: `surfacing_{name}_count` for the 31 counters of `session_replay_features`, from `event_count` to `selection_copy_count`, and `surfacing_{token}_path_visit_count` for the 14 path tokens, where `404` is `not_found` |
+| Sums | `INT64`: `surfacing_mouse_sum_x_millipx`, `surfacing_mouse_sum_y_millipx`, `surfacing_mouse_sum_x_squared_px2`, `surfacing_mouse_sum_y_squared_px2`, `surfacing_mouse_distance_traveled_millipx`, `surfacing_mouse_velocity_sum_px_per_s`, `surfacing_mouse_velocity_sum_of_squares_px2_per_s2`, `surfacing_total_scroll_magnitude_millipx`, `surfacing_inter_action_gap_sum_ms`, `surfacing_inter_action_gap_sum_of_squares_ms2`, `surfacing_network_request_duration_sum_us`, `surfacing_network_request_duration_sum_of_squares_ms2` |
+| Maxima | `INT64`: `surfacing_max_scroll_y_millipx`, `surfacing_max_idle_gap_ms` |
+| Sets | `surfacing_visited_url_fingerprints` (`UTF8`), `surfacing_click_target_ids` and `surfacing_form_field_ids` (`INT64`) |
+
+- Ingestion stores a float sum as a double. The index stores it as an integer in the unit at the end of its name, rounded once per block, so that a fold is exact in any order. `millipx` is 1/1000 of a pixel and `us` is a microsecond. Ingestion's mouse velocity is in pixels per millisecond, so `surfacing_mouse_velocity_sum_px_per_s` is 1000 times its sum and `surfacing_mouse_velocity_sum_of_squares_px2_per_s2` 1000000 times its sum of squares.
+- The sets hold distinct values, sorted. The click targets and form fields are rrweb node IDs. The visited pages are fingerprints in the format of `url_fingerprints`, under the same key: ingestion stores an md5 of each URL instead. Each set keeps the first 1000 values of a block, as ingestion keeps 1000 per batch. The sweep counts a set with `uniqCombinedMerge(12)`, and the length of the folded set is the exact count.
+- A page URL is the trimmed `data.href`, or else `data.payload.href`, of any event, as `hrefFrom` in `rrweb-types.ts` reads it. The path token counts compare the segments of its path with a fixed list of words, so they hold no other part of a URL.
+- The recorder skips an event without a positive timestamp, which ingestion's parse step drops before its recorder. The booster's rates divide by the session duration, which a sampler takes from `first_ts_ms` and `last_ts_ms`. Those also count events with a timestamp of zero or less.
+- The columns are null in a row from a recorder that predates them, and absent from files written before them. A sampler falls back to the session's live score for such a session.
+- Ingestion corrects the event times of a message by the sender's clock skew, rounded to 5 minutes, and the recorder does not. A block whose messages got different corrections can therefore differ in its time-based values.
+- A field of an unexpected JSON type counts as missing, where JavaScript can coerce it or produce NaN, for example a mouse position without coordinates. Ingestion's parse step lets such fields through.
+
#### Watermark files
After a flush, the sink writes one JSON file per index and Kafka partition of `ai_research_v4_index`, so a reader can learn from the bucket how far the index is complete:
diff --git a/rust/ai-research/replay-core-v4/src/bytescan.rs b/rust/ai-research/replay-core-v4/src/bytescan.rs
index 5731f63fb5a2..2479dbd14411 100644
--- a/rust/ai-research/replay-core-v4/src/bytescan.rs
+++ b/rust/ai-research/replay-core-v4/src/bytescan.rs
@@ -351,6 +351,108 @@ impl<'b> Iterator for ArrayItems<'b> {
}
}
+/// The top-level `keys` of each object item of the array that starts at `start`, found in the same pass that finds
+/// where the item ends, so that an array of small objects is scanned once. Items that are not objects are skipped. As
+/// in a parse, the last of a repeated key wins, and an escaped key is not one of `keys`.
+pub fn array_object_fields<'b, const N: usize>(
+ bytes: &'b [u8],
+ start: usize,
+ keys: [&'b [u8]; N],
+) -> Result<ArrayObjectFields<'b, N>> {
+ if bytes.get(start) != Some(&b'[') {
+ return Err(ScanError("not an array"));
+ }
+ Ok(ArrayObjectFields {
+ bytes,
+ keys,
+ pos: start + 1,
+ done: false,
+ first: true,
+ })
+}
+
+pub struct ArrayObjectFields<'b, const N: usize> {
+ bytes: &'b [u8],
+ keys: [&'b [u8]; N],
+ pos: usize,
+ done: bool,
+ first: bool,
+}
+
+impl<const N: usize> ArrayObjectFields<'_, N> {
+ /// Reads the object at `pos`, and returns its end and its fields.
+ fn object(&self, mut pos: usize) -> Result<(usize, [Option<Span>; N])> {
+ let bytes = self.bytes;
+ let mut found = [None; N];
+ pos = skip_ws(bytes, pos + 1);
+ if bytes.get(pos) == Some(&b'}') {
+ return Ok((pos + 1, found));
+ }
+ loop {
+ if bytes.get(pos) != Some(&b'"') {
+ return Err(ScanError("expected object key"));
+ }
+ let key_end = skip_string(bytes, pos)?;
+ let key = &bytes[pos + 1..key_end - 1];
+ pos = skip_ws(bytes, key_end);
+ if bytes.get(pos) != Some(&b':') {
+ return Err(ScanError("expected ':' after object key"));
+ }
+ let value = locate_value(bytes, pos + 1)?;
+ if let Some(index) = self.keys.iter().position(|&k| k == key) {
+ found[index] = Some(value);
+ }
+ pos = skip_ws(bytes, value.1);
+ match bytes.get(pos) {
+ Some(b',') => pos = skip_ws(bytes, pos + 1),
+ Some(b'}') => return Ok((pos + 1, found)),
+ _ => return Err(ScanError("expected ',' between object entries")),
+ }
+ }
+ }
+}
+
+impl<const N: usize> Iterator for ArrayObjectFields<'_, N> {
+ type Item = Result<[Option<Span>; N]>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ while !self.done {
+ let bytes = self.bytes;
+ let mut pos = skip_ws(bytes, self.pos);
+ if bytes.get(pos) == Some(&b']') {
+ self.done = true;
+ return None;
+ }
+ if !self.first {
+ if bytes.get(pos) != Some(&b',') {
+ self.done = true;
+ return Some(Err(ScanError("expected ',' between array items")));
+ }
+ pos = skip_ws(bytes, pos + 1);
+ }
+ self.first = false;
+ let item = if bytes.get(pos) == Some(&b'{') {
+ self.object(pos).map(|(end, found)| (end, Some(found)))
+ } else {
+ locate_value(bytes, pos).map(|span| (span.1, None))
+ };
+ match item {
+ Ok((end, found)) => {
+ self.pos = end;
+ if let Some(found) = found {
+ return Some(Ok(found));
+ }
+ }
+ Err(error) => {
+ self.done = true;
+ return Some(Err(error));
+ }
+ }
+ }
+ None
+ }
+}
+
/// Parse a scanned scalar span as a JSON number, mirroring what `JSON.parse` would produce (an f64).
/// Returns `None` for non-numbers. Accepts the full JSON number grammar (fractions, exponents); the
/// charset pre-check keeps Rust-isms like `inf`/`NaN` out.
@@ -360,6 +462,19 @@ pub fn parse_number(bytes: &[u8], span: Span) -> Option<f64> {
if first != b'-' && !first.is_ascii_digit() {
return None;
}
+ // Most numbers in a recording are small integers, which an f64 holds exactly, so they skip the float parser.
+ let digits = raw.strip_prefix(b"-").unwrap_or(raw);
+ if !digits.is_empty() && digits.len() <= 15 && digits.iter().all(u8::is_ascii_digit) {
+ let value = digits
+ .iter()
+ .fold(0i64, |value, digit| value * 10 + i64::from(digit - b'0'))
+ as f64;
+ return Some(if digits.len() < raw.len() {
+ -value
+ } else {
+ value
+ });
+ }
if !raw
.iter()
.all(|b| b.is_ascii_digit() || matches!(b, b'-' | b'+' | b'.' | b'e' | b'E'))
diff --git a/rust/ai-research/replay-core-v4/src/inspect/clicks.rs b/rust/ai-research/replay-core-v4/src/inspect/clicks.rs
index 1c87f5f5cab3..c01649a42811 100644
--- a/rust/ai-research/replay-core-v4/src/inspect/clicks.rs
+++ b/rust/ai-research/replay-core-v4/src/inspect/clicks.rs
@@ -41,7 +41,9 @@ impl ClickTracker {
self.run = 1;
return ClickKind::Plain;
};
- if ts - last_ts < RUN_WINDOW_MS && (x - last_x).hypot(y - last_y) < RUN_DISTANCE_PX {
+ let (dx, dy) = (x - last_x, y - last_y);
+ // `Math.sqrt(dx * dx + dy * dy)`, which can round differently from `hypot`.
+ if ts - last_ts < RUN_WINDOW_MS && (dx * dx + dy * dy).sqrt() < RUN_DISTANCE_PX {
self.run += 1;
return if self.run >= RAGE_RUN_LENGTH {
ClickKind::Rage
diff --git a/rust/ai-research/replay-core-v4/src/inspect/events.rs b/rust/ai-research/replay-core-v4/src/inspect/events.rs
index b5066041ea16..120996c19b38 100644
--- a/rust/ai-research/replay-core-v4/src/inspect/events.rs
+++ b/rust/ai-research/replay-core-v4/src/inspect/events.rs
@@ -1,10 +1,12 @@
//! Event-level counts for one block: activity, interactions, navigation, console and network.
+use std::borrow::Cow;
use std::collections::BTreeSet;
use std::ops::Bound;
use crate::bytescan::{self as scan, Span};
+use super::features::js_trim;
use super::rrweb::*;
use super::stats::BlockStats;
use crate::scan::message::Event;
@@ -14,7 +16,8 @@ const ACTIVITY_THRESHOLD_MS: f64 = 5000.0;
/// The distinct times an open inactive segment keeps. Past this the oldest go, so a late active event older than all
/// that remain changes nothing. Out-of-order input then costs a log-time insert, and the state stays small.
const MAX_INACTIVE_TIMES: usize = 1024;
-/// The status field of a `posthog/network@1` payload, as `session-feature-recorder.ts` reads it.
+/// The duration and status fields of a `posthog/network@1` payload, as `session-feature-recorder.ts` reads them.
+const POSTHOG_NETWORK_DURATION_KEY: &[u8] = b"39";
const POSTHOG_NETWORK_STATUS_KEY: &[u8] = b"21";
/// The longest DNS name. A longer host resolves nowhere, so it names no site.
const MAX_HOST_BYTES: usize = 253;
@@ -73,11 +76,129 @@ pub(crate) fn number(bytes: &[u8], span: Option<Span>) -> Option<f64> {
span.and_then(|span| scan::parse_number(bytes, span))
}
-pub(crate) fn text<'b>(bytes: &'b [u8], span: Option<Span>) -> Option<std::borrow::Cow<'b, str>> {
+pub(crate) fn text<'b>(bytes: &'b [u8], span: Option<Span>) -> Option<Cow<'b, str>> {
span.filter(|&span| scan::is_string(bytes, span))
.and_then(|span| scan::unescape(bytes, span).ok())
}
+/// One request of an `rrweb/network@1` event, as `session-feature-recorder.ts` reads it.
+#[derive(Debug, Clone, Copy)]
+pub struct NetworkRequest {
+ pub duration: Option<f64>,
+ /// `req.status ?? req.responseStatus`
+ pub status: Option<f64>,
+}
+
+/// The fields of an event's `data`, and of `data.payload`, that the counters and the feature recorder read. Each object
+/// is read once per event, for all of them.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct DataFields {
+ /// `type`: the kind of a mouse interaction.
+ pub interaction: Option<Span>,
+ pub x: Option<Span>,
+ pub y: Option<Span>,
+ pub id: Option<Span>,
+ pub text: Option<Span>,
+ pub plugin: Option<Span>,
+ pub payload: Option<Span>,
+ pub href: Option<Span>,
+ pub tag: Option<Span>,
+ pub width: Option<Span>,
+ pub height: Option<Span>,
+ pub payload_href: Option<Span>,
+ pub level: Option<Span>,
+ pub network_duration: Option<Span>,
+ pub network_status: Option<Span>,
+ /// The requests of an `rrweb/network@1` event, which [`DataFields::for_each_network_request`] reads.
+ network_requests: Option<Span>,
+}
+
+impl DataFields {
+ /// Reads the fields of the events that use them. A snapshot, a mutation or a mouse move's positions are large and
+ /// use none of them, so their `data` is read only when it has a key where `hrefFrom` looks for a page.
+ pub fn read(inner: &[u8], event: &Event) -> Self {
+ let Some(data) = event.data else {
+ return Self::default();
+ };
+ let used = match event.kind {
+ Some(META | CUSTOM | PLUGIN) => true,
+ Some(INCREMENTAL_SNAPSHOT) => matches!(
+ event.source,
+ Some(MOUSE_INTERACTION | SCROLL | VIEWPORT_RESIZE | INPUT)
+ ),
+ _ => false,
+ };
+ if !used && !event.may_carry_page_url {
+ return Self::default();
+ }
+ let [interaction, x, y, id, text, plugin, payload, href, tag, width, height] = fields(
+ inner,
+ data,
+ [
+ b"type", b"x", b"y", b"id", b"text", b"plugin", b"payload", b"href", b"tag",
+ b"width", b"height",
+ ],
+ );
+ let [payload_href, level, network_requests, network_duration, network_status] = payload
+ .map(|payload| {
+ fields(
+ inner,
+ payload,
+ [
+ b"href",
+ b"level",
+ b"requests",
+ POSTHOG_NETWORK_DURATION_KEY,
+ POSTHOG_NETWORK_STATUS_KEY,
+ ],
+ )
+ })
+ .unwrap_or_default();
+ let network_plugin = event.kind == Some(PLUGIN)
+ && self::text(inner, plugin).as_deref() == Some("rrweb/network@1");
+ Self {
+ interaction,
+ x,
+ y,
+ id,
+ text,
+ plugin,
+ payload,
+ href,
+ tag,
+ width,
+ height,
+ payload_href,
+ level,
+ network_duration,
+ network_status,
+ network_requests: network_requests.filter(|_| network_plugin),
+ }
+ }
+
+ /// Calls `each` with every request of an `rrweb/network@1` event, in order, as one pass reads them. The requests
+ /// are not kept, because an event of many tiny requests would take far more memory as a list than in its payload.
+ pub fn for_each_network_request(&self, inner: &[u8], mut each: impl FnMut(NetworkRequest)) {
+ let Some(items) = self
+ .network_requests
+ .and_then(|r| scan::array_items(inner, r).ok())
+ else {
+ return;
+ };
+ for request in items.filter_map(Result::ok) {
+ let [duration, status, response_status] =
+ fields(inner, request, [b"duration", b"status", b"responseStatus"]);
+ let status = status
+ .filter(|&s| !scan::is_null(inner, s))
+ .or(response_status);
+ each(NetworkRequest {
+ duration: number(inner, duration),
+ status: number(inner, status),
+ });
+ }
+ }
+}
+
/// Counts what one block's events did. Navigation and active time need the session's earlier events, so the caller
/// keeps that state.
#[derive(Default)]
@@ -93,6 +214,7 @@ impl EventCounter {
&mut self,
inner: &[u8],
event: &Event,
+ data: &DataFields,
last_href: &mut Option<String>,
activity: &mut Activity,
) -> bool {
@@ -110,18 +232,15 @@ impl EventCounter {
stats.first_ts_ms = Some(stats.first_ts_ms.map_or(ts, |first| first.min(ts)));
stats.last_ts_ms = Some(stats.last_ts_ms.map_or(ts, |last| last.max(ts)));
}
- let Some(data) = event.data else { return false };
+ if event.data.is_none() {
+ return false;
+ }
match (event.kind, event.source) {
(Some(META | CUSTOM), _) => {
- let [href, payload, tag, width, height] = fields(
- inner,
- data,
- [b"href", b"payload", b"tag", b"width", b"height"],
- );
if event.kind == Some(META) {
- set_viewport(stats, inner, event, width, height);
+ set_viewport(stats, inner, event, data.width, data.height);
}
- let Some(href) = page_url(inner, event.kind, href, payload, tag) else {
+ let Some(href) = href_from(inner, event.kind, data) else {
return false;
};
if stats.host.is_none() {
@@ -129,13 +248,13 @@ impl EventCounter {
}
if last_href.as_deref() != Some(&*href) {
stats.navigations += 1;
- *last_href = Some(href);
+ *last_href = Some(href.into_owned());
}
return true;
}
(Some(INCREMENTAL_SNAPSHOT), Some(MOUSE_INTERACTION)) => {
stats.mouse_interactions += 1;
- let interaction = number(inner, field(inner, data, b"type")).map(|t| t as u8);
+ let interaction = number(inner, data.interaction).map(|t| t as u8);
if interaction.is_some_and(is_click) {
stats.clicks += 1;
}
@@ -153,8 +272,7 @@ impl EventCounter {
(Some(INCREMENTAL_SNAPSHOT), Some(SCROLL)) => stats.scrolls += 1,
(Some(INCREMENTAL_SNAPSHOT), Some(VIEWPORT_RESIZE)) => {
stats.viewport_resizes += 1;
- let [width, height] = fields(inner, data, [b"width", b"height"]);
- set_viewport(stats, inner, event, width, height);
+ set_viewport(stats, inner, event, data.width, data.height);
}
(Some(INCREMENTAL_SNAPSHOT), Some(INPUT)) => stats.keypresses += 1,
(Some(INCREMENTAL_SNAPSHOT), Some(MEDIA_INTERACTION)) => stats.media_events += 1,
@@ -165,41 +283,23 @@ impl EventCounter {
false
}
- fn observe_plugin(&mut self, inner: &[u8], data: Span) {
+ /// Counts one request of an `rrweb/network@1` event. The inspector reads an event's requests in one pass for this
+ /// and the feature recorder.
+ pub fn count_network_request(&mut self, status: Option<f64>) {
+ count_request(&mut self.stats, status);
+ }
+
+ fn observe_plugin(&mut self, inner: &[u8], data: &DataFields) {
let stats = &mut self.stats;
- let payload = field(inner, data, b"payload");
- match text(inner, field(inner, data, b"plugin")).as_deref() {
- Some("rrweb/console@1") => match payload
- .and_then(|p| text(inner, field(inner, p, b"level")))
- .as_deref()
- {
+ match text(inner, data.plugin).as_deref() {
+ Some("rrweb/console@1") => match text(inner, data.level).as_deref() {
Some("error") => stats.console_errors += 1,
Some("warn") => stats.console_warnings += 1,
_ => {}
},
- Some("rrweb/network@1") => {
- let Some(requests) = payload.and_then(|p| field(inner, p, b"requests")) else {
- return;
- };
- let Ok(items) = scan::array_items(inner, requests) else {
- return;
- };
- for request in items.filter_map(Result::ok) {
- // `req.status ?? req.responseStatus`, as `session-feature-recorder.ts` reads it.
- let [status, response_status] =
- fields(inner, request, [b"status", b"responseStatus"]);
- let status = status
- .filter(|&s| !scan::is_null(inner, s))
- .or(response_status);
- count_request(stats, number(inner, status));
- }
- }
Some("posthog/network@1") => {
- if let Some(payload) = payload.filter(|&p| scan::is_object(inner, p)) {
- count_request(
- stats,
- number(inner, field(inner, payload, POSTHOG_NETWORK_STATUS_KEY)),
- );
+ if data.payload.is_some_and(|p| scan::is_object(inner, p)) {
+ count_request(stats, number(inner, data.network_status));
}
}
_ => {}
@@ -240,32 +340,28 @@ fn set_viewport(
);
}
-/// The page URL of a Meta or custom event, by the rules of [`page_url`].
+/// The page URL of a Meta or custom event, by the rules of [`href_from`].
pub fn event_page_url(inner: &[u8], event: &Event) -> Option<String> {
if !matches!(event.kind, Some(META | CUSTOM)) {
return None;
}
- let [href, payload, tag] = fields(inner, event.data?, [b"href", b"payload", b"tag"]);
- page_url(inner, event.kind, href, payload, tag)
+ href_from(inner, event.kind, &DataFields::read(inner, event)).map(Cow::into_owned)
}
-/// The page URL an event carries, as `hrefFrom` in `rrweb-types.ts` reads it: a Meta event's `href`, or the
-/// `payload.href` of a custom event such as posthog-js's `$url_changed`. JSON-LD custom events carry no page.
-fn page_url(
- inner: &[u8],
- kind: Option<u8>,
- href: Option<Span>,
- payload: Option<Span>,
- tag: Option<Span>,
-) -> Option<String> {
- if kind == Some(CUSTOM) && text(inner, tag).as_deref() == Some("$json_ld") {
+/// The page URL an event carries, as `hrefFrom` in `rrweb-types.ts` reads it: `data.href`, such as a Meta event's, or
+/// `data.payload.href`, such as that of posthog-js's `$url_changed` custom event, trimmed as JavaScript trims.
+/// JSON-LD custom events carry no page.
+pub fn href_from<'b>(inner: &'b [u8], kind: Option<u8>, data: &DataFields) -> Option<Cow<'b, str>> {
+ if kind == Some(CUSTOM) && text(inner, data.tag).as_deref() == Some("$json_ld") {
return None;
}
- let payload_href = payload.and_then(|p| field(inner, p, b"href"));
- [href, payload_href]
+ [data.href, data.payload_href]
.into_iter()
.filter_map(|span| text(inner, span))
- .map(|href| href.trim().to_owned())
+ .map(|href| match href {
+ Cow::Borrowed(href) => Cow::Borrowed(js_trim(href)),
+ Cow::Owned(href) => Cow::Owned(js_trim(&href).to_owned()),
+ })
.find(|href| !href.is_empty())
}
@@ -521,7 +617,11 @@ mod tests {
let mut counter = EventCounter::default();
let (mut last_href, mut activity) = (None, Activity::default());
for event in &message.events {
- counter.observe(&message.inner, event, &mut last_href, &mut activity);
+ let data = DataFields::read(&message.inner, event);
+ counter.observe(&message.inner, event, &data, &mut last_href, &mut activity);
+ data.for_each_network_request(&message.inner, |request| {
+ counter.count_network_request(request.status);
+ });
}
let stats = counter.finish();
diff --git a/rust/ai-research/replay-core-v4/src/inspect/features.rs b/rust/ai-research/replay-core-v4/src/inspect/features.rs
new file mode 100644
index 000000000000..c8d72da07e48
--- /dev/null
+++ b/rust/ai-research/replay-core-v4/src/inspect/features.rs
@@ -0,0 +1,1174 @@
+//! The values of ingestion's surfacing feature recorder (`session-feature-recorder.ts`), one block at a time.
+//!
+//! Ingestion records them per batch and the surfacing sweep merges a session's batch rows. The recorder starts fresh at
+//! each block, as ingestion's does at each batch, so a block row holds what one batch row of `session_replay_features`
+//! holds, and a session's values are the fold of its block rows: [`SurfacingFeatures::merge`].
+
+use std::borrow::Cow;
+use std::collections::{HashMap, HashSet};
+
+use crate::bytescan::{self as scan, Span};
+use crate::scan::message::Event;
+
+use super::clicks::{ClickKind, ClickTracker};
+use super::events::{href_from, number, text, DataFields, NetworkRequest};
+use super::rrweb::*;
+
+/// `MAX_UNIQUE_VALUES`: the values each set of a block keeps, and the input fields it remembers for backspaces.
+pub const MAX_UNIQUE_VALUES: usize = 1000;
+
+/// `PATH_TOKEN_ALLOWLIST` in its order, each with the name its count has in `session_replay_features`.
+pub const PATH_TOKENS: [(&str, &str); 14] = [
+ ("login", "login"),
+ ("signup", "signup"),
+ ("checkout", "checkout"),
+ ("cart", "cart"),
+ ("billing", "billing"),
+ ("settings", "settings"),
+ ("account", "account"),
+ ("error", "error"),
+ ("404", "not_found"),
+ ("admin", "admin"),
+ ("dashboard", "dashboard"),
+ ("onboarding", "onboarding"),
+ ("cancel", "cancel"),
+ ("refund", "refund"),
+];
+
+const LONG_IDLE_GAP_MS: f64 = 30_000.0;
+const SELECTION_COPY_WINDOW_MS: f64 = 2_000.0;
+const QUICK_BACK_WINDOW_MS: f64 = 2_000.0;
+const RAPID_SCROLL_REVERSAL_MS: f64 = 500.0;
+const ERROR_AFTER_ACTION_MS: f64 = 5_000.0;
+
+/// The largest node ID a set keeps. Past it a JavaScript number does not hold every integer, and the index refuses it.
+const MAX_NODE_ID: f64 = 9_007_199_254_740_991.0;
+
+/// One block's values as `SessionFeatureRecorder.end()` returns them for a batch. Float sums keep the exact values the
+/// TypeScript computes, in the same order of operations.
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct BlockFeatures {
+ pub event_count: u64,
+ pub mouse_position_count: u64,
+ pub mouse_sum_x: f64,
+ pub mouse_sum_x_squared: f64,
+ pub mouse_sum_y: f64,
+ pub mouse_sum_y_squared: f64,
+ pub mouse_distance_traveled: f64,
+ pub mouse_direction_change_count: u64,
+ pub mouse_velocity_sum: f64,
+ pub mouse_velocity_sum_of_squares: f64,
+ pub mouse_velocity_count: u64,
+ pub scroll_event_count: u64,
+ pub total_scroll_magnitude: f64,
+ pub scroll_direction_reversal_count: u64,
+ pub rapid_scroll_reversal_count: u64,
+ pub scroll_to_top_count: u64,
+ pub click_count: u64,
+ pub keypress_count: u64,
+ pub mouse_activity_count: u64,
+ pub rage_click_count: u64,
+ pub dead_click_count: u64,
+ pub backspace_count: u64,
+ pub inter_action_gap_count: u64,
+ pub inter_action_gap_sum_ms: f64,
+ pub inter_action_gap_sum_of_squares_ms: f64,
+ pub max_idle_gap_ms: f64,
+ pub long_idle_gap_count: u64,
+ pub quick_back_count: u64,
+ pub page_visit_count: u64,
+ /// The distinct page URLs in plaintext, sorted. The writer stores them as fingerprints under the session key, where
+ /// ingestion stores their md5.
+ pub visited_urls: Vec<String>,
+ pub path_token_counts: [u64; PATH_TOKENS.len()],
+ pub console_error_count: u64,
+ pub console_error_after_click_count: u64,
+ pub console_warn_count: u64,
+ pub network_request_count: u64,
+ pub network_failed_request_count: u64,
+ pub network_4xx_count: u64,
+ pub network_5xx_count: u64,
+ pub network_request_duration_sum: f64,
+ pub network_request_duration_sum_of_squares: f64,
+ pub network_request_duration_count: u64,
+ pub mutation_count: u64,
+ pub viewport_resize_count: u64,
+ pub touch_event_count: u64,
+ pub max_scroll_y: f64,
+ /// Sorted.
+ pub click_target_ids: Vec<i64>,
+ /// Sorted.
+ pub form_field_ids: Vec<i64>,
+ pub text_selection_count: u64,
+ pub selection_copy_count: u64,
+}
+
+/// What the recorder knows about the block so far. It never outlives the block.
+#[derive(Default)]
+pub struct FeatureRecorder {
+ values: BlockFeatures,
+ visited_urls: HashSet<String>,
+ click_target_ids: HashSet<i64>,
+ form_field_ids: HashSet<i64>,
+ /// The last text length of each input, by the bits of its ID, for the backspace count.
+ input_text_lengths: HashMap<u64, usize>,
+ last_mouse: Option<MousePosition>,
+ last_mouse_delta: Option<(f64, f64)>,
+ scroll: ScrollState,
+ clicks: ClickTracker,
+ /// The last click or keypress.
+ last_action_ts: Option<f64>,
+ last_navigation_ts: Option<f64>,
+ /// Valid while `last_navigation_ts` is set. Reused, so that a navigation allocates only for a new page.
+ last_navigation_url: String,
+ /// The path tokens of `last_navigation_url`, one bit per token, so that the same page is parsed once.
+ last_navigation_tokens: u16,
+ last_selection_ts: Option<f64>,
+}
+
+#[derive(Clone, Copy)]
+struct MousePosition {
+ x: f64,
+ y: f64,
+ ts: f64,
+}
+
+#[derive(Default)]
+struct ScrollState {
+ last_up: Option<bool>,
+ last_ts: Option<f64>,
+ last_y: Option<f64>,
+ /// `lastScrollId`: JavaScript `null` or a node ID.
+ last_id: Option<f64>,
+}
+
+impl FeatureRecorder {
+ /// Records one event of the block, apart from the requests of an `rrweb/network@1` event, which reach
+ /// [`FeatureRecorder::track_network_request`]. Returns whether the event counts: ingestion's parse step drops an
+ /// event without a positive timestamp before its recorder sees it, so such an event changes nothing here.
+ pub fn observe(&mut self, inner: &[u8], event: &Event, data: &DataFields) -> bool {
+ let Some(ts) = event.timestamp_ms.filter(|&ts| ts > 0.0) else {
+ return false;
+ };
+ self.values.event_count += 1;
+ let source = event
+ .source
+ .filter(|_| event.kind == Some(INCREMENTAL_SNAPSHOT));
+ let interaction = number(inner, data.interaction);
+ let is_click = source == Some(MOUSE_INTERACTION)
+ && interaction.is_some_and(|t| {
+ [CLICK, DOUBLE_CLICK, TOUCH_END, CONTEXT_MENU]
+ .map(f64::from)
+ .contains(&t)
+ });
+ let is_keypress = source == Some(INPUT);
+
+ // The order of `aggregateFeatures`: a click reads the navigation state before a navigation on the same event
+ // sets it.
+ match source {
+ Some(MOUSE_MOVE) => self.track_mouse_positions(inner, ts, event.positions_start),
+ Some(SCROLL) => self.track_scroll(inner, ts, data),
+ _ => {}
+ }
+ if is_click {
+ self.track_click(inner, ts, data);
+ }
+ if is_keypress {
+ self.track_keypress(inner, ts, data);
+ }
+ if is_click || is_keypress {
+ self.track_inter_action_gap(ts);
+ }
+ if let Some(href) = href_from(inner, event.kind, data) {
+ self.track_navigation(ts, &href);
+ }
+ if event.kind == Some(PLUGIN) {
+ self.track_plugin(inner, ts, data);
+ }
+ let values = &mut self.values;
+ match source {
+ Some(SELECTION) => {
+ values.text_selection_count += 1;
+ self.last_selection_ts = Some(ts);
+ }
+ Some(MUTATION) => values.mutation_count += 1,
+ Some(VIEWPORT_RESIZE) => values.viewport_resize_count += 1,
+ _ => {}
+ }
+ let touch_interaction =
+ interaction.is_some_and(|t| [TOUCH_START, TOUCH_END].map(f64::from).contains(&t));
+ if source == Some(TOUCH_MOVE) || (source == Some(MOUSE_INTERACTION) && touch_interaction) {
+ values.touch_event_count += 1;
+ }
+ if matches!(source, Some(MOUSE_INTERACTION | MOUSE_MOVE | TOUCH_MOVE)) {
+ values.mouse_activity_count += 1;
+ }
+ true
+ }
+
+ fn track_mouse_positions(&mut self, inner: &[u8], ts: f64, positions: Option<u32>) {
+ let Some(items) = positions.and_then(|start| {
+ scan::array_object_fields(inner, start as usize, [b"x", b"y", b"timeOffset"]).ok()
+ }) else {
+ return;
+ };
+ for [x, y, offset] in items.map_while(Result::ok) {
+ // The TypeScript skips a falsy item, and turns its sums into NaN on an item without coordinates.
+ let (Some(x), Some(y)) = (number(inner, x), number(inner, y)) else {
+ continue;
+ };
+ let ts = ts + number(inner, offset).unwrap_or(0.0);
+ let values = &mut self.values;
+ values.mouse_position_count += 1;
+ values.mouse_sum_x += x;
+ values.mouse_sum_x_squared += x * x;
+ values.mouse_sum_y += y;
+ values.mouse_sum_y_squared += y * y;
+ if let Some(last) = self.last_mouse {
+ let (dx, dy) = (x - last.x, y - last.y);
+ let distance = (dx * dx + dy * dy).sqrt();
+ values.mouse_distance_traveled += distance;
+ if let Some((last_dx, last_dy)) = self.last_mouse_delta {
+ if dx * last_dx + dy * last_dy < 0.0 {
+ values.mouse_direction_change_count += 1;
+ }
+ }
+ self.last_mouse_delta = Some((dx, dy));
+ let dt = ts - last.ts;
+ if dt > 0.0 {
+ let velocity = distance / dt;
+ values.mouse_velocity_sum += velocity;
+ values.mouse_velocity_sum_of_squares += velocity * velocity;
+ values.mouse_velocity_count += 1;
+ }
+ }
+ self.last_mouse = Some(MousePosition { x, y, ts });
+ }
+ }
+
+ fn track_scroll(&mut self, inner: &[u8], ts: f64, data: &DataFields) {
+ let values = &mut self.values;
+ values.scroll_event_count += 1;
+ let Some(y) = number(inner, data.y) else {
+ return;
+ };
+ if y > values.max_scroll_y {
+ values.max_scroll_y = y;
+ }
+ // `scrollId !== this.lastScrollId`: a missing ID never equals the last one, so it resets every time.
+ let id = match data.id {
+ None => None,
+ Some(id) if scan::is_null(inner, id) => Some(None),
+ Some(id) => number(inner, Some(id)).map(Some),
+ };
+ let scroll = &mut self.scroll;
+ if id.is_none_or(|id| id != scroll.last_id) {
+ scroll.last_y = None;
+ scroll.last_up = None;
+ scroll.last_id = id.flatten();
+ }
+ let Some(last_y) = scroll.last_y else {
+ scroll.last_y = Some(y);
+ scroll.last_ts = Some(ts);
+ return;
+ };
+ let delta = y - last_y;
+ values.total_scroll_magnitude += delta.abs();
+ if last_y > 0.0 && y == 0.0 {
+ values.scroll_to_top_count += 1;
+ }
+ if delta != 0.0 {
+ let up = delta < 0.0;
+ if scroll.last_up.is_some_and(|last_up| last_up != up) {
+ values.scroll_direction_reversal_count += 1;
+ if scroll
+ .last_ts
+ .is_some_and(|last_ts| ts - last_ts < RAPID_SCROLL_REVERSAL_MS)
+ {
+ values.rapid_scroll_reversal_count += 1;
+ }
+ }
+ scroll.last_up = Some(up);
+ }
+ scroll.last_y = Some(y);
+ scroll.last_ts = Some(ts);
+ }
+
+ fn track_click(&mut self, inner: &[u8], ts: f64, data: &DataFields) {
+ self.values.click_count += 1;
+ if let Some(id) = node_id(inner, data.id) {
+ if self.click_target_ids.len() < MAX_UNIQUE_VALUES {
+ self.click_target_ids.insert(id);
+ }
+ }
+ let position = number(inner, data.x).zip(number(inner, data.y));
+ match self.clicks.click(ts, position) {
+ ClickKind::Rage => self.values.rage_click_count += 1,
+ ClickKind::AfterDead => self.values.dead_click_count += 1,
+ ClickKind::Plain => {}
+ }
+ }
+
+ fn track_keypress(&mut self, inner: &[u8], ts: f64, data: &DataFields) {
+ self.values.keypress_count += 1;
+ if let Some(id) = number(inner, data.id) {
+ if let Some(id) = node_id(inner, data.id) {
+ if self.form_field_ids.len() < MAX_UNIQUE_VALUES {
+ self.form_field_ids.insert(id);
+ }
+ }
+ if let Some(length) = data.text.and_then(|text| utf16_length(inner, text)) {
+ // A JavaScript `Map` holds 0 and -0 as one key.
+ let key = (id + 0.0).to_bits();
+ let previous = self.input_text_lengths.get(&key).copied();
+ if let Some(previous) = previous.filter(|&previous| length < previous) {
+ self.values.backspace_count += (previous - length) as u64;
+ }
+ if previous.is_some() || self.input_text_lengths.len() < MAX_UNIQUE_VALUES {
+ self.input_text_lengths.insert(key, length);
+ }
+ }
+ }
+ if self
+ .last_selection_ts
+ .is_some_and(|selected| ts - selected <= SELECTION_COPY_WINDOW_MS)
+ {
+ self.values.selection_copy_count += 1;
+ self.last_selection_ts = None;
+ }
+ }
+
+ fn track_inter_action_gap(&mut self, ts: f64) {
+ if let Some(last) = self.last_action_ts {
+ let gap = ts - last;
+ if gap > 0.0 {
+ let values = &mut self.values;
+ values.inter_action_gap_count += 1;
+ values.inter_action_gap_sum_ms += gap;
+ values.inter_action_gap_sum_of_squares_ms += gap * gap;
+ if gap > values.max_idle_gap_ms {
+ values.max_idle_gap_ms = gap;
+ }
+ if gap > LONG_IDLE_GAP_MS {
+ values.long_idle_gap_count += 1;
+ }
+ }
+ }
+ self.last_action_ts = Some(ts);
+ }
+
+ fn track_navigation(&mut self, ts: f64, url: &str) {
+ self.values.page_visit_count += 1;
+ if self.visited_urls.len() < MAX_UNIQUE_VALUES && !self.visited_urls.contains(url) {
+ self.visited_urls.insert(url.to_owned());
+ }
+ self.clicks.navigated();
+ let same_page = self.last_navigation_ts.is_some() && self.last_navigation_url == url;
+ let tokens = if same_page {
+ self.last_navigation_tokens
+ } else {
+ path_tokens(url)
+ };
+ for (index, count) in self.values.path_token_counts.iter_mut().enumerate() {
+ if tokens & (1 << index) != 0 {
+ *count += 1;
+ }
+ }
+ if self
+ .last_navigation_ts
+ .is_some_and(|last| ts - last < QUICK_BACK_WINDOW_MS)
+ && !same_page
+ {
+ self.values.quick_back_count += 1;
+ }
+ if !same_page {
+ self.last_navigation_url.clear();
+ self.last_navigation_url.push_str(url);
+ self.last_navigation_tokens = tokens;
+ }
+ self.last_navigation_ts = Some(ts);
+ }
+
+ fn track_plugin(&mut self, inner: &[u8], ts: f64, data: &DataFields) {
+ let Some(plugin) = text(inner, data.plugin) else {
+ return;
+ };
+ match &*plugin {
+ "rrweb/console@1" => match text(inner, data.level).as_deref() {
+ Some("error") => {
+ self.values.console_error_count += 1;
+ if self
+ .last_action_ts
+ .is_some_and(|last| (0.0..ERROR_AFTER_ACTION_MS).contains(&(ts - last)))
+ {
+ self.values.console_error_after_click_count += 1;
+ }
+ }
+ Some("warn") => self.values.console_warn_count += 1,
+ _ => {}
+ },
+ // `if (!payload) return`, then `payload[39]` and `payload[21]`.
+ "posthog/network@1" => {
+ if data.payload.is_some_and(|p| js_truthy(inner, p)) {
+ self.track_network_request(NetworkRequest {
+ duration: number(inner, data.network_duration),
+ status: number(inner, data.network_status),
+ });
+ }
+ }
+ _ => {}
+ }
+ }
+
+ /// Records one request of an `rrweb/network@1` event. The inspector reads an event's requests in one pass for this
+ /// and the event counter.
+ pub fn track_network_request(&mut self, request: NetworkRequest) {
+ let NetworkRequest { duration, status } = request;
+ let values = &mut self.values;
+ values.network_request_count += 1;
+ if let Some(status) = status {
+ if (400.0..500.0).contains(&status) {
+ values.network_4xx_count += 1;
+ values.network_failed_request_count += 1;
+ } else if (500.0..600.0).contains(&status) {
+ values.network_5xx_count += 1;
+ values.network_failed_request_count += 1;
+ } else if status >= 600.0 {
+ values.network_failed_request_count += 1;
+ }
+ }
+ if let Some(duration) = duration.filter(|&d| d > 0.0) {
+ values.network_request_duration_sum += duration;
+ values.network_request_duration_sum_of_squares += duration * duration;
+ values.network_request_duration_count += 1;
+ }
+ }
+
+ pub fn finish(self) -> BlockFeatures {
+ fn sorted<T: Ord>(values: HashSet<T>) -> Vec<T> {
+ let mut values: Vec<T> = values.into_iter().collect();
+ values.sort_unstable();
+ values
+ }
+ BlockFeatures {
+ visited_urls: sorted(self.visited_urls),
+ click_target_ids: sorted(self.click_target_ids),
+ form_field_ids: sorted(self.form_field_ids),
+ ..self.values
+ }
+ }
+}
+
+/// The surfacing values of a block as the index stores them, or of a session as [`SurfacingFeatures::merge`] folds
+/// them. Each float sum of [`BlockFeatures`] is an integer in the unit its name gives, rounded once per block, so that
+/// the fold is exact and does not depend on the order of the blocks.
+#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord)]
+pub struct SurfacingFeatures {
+ pub event_count: u64,
+ pub click_count: u64,
+ pub keypress_count: u64,
+ pub mouse_activity_count: u64,
+ pub rage_click_count: u64,
+ pub dead_click_count: u64,
+ pub quick_back_count: u64,
+ pub page_visit_count: u64,
+ pub text_selection_count: u64,
+ pub scroll_event_count: u64,
+ pub console_error_count: u64,
+ pub console_error_after_click_count: u64,
+ pub console_warn_count: u64,
+ pub network_request_count: u64,
+ pub network_failed_request_count: u64,
+ pub network_4xx_count: u64,
+ pub network_5xx_count: u64,
+ pub network_request_duration_count: u64,
+ pub mouse_position_count: u64,
+ pub mouse_direction_change_count: u64,
+ pub mouse_velocity_count: u64,
+ pub scroll_direction_reversal_count: u64,
+ pub rapid_scroll_reversal_count: u64,
+ pub scroll_to_top_count: u64,
+ pub inter_action_gap_count: u64,
+ pub long_idle_gap_count: u64,
+ pub backspace_count: u64,
+ pub mutation_count: u64,
+ pub viewport_resize_count: u64,
+ pub touch_event_count: u64,
+ pub selection_copy_count: u64,
+ /// In [`PATH_TOKENS`] order.
+ pub path_visit_counts: [u64; PATH_TOKENS.len()],
+
+ pub mouse_sum_x_millipx: i64,
+ pub mouse_sum_x_squared_px2: i64,
+ pub mouse_sum_y_millipx: i64,
+ pub mouse_sum_y_squared_px2: i64,
+ pub mouse_distance_traveled_millipx: i64,
+ pub mouse_velocity_sum_px_per_s: i64,
+ pub mouse_velocity_sum_of_squares_px2_per_s2: i64,
+ pub total_scroll_magnitude_millipx: i64,
+ pub inter_action_gap_sum_ms: i64,
+ pub inter_action_gap_sum_of_squares_ms2: i64,
+ pub network_request_duration_sum_us: i64,
+ pub network_request_duration_sum_of_squares_ms2: i64,
+
+ pub max_scroll_y_millipx: i64,
+ pub max_idle_gap_ms: i64,
+
+ /// The distinct page URLs as [`crate::refs::url_fingerprint`]s under the session's fingerprint key, sorted. The
+ /// inspector leaves this empty: the writer fills it, because only the writer holds the session key.
+ pub visited_url_fingerprints: Vec<String>,
+ /// Sorted.
+ pub click_target_ids: Vec<i64>,
+ /// Sorted.
+ pub form_field_ids: Vec<i64>,
+}
+
+impl SurfacingFeatures {
+ /// Folds another block of the session into this one: counts and sums add, maxima take the larger, and sets take
+ /// the union with no cap. The result does not depend on the order or grouping of the blocks.
+ pub fn merge(&mut self, other: &SurfacingFeatures) {
+ macro_rules! add {
+ ($($field:ident),*) => { $( self.$field = self.$field.saturating_add(other.$field); )* };
+ }
+ add!(
+ event_count,
+ click_count,
+ keypress_count,
+ mouse_activity_count,
+ rage_click_count,
+ dead_click_count,
+ quick_back_count,
+ page_visit_count,
+ text_selection_count,
+ scroll_event_count,
+ console_error_count,
+ console_error_after_click_count,
+ console_warn_count,
+ network_request_count,
+ network_failed_request_count,
+ network_4xx_count,
+ network_5xx_count,
+ network_request_duration_count,
+ mouse_position_count,
+ mouse_direction_change_count,
+ mouse_velocity_count,
+ scroll_direction_reversal_count,
+ rapid_scroll_reversal_count,
+ scroll_to_top_count,
+ inter_action_gap_count,
+ long_idle_gap_count,
+ backspace_count,
+ mutation_count,
+ viewport_resize_count,
+ touch_event_count,
+ selection_copy_count,
+ mouse_sum_x_millipx,
+ mouse_sum_x_squared_px2,
+ mouse_sum_y_millipx,
+ mouse_sum_y_squared_px2,
+ mouse_distance_traveled_millipx,
+ mouse_velocity_sum_px_per_s,
+ mouse_velocity_sum_of_squares_px2_per_s2,
+ total_scroll_magnitude_millipx,
+ inter_action_gap_sum_ms,
+ inter_action_gap_sum_of_squares_ms2,
+ network_request_duration_sum_us,
+ network_request_duration_sum_of_squares_ms2
+ );
+ for (count, other) in self
+ .path_visit_counts
+ .iter_mut()
+ .zip(other.path_visit_counts)
+ {
+ *count = count.saturating_add(other);
+ }
+ self.max_scroll_y_millipx = self.max_scroll_y_millipx.max(other.max_scroll_y_millipx);
+ self.max_idle_gap_ms = self.max_idle_gap_ms.max(other.max_idle_gap_ms);
+ union(
+ &mut self.visited_url_fingerprints,
+ &other.visited_url_fingerprints,
+ );
+ union(&mut self.click_target_ids, &other.click_target_ids);
+ union(&mut self.form_field_ids, &other.form_field_ids);
+ }
+}
+
+impl From<&BlockFeatures> for SurfacingFeatures {
+ fn from(block: &BlockFeatures) -> Self {
+ SurfacingFeatures {
+ event_count: block.event_count,
+ click_count: block.click_count,
+ keypress_count: block.keypress_count,
+ mouse_activity_count: block.mouse_activity_count,
+ rage_click_count: block.rage_click_count,
+ dead_click_count: block.dead_click_count,
+ quick_back_count: block.quick_back_count,
+ page_visit_count: block.page_visit_count,
+ text_selection_count: block.text_selection_count,
+ scroll_event_count: block.scroll_event_count,
+ console_error_count: block.console_error_count,
+ console_error_after_click_count: block.console_error_after_click_count,
+ console_warn_count: block.console_warn_count,
+ network_request_count: block.network_request_count,
+ network_failed_request_count: block.network_failed_request_count,
+ network_4xx_count: block.network_4xx_count,
+ network_5xx_count: block.network_5xx_count,
+ network_request_duration_count: block.network_request_duration_count,
+ mouse_position_count: block.mouse_position_count,
+ mouse_direction_change_count: block.mouse_direction_change_count,
+ mouse_velocity_count: block.mouse_velocity_count,
+ scroll_direction_reversal_count: block.scroll_direction_reversal_count,
+ rapid_scroll_reversal_count: block.rapid_scroll_reversal_count,
+ scroll_to_top_count: block.scroll_to_top_count,
+ inter_action_gap_count: block.inter_action_gap_count,
+ long_idle_gap_count: block.long_idle_gap_count,
+ backspace_count: block.backspace_count,
+ mutation_count: block.mutation_count,
+ viewport_resize_count: block.viewport_resize_count,
+ touch_event_count: block.touch_event_count,
+ selection_copy_count: block.selection_copy_count,
+ path_visit_counts: block.path_token_counts,
+ mouse_sum_x_millipx: fixed(block.mouse_sum_x, 1e3),
+ mouse_sum_x_squared_px2: fixed(block.mouse_sum_x_squared, 1.0),
+ mouse_sum_y_millipx: fixed(block.mouse_sum_y, 1e3),
+ mouse_sum_y_squared_px2: fixed(block.mouse_sum_y_squared, 1.0),
+ mouse_distance_traveled_millipx: fixed(block.mouse_distance_traveled, 1e3),
+ // The TypeScript's velocity is in pixels per millisecond.
+ mouse_velocity_sum_px_per_s: fixed(block.mouse_velocity_sum, 1e3),
+ mouse_velocity_sum_of_squares_px2_per_s2: fixed(
+ block.mouse_velocity_sum_of_squares,
+ 1e6,
+ ),
+ total_scroll_magnitude_millipx: fixed(block.total_scroll_magnitude, 1e3),
+ inter_action_gap_sum_ms: fixed(block.inter_action_gap_sum_ms, 1.0),
+ inter_action_gap_sum_of_squares_ms2: fixed(
+ block.inter_action_gap_sum_of_squares_ms,
+ 1.0,
+ ),
+ network_request_duration_sum_us: fixed(block.network_request_duration_sum, 1e3),
+ network_request_duration_sum_of_squares_ms2: fixed(
+ block.network_request_duration_sum_of_squares,
+ 1.0,
+ ),
+ max_scroll_y_millipx: fixed(block.max_scroll_y, 1e3),
+ max_idle_gap_ms: fixed(block.max_idle_gap_ms, 1.0),
+ visited_url_fingerprints: Vec::new(),
+ click_target_ids: block.click_target_ids.clone(),
+ form_field_ids: block.form_field_ids.clone(),
+ }
+ }
+}
+
+/// `value` in units `scale` times smaller, rounded half away from zero. A value past the `i64` range saturates.
+fn fixed(value: f64, scale: f64) -> i64 {
+ (value * scale).round() as i64
+}
+
+/// The sorted union of two sorted sets.
+fn union<T: Ord + Clone>(values: &mut Vec<T>, other: &[T]) {
+ if other.is_empty() {
+ return;
+ }
+ values.extend_from_slice(other);
+ values.sort_unstable();
+ values.dedup();
+}
+
+/// A node ID as a set keeps it. The TypeScript keeps any value, but rrweb's node IDs are integers.
+fn node_id(inner: &[u8], span: Option<Span>) -> Option<i64> {
+ number(inner, span)
+ .filter(|id| id.fract() == 0.0 && id.abs() <= MAX_NODE_ID)
+ .map(|id| id as i64)
+}
+
+/// `text.length` in JavaScript: the UTF-16 code units of a JSON string, counted on its escaped bytes so that nothing
+/// is decoded or allocated. An escape is one unit, and so is each half of an escaped surrogate pair.
+fn utf16_length(bytes: &[u8], span: Span) -> Option<usize> {
+ if !scan::is_string(bytes, span) || span.1 < span.0 + 2 {
+ return None;
+ }
+ let raw = &bytes[span.0 + 1..span.1 - 1];
+ let (mut at, mut units) = (0, 0);
+ while let Some(&byte) = raw.get(at) {
+ let (width, length) = match byte {
+ b'\\' if raw.get(at + 1) == Some(&b'u') => (6, 1),
+ b'\\' => (2, 1),
+ 0x00..=0x7f => (1, 1),
+ 0xc0..=0xdf => (2, 1),
+ 0xe0..=0xef => (3, 1),
+ 0xf0..=0xff => (4, 2),
+ // A continuation byte, which a valid sequence never starts with.
+ _ => (1, 0),
+ };
+ at += width;
+ units += length;
+ }
+ Some(units)
+}
+
+/// JavaScript truthiness of a JSON value.
+fn js_truthy(bytes: &[u8], span: Span) -> bool {
+ match bytes.get(span.0) {
+ Some(b'n' | b'f') => false,
+ Some(b'"') => span.1 - span.0 > 2,
+ Some(b'-' | b'0'..=b'9') => scan::parse_number(bytes, span).is_some_and(|n| n != 0.0),
+ _ => true,
+ }
+}
+
+/// The [`PATH_TOKENS`] among the segments of the URL's path, one bit per token in their order. A token counts once per
+/// URL. `new URL(url).pathname`, or for a URL that does not parse, a relative path up to its query or fragment.
+fn path_tokens(url: &str) -> u16 {
+ let parsed = url::Url::parse(url);
+ let path = match &parsed {
+ Ok(parsed) => parsed.path(),
+ Err(_) if url.starts_with('/') => url.split(['?', '#']).next().unwrap_or_default(),
+ Err(_) => return 0,
+ };
+ let mut tokens = 0;
+ for segment in path.split('/').filter(|s| !s.is_empty()) {
+ // A parsed path is ASCII. A relative one can hold a character that lowercases to ASCII, such as the Kelvin sign.
+ let lowered = if segment.is_ascii() {
+ Cow::Borrowed(segment)
+ } else {
+ Cow::Owned(segment.to_lowercase())
+ };
+ for (index, (token, _)) in PATH_TOKENS.iter().enumerate() {
+ if lowered.eq_ignore_ascii_case(token) {
+ tokens |= 1 << index;
+ }
+ }
+ }
+ tokens
+}
+
+/// JavaScript's `String.prototype.trim`: its white space and line terminators, which differ from Rust's
+/// [`str::trim`] in U+FEFF and U+0085.
+pub(crate) fn js_trim(text: &str) -> &str {
+ text.trim_matches(|c| {
+ matches!(
+ c,
+ '\u{9}'..='\u{d}'
+ | ' '
+ | '\u{a0}'
+ | '\u{1680}'
+ | '\u{2000}'..='\u{200a}'
+ | '\u{2028}'
+ | '\u{2029}'
+ | '\u{202f}'
+ | '\u{205f}'
+ | '\u{3000}'
+ | '\u{feff}'
+ )
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use serde_json::{json, Value};
+
+ use super::*;
+ use crate::inspect::session::{BlockInspection, SessionInspector};
+ use crate::scan::message::read_message;
+
+ // Expected values traced through `session-feature-recorder.ts` over the same events.
+
+ fn record_blocks(blocks: &[Vec<Value>]) -> Vec<BlockFeatures> {
+ let mut session = SessionInspector::default();
+ blocks
+ .iter()
+ .map(|events| {
+ let inner = json!({"event": "$snapshot_items", "properties": {
+ "$session_id": "s", "$window_id": "w", "$snapshot_source": "web", "$snapshot_items": events,
+ }});
+ let payload = json!({"distinct_id": "d", "data": inner.to_string()});
+ let message = read_message(payload.to_string().into_bytes()).unwrap();
+ let mut block = BlockInspection::default();
+ session.observe_message(&mut block, &message);
+ block.finish().features
+ })
+ .collect()
+ }
+
+ fn record(events: Vec<Value>) -> BlockFeatures {
+ record_blocks(&[events]).remove(0)
+ }
+
+ fn incremental(ts: i64, data: Value) -> Value {
+ json!({"type": 3, "timestamp": ts, "data": data})
+ }
+
+ fn click(ts: i64, x: f64, y: f64, id: i64, kind: Value) -> Value {
+ incremental(
+ ts,
+ json!({"source": 2, "type": kind, "x": x, "y": y, "id": id}),
+ )
+ }
+
+ fn scroll(ts: i64, mut target: Value) -> Value {
+ target["source"] = json!(3);
+ incremental(ts, target)
+ }
+
+ fn input(ts: i64, id: Option<i64>, text: Value) -> Value {
+ let mut data = json!({"source": 5, "text": text});
+ if let Some(id) = id {
+ data["id"] = json!(id);
+ }
+ incremental(ts, data)
+ }
+
+ fn page(ts: i64, data: Value) -> Value {
+ json!({"type": 4, "timestamp": ts, "data": data})
+ }
+
+ fn plugin(ts: i64, name: &str, payload: Value) -> Value {
+ json!({"type": 6, "timestamp": ts, "data": {"plugin": name, "payload": payload}})
+ }
+
+ #[test]
+ fn mouse_positions() {
+ let f = record(vec![incremental(
+ 1000,
+ json!({"source": 1, "positions": [
+ {"x": 0, "y": 0, "timeOffset": 0}, null, {"x": 3, "y": 4, "timeOffset": 10},
+ {"x": 0, "y": 0, "timeOffset": 20}, {"x": 0.5, "y": 0},
+ ]}),
+ )]);
+ assert_eq!(
+ (
+ f.mouse_position_count,
+ f.mouse_sum_x,
+ f.mouse_sum_x_squared,
+ f.mouse_sum_y,
+ f.mouse_sum_y_squared
+ ),
+ (4, 3.5, 9.25, 4.0, 16.0)
+ );
+ assert_eq!(
+ (
+ f.mouse_distance_traveled,
+ f.mouse_direction_change_count,
+ f.mouse_velocity_sum
+ ),
+ (10.5, 2, 1.0)
+ );
+ assert_eq!(
+ (f.mouse_velocity_sum_of_squares, f.mouse_velocity_count),
+ (0.5, 2)
+ );
+ }
+
+ #[test]
+ fn scrolls() {
+ // A scroll without a target starts again every time; a null target continues after it.
+ let f = record(vec![
+ scroll(1000, json!({"id": 1, "y": 0})),
+ scroll(1100, json!({"id": 1, "y": 300})),
+ scroll(1200, json!({"id": 1, "y": 200})),
+ scroll(2200, json!({"id": 1, "y": 900})),
+ scroll(2300, json!({"id": 1, "y": 0})),
+ scroll(2400, json!({"y": 100})),
+ scroll(2500, json!({"y": 400})),
+ scroll(2600, json!({"id": null, "y": 100})),
+ scroll(2700, json!({"id": 1})),
+ ]);
+ assert_eq!(
+ (
+ f.scroll_event_count,
+ f.total_scroll_magnitude,
+ f.scroll_direction_reversal_count,
+ f.rapid_scroll_reversal_count,
+ f.scroll_to_top_count,
+ f.max_scroll_y
+ ),
+ (9, 2300.0, 3, 2, 1, 900.0)
+ );
+ }
+
+ #[test]
+ fn clicks_rage_dead_and_touches() {
+ let f = record(vec![
+ click(1000, 100.0, 100.0, 5, json!(2)),
+ click(1200, 110.0, 105.0, 5, json!(2)),
+ click(1400, 112.0, 100.0, 6, json!(2)),
+ click(1500, 112.0, 101.0, 6, json!(4)),
+ click(4000, 500.0, 500.0, 7, json!(2)),
+ // The lone click at 4000 did nothing before this context menu, so it was dead.
+ click(7000, 900.0, 900.0, 8, json!(3)),
+ page(7500, json!({"href": "https://shop.example.com/a"})),
+ click(10000, 100.0, 100.0, 9, json!(9)),
+ incremental(13000, json!({"source": 2, "type": 2, "id": 10})),
+ click(13100, 1.0, 1.0, 11, json!(1)),
+ click(13200, 1.0, 1.0, 12, json!("2")),
+ click(13300, 1.0, 1.0, 13, json!(7)),
+ incremental(13400, json!({"source": 6, "positions": []})),
+ ]);
+ assert_eq!(
+ (
+ f.click_count,
+ f.rage_click_count,
+ f.dead_click_count,
+ f.touch_event_count,
+ f.mouse_activity_count
+ ),
+ (8, 2, 1, 3, 12)
+ );
+ assert_eq!(f.click_target_ids, [5, 6, 7, 8, 9, 10]);
+ assert_eq!(
+ (
+ f.inter_action_gap_count,
+ f.inter_action_gap_sum_ms,
+ f.inter_action_gap_sum_of_squares_ms,
+ f.max_idle_gap_ms
+ ),
+ (7, 12_000.0, 33_340_000.0, 3000.0)
+ );
+ }
+
+ #[test]
+ fn typing_backspaces_in_utf16_units_and_copies() {
+ let selection = |ts: i64| incremental(ts, json!({"source": 14, "ranges": []}));
+ let f = record(vec![
+ input(1000, Some(20), json!("hello")),
+ input(1100, Some(20), json!("hel")),
+ input(1200, Some(21), json!("\u{1f600}ab")),
+ input(1300, Some(21), json!("a")),
+ input(1400, Some(22), json!("a\n\"b\" \u{1}")),
+ input(1500, Some(22), json!("")),
+ input(1600, None, json!("x")),
+ input(1700, Some(23), json!(4)),
+ selection(2000),
+ input(3500, Some(20), json!("he")),
+ input(3600, Some(20), json!("h")),
+ selection(5000),
+ input(7001, Some(20), json!("")),
+ ]);
+ assert_eq!(
+ (
+ f.keypress_count,
+ f.backspace_count,
+ f.text_selection_count,
+ f.selection_copy_count
+ ),
+ (11, 15, 2, 1)
+ );
+ assert_eq!(f.form_field_ids, [20, 21, 22, 23]);
+ }
+
+ #[test]
+ fn gaps_and_console_errors_after_an_action() {
+ let console = |ts: i64, level: &str| plugin(ts, "rrweb/console@1", json!({"level": level}));
+ let f = record(vec![
+ console(500, "error"),
+ click(1000, 1.0, 1.0, 1, json!(2)),
+ click(1000, 1.0, 1.0, 1, json!(2)),
+ input(41000, Some(2), json!("a")),
+ // Out of order: no gap counts, but the next starts from here.
+ click(2000, 1.0, 1.0, 1, json!(2)),
+ console(6999, "error"),
+ console(7000, "error"),
+ console(1500, "error"),
+ console(7100, "warn"),
+ console(7200, "info"),
+ ]);
+ assert_eq!(
+ (
+ f.inter_action_gap_count,
+ f.inter_action_gap_sum_ms,
+ f.max_idle_gap_ms,
+ f.long_idle_gap_count
+ ),
+ (1, 40_000.0, 40_000.0, 1)
+ );
+ assert_eq!(
+ (
+ f.console_error_count,
+ f.console_error_after_click_count,
+ f.console_warn_count
+ ),
+ (4, 1, 1)
+ );
+ }
+
+ #[test]
+ fn navigations_from_any_event_with_javascript_trim_and_path_tokens() {
+ let changed = |ts: i64, tag: &str, href: &str| json!({"type": 5, "timestamp": ts, "data": {"tag": tag, "payload": {"href": href}}});
+ let f = record(vec![
+ page(
+ 1000,
+ json!({"href": "https://shop.example.com/Account/Settings?tab=billing#checkout"}),
+ ),
+ changed(1500, "$url_changed", "https://shop.example.com/cart"),
+ page(1800, json!({"href": "https://shop.example.com/cart"})),
+ changed(1900, "$json_ld", "https://shop.example.com/login"),
+ page(
+ 5000,
+ json!({"href": " ", "payload": {"href": "\u{feff}https://shop.example.com/login/LOGIN\u{a0}"}}),
+ ),
+ page(9000, json!({"href": "/settings/404?next=/admin"})),
+ page(12000, json!({"href": "/chec\u{212a}out"})),
+ page(15000, json!({"href": "dashboard"})),
+ page(
+ 18000,
+ json!({"href": "https://shop.example.com/onboarding/../refund"}),
+ ),
+ incremental(
+ 40000,
+ json!({"source": 2, "type": 2, "x": 1, "y": 1, "id": 1, "href": "https://shop.example.com/b"}),
+ ),
+ plugin(
+ 40500,
+ "rrweb/console@1",
+ json!({"level": "log", "href": "https://shop.example.com/c"}),
+ ),
+ incremental(
+ 41000,
+ json!({"source": 0, "adds": [], "removes": [], "texts": [], "attributes": [],
+ "payload": {"href": "https://shop.example.com/d"}}),
+ ),
+ ]);
+ assert_eq!(
+ (f.page_visit_count, f.quick_back_count, f.visited_urls.len()),
+ (11, 3, 10)
+ );
+ assert!(f
+ .visited_urls
+ .contains(&"https://shop.example.com/login/LOGIN".to_owned()));
+ // login, signup, checkout, cart, billing, settings, account, error, 404, admin, dashboard, onboarding, cancel, refund
+ assert_eq!(
+ f.path_token_counts,
+ [1, 0, 1, 2, 0, 2, 1, 0, 1, 0, 0, 0, 0, 1]
+ );
+ }
+
+ #[test]
+ fn network_requests_from_both_plugins() {
+ let f = record(vec![
+ plugin(
+ 1000,
+ "rrweb/network@1",
+ json!({"requests": [
+ {"status": 200, "duration": 12.5}, {"status": 404, "duration": 0},
+ {"status": null, "responseStatus": 503, "duration": 0.25}, {"responseStatus": 600}, {"status": "500"}, 7,
+ ]}),
+ ),
+ plugin(1100, "posthog/network@1", json!({"21": 500, "39": 100})),
+ plugin(1200, "posthog/network@1", json!("")),
+ plugin(1300, "posthog/network@1", json!("x")),
+ plugin(1400, "posthog/network@1", json!(0)),
+ plugin(1500, "other/plugin@1", json!({"requests": [{}]})),
+ ]);
+ assert_eq!(
+ (
+ f.network_request_count,
+ f.network_failed_request_count,
+ f.network_4xx_count,
+ f.network_5xx_count
+ ),
+ (8, 4, 1, 2)
+ );
+ assert_eq!(
+ (
+ f.network_request_duration_sum,
+ f.network_request_duration_sum_of_squares,
+ f.network_request_duration_count
+ ),
+ (112.75, 10_156.312_5, 3)
+ );
+ }
+
+ #[test]
+ fn skips_the_events_ingestion_drops_and_starts_again_at_each_block() {
+ let tap = |ts: Value, id: i64| json!({"type": 3, "timestamp": ts, "data": {"source": 2, "type": 2, "id": id}});
+ let f = record(vec![
+ json!({"type": 3, "data": {"source": 2, "type": 2, "id": 1}}),
+ tap(json!(0), 2),
+ tap(json!(-5), 3),
+ tap(json!("1000"), 4),
+ tap(json!(9e15), 5),
+ tap(json!(1000), 6),
+ json!({"type": 3, "timestamp": 1001}),
+ ]);
+ assert_eq!(
+ (f.event_count, f.click_count, f.click_target_ids),
+ (2, 1, vec![6])
+ );
+
+ // Three quick clicks are a rage click in one batch, and none across two.
+ let at = |ts: i64| click(ts, 10.0, 10.0, 1, json!(2));
+ let blocks = record_blocks(&[vec![at(1000), at(1100)], vec![at(1200)]]);
+ assert_eq!(blocks.iter().map(|b| b.rage_click_count).sum::<u64>(), 0);
+ assert_eq!(
+ record(vec![at(1000), at(1100), at(1200)]).rage_click_count,
+ 1
+ );
+ }
+
+ #[test]
+ fn keeps_the_first_thousand_values_of_each_set_and_input() {
+ let mut events: Vec<Value> = (1..=1001)
+ .map(|id| incremental(1000 + id, json!({"source": 2, "type": 2, "id": id, "href": format!("https://a.example/{id}")})))
+ .collect();
+ events.extend((1..=1001).map(|id| input(5000 + id, Some(id), json!("ab"))));
+ // The text of the 1001st input is not remembered, so only the first shrink counts.
+ events.extend([
+ input(7000, Some(1001), json!("a")),
+ input(7001, Some(1), json!("a")),
+ ]);
+ let f = record(events);
+ let mut pages: Vec<String> = (1..=1000)
+ .map(|id| format!("https://a.example/{id}"))
+ .collect();
+ pages.sort_unstable();
+ assert_eq!(f.visited_urls, pages);
+ assert_eq!(f.click_target_ids, (1..=1000).collect::<Vec<_>>());
+ assert_eq!(f.form_field_ids, (1..=1000).collect::<Vec<_>>());
+ assert_eq!(f.backspace_count, 1);
+ }
+
+ #[test]
+ fn stores_each_float_sum_in_the_unit_its_column_names() {
+ let block = BlockFeatures {
+ mouse_sum_x: 1182.25,
+ mouse_sum_y_squared: 647_657.25,
+ mouse_distance_traveled: -1.0625,
+ mouse_velocity_sum: 0.5,
+ mouse_velocity_sum_of_squares: 0.25,
+ inter_action_gap_sum_of_squares_ms: 1e300,
+ network_request_duration_sum: 1170.35,
+ network_request_duration_sum_of_squares: 2.5,
+ max_scroll_y: 12_000.25,
+ max_idle_gap_ms: 40_000.0,
+ ..BlockFeatures::default()
+ };
+ let stored = SurfacingFeatures::from(&block);
+ assert_eq!(
+ [
+ stored.mouse_sum_x_millipx,
+ stored.mouse_sum_y_squared_px2,
+ stored.mouse_distance_traveled_millipx,
+ stored.mouse_velocity_sum_px_per_s,
+ stored.mouse_velocity_sum_of_squares_px2_per_s2,
+ stored.inter_action_gap_sum_of_squares_ms2,
+ stored.network_request_duration_sum_us,
+ stored.network_request_duration_sum_of_squares_ms2,
+ stored.max_scroll_y_millipx,
+ stored.max_idle_gap_ms,
+ ],
+ // Halves round away from zero, and a value past the range saturates.
+ [
+ 1_182_250,
+ 647_657,
+ -1_063,
+ 500,
+ 250_000,
+ i64::MAX,
+ 1_170_350,
+ 3,
+ 12_000_250,
+ 40_000
+ ]
+ );
+ }
+}
diff --git a/rust/ai-research/replay-core-v4/src/inspect/mod.rs b/rust/ai-research/replay-core-v4/src/inspect/mod.rs
index 8836d7d3ff28..2cdb26f1e535 100644
--- a/rust/ai-research/replay-core-v4/src/inspect/mod.rs
+++ b/rust/ai-research/replay-core-v4/src/inspect/mod.rs
@@ -4,6 +4,7 @@ pub mod clicks;
pub mod css;
pub mod dom;
pub mod events;
+pub mod features;
pub mod rrweb;
pub mod session;
pub mod stats;
diff --git a/rust/ai-research/replay-core-v4/src/inspect/session.rs b/rust/ai-research/replay-core-v4/src/inspect/session.rs
index 436326e4c446..9b009decdf6d 100644
--- a/rust/ai-research/replay-core-v4/src/inspect/session.rs
+++ b/rust/ai-research/replay-core-v4/src/inspect/session.rs
@@ -4,7 +4,8 @@ use std::sync::Arc;
use super::clicks::{ClickKind, ClickTracker};
use super::dom::{DomInspector, RemoteUrls, Srcset, UrlLimit};
-use super::events::{fields, number, Activity, EventCounter};
+use super::events::{number, Activity, DataFields, EventCounter};
+use super::features::{BlockFeatures, FeatureRecorder, SurfacingFeatures};
use super::rrweb::{
is_click, CUSTOM, FULL_SNAPSHOT, INCREMENTAL_SNAPSHOT, META, MOUSE_INTERACTION, MUTATION,
};
@@ -46,6 +47,7 @@ pub struct SessionInspector {
#[derive(Default)]
pub struct BlockInspection {
counter: EventCounter,
+ features: FeatureRecorder,
moments: Vec<Moment>,
image_urls: Vec<String>,
remote_urls: RemoteUrls,
@@ -55,6 +57,8 @@ pub struct BlockInspection {
/// What the inspector learned about one block.
pub struct Inspected {
pub stats: BlockStats,
+ /// The surfacing values as the TypeScript recorder computes them, which `stats.surfacing` stores in index units.
+ pub features: BlockFeatures,
pub moments: Vec<Moment>,
/// Remote image URLs the block's DOM referenced, for the image fetcher.
pub image_urls: Vec<String>,
@@ -68,8 +72,12 @@ pub struct Inspected {
impl BlockInspection {
pub fn finish(mut self) -> Inspected {
+ let features = self.features.finish();
+ let mut stats = self.counter.finish();
+ stats.surfacing = SurfacingFeatures::from(&features);
Inspected {
- stats: self.counter.finish(),
+ stats,
+ features,
moments: self.moments,
image_urls: self.image_urls,
url_limit: self.remote_urls.limit,
@@ -139,10 +147,18 @@ impl SessionInspector {
let index = block.events;
block.events += 1;
let navigations = block.counter.stats.navigations;
+ let data = DataFields::read(inner, event);
let carried_href =
block
.counter
- .observe(inner, event, &mut self.last_href, &mut self.activity);
+ .observe(inner, event, &data, &mut self.last_href, &mut self.activity);
+ let recorded = block.features.observe(inner, event, &data);
+ data.for_each_network_request(inner, |request| {
+ block.counter.count_network_request(request.status);
+ if recorded {
+ block.features.track_network_request(request);
+ }
+ });
let navigated = block.counter.stats.navigations > navigations;
let waiting = block.counter.stats.events_before_first_full_snapshot;
@@ -182,19 +198,17 @@ impl SessionInspector {
}
_ => None,
};
- if event.kind == Some(INCREMENTAL_SNAPSHOT) && event.source == Some(MOUSE_INTERACTION) {
- if let Some(data) = event.data {
- let [interaction, x, y] = fields(inner, data, [b"type", b"x", b"y"]);
- if number(inner, interaction).is_some_and(|t| is_click(t as u8)) {
- let position = number(inner, x).zip(number(inner, y));
- match self.clicks.click(ts, position) {
- ClickKind::Rage => stats.rage_clicks += 1,
- ClickKind::AfterDead => stats.dead_clicks += 1,
- ClickKind::Plain => {}
- }
- moment = Some("click");
- }
+ if event.kind == Some(INCREMENTAL_SNAPSHOT)
+ && event.source == Some(MOUSE_INTERACTION)
+ && number(inner, data.interaction).is_some_and(|t| is_click(t as u8))
+ {
+ let position = number(inner, data.x).zip(number(inner, data.y));
+ match self.clicks.click(ts, position) {
+ ClickKind::Rage => stats.rage_clicks += 1,
+ ClickKind::AfterDead => stats.dead_clicks += 1,
+ ClickKind::Plain => {}
}
+ moment = Some("click");
}
if let Some(kind) = moment {
if block.moments.len() < MAX_MOMENTS_PER_BLOCK {
diff --git a/rust/ai-research/replay-core-v4/src/inspect/stats.rs b/rust/ai-research/replay-core-v4/src/inspect/stats.rs
index aef9dcbdacdb..f9d626aabb30 100644
--- a/rust/ai-research/replay-core-v4/src/inspect/stats.rs
+++ b/rust/ai-research/replay-core-v4/src/inspect/stats.rs
@@ -1,6 +1,8 @@
//! One row of the v4 block index. Every value covers its block only, so a session row is a fold of its block rows:
//! [`fold_session`].
+use super::features::SurfacingFeatures;
+
/// Fingerprints a block or session row keeps. A session row with this many has visited this many pages or more.
pub const MAX_URL_FINGERPRINTS: usize = 10;
/// Window IDs a block or session row keeps. A session row with this many has this many windows or more.
@@ -86,6 +88,9 @@ pub struct BlockStats {
pub written_jsonl_bytes: u64,
/// The longest of those lines.
pub max_written_event_bytes: u64,
+
+ /// What ingestion's surfacing feature recorder records for the block's events as one batch.
+ pub surfacing: SurfacingFeatures,
}
impl BlockStats {
@@ -124,6 +129,7 @@ impl BlockStats {
MAX_URL_FINGERPRINTS,
);
union(&mut self.window_ids, &later.window_ids, MAX_WINDOW_IDS);
+ self.surfacing.merge(&later.surfacing);
self.touch_input |= later.touch_input;
self.set_viewport(
later.viewport_ts_ms,
@@ -207,6 +213,8 @@ fn min_option(a: Option<i64>, b: Option<i64>) -> Option<i64> {
#[cfg(test)]
mod tests {
+ use std::ops::Range;
+
use super::*;
fn block(first: i64, last: i64, host: &str, full_snapshots: u32, waiting: u32) -> BlockStats {
@@ -281,6 +289,77 @@ mod tests {
);
}
+ #[test]
+ fn folds_the_surfacing_features_the_same_in_any_order_and_grouping() {
+ let page = |n: usize| format!("{n:016x}");
+ let block =
+ |first: i64, clicks: u64, distance: i64, max_gap: i64, ids: Range<i64>| BlockStats {
+ first_ts_ms: Some(first),
+ surfacing: SurfacingFeatures {
+ click_count: clicks,
+ path_visit_counts: std::array::from_fn(|token| clicks * token as u64),
+ mouse_distance_traveled_millipx: distance,
+ mouse_sum_x_millipx: -distance,
+ max_idle_gap_ms: max_gap,
+ visited_url_fingerprints: ids.clone().map(|id| page(id as usize % 7)).collect(),
+ click_target_ids: ids.collect(),
+ ..SurfacingFeatures::default()
+ },
+ ..BlockStats::default()
+ };
+ let blocks = [
+ block(1_000, 3, 1_500, 40_000, 0..1_000),
+ block(2_000, 5, 2_500, 100, 500..1_500),
+ block(64_000, 1, 0, 70_000, 1_400..2_000),
+ BlockStats::default(),
+ ];
+ let whole = fold_session(blocks.to_vec()).surfacing;
+ let mut pages: Vec<String> = (0..7).map(page).collect();
+ pages.sort_unstable();
+ assert_eq!(
+ (
+ whole.click_count,
+ whole.path_visit_counts[2],
+ whole.mouse_distance_traveled_millipx,
+ whole.mouse_sum_x_millipx,
+ whole.max_idle_gap_ms,
+ whole.visited_url_fingerprints.clone(),
+ whole.click_target_ids.clone(),
+ ),
+ // The sets keep every distinct value, past the cap of one block.
+ (9, 18, 4_000, -4_000, 70_000, pages, (0..2_000).collect())
+ );
+ // Every order, each cut into two groups that fold first, as a reader that folds day by day does.
+ for order in permutations(blocks.len()) {
+ let ordered: Vec<BlockStats> = order.iter().map(|&i| blocks[i].clone()).collect();
+ for cut in 0..=ordered.len() {
+ let (earlier, later) = ordered.split_at(cut);
+ let parts = vec![fold_session(earlier.to_vec()), fold_session(later.to_vec())];
+ assert_eq!(
+ fold_session(parts).surfacing,
+ whole,
+ "{order:?} cut at {cut}"
+ );
+ }
+ }
+ }
+
+ fn permutations(n: usize) -> Vec<Vec<usize>> {
+ if n == 0 {
+ return vec![Vec::new()];
+ }
+ permutations(n - 1)
+ .into_iter()
+ .flat_map(|rest| {
+ (0..n).map(move |at| {
+ let mut order = rest.clone();
+ order.insert(at, n - 1);
+ order
+ })
+ })
+ .collect()
+ }
+
#[test]
fn counts_the_waiting_events_of_a_block_that_overlaps_the_snapshots_block() {
let main = block(1_000, 5_000, "a.example.com", 1, 2);
diff --git a/rust/ai-research/replay-core-v4/src/scan/message.rs b/rust/ai-research/replay-core-v4/src/scan/message.rs
index 52dfb4d99e42..892935f1e483 100644
--- a/rust/ai-research/replay-core-v4/src/scan/message.rs
+++ b/rust/ai-research/replay-core-v4/src/scan/message.rs
@@ -46,6 +46,12 @@ pub struct Event {
pub source: Option<u8>,
/// The event carries `cv` [`CV_VERSION`]: parts of its data are gzip or zstd stored as latin-1 codepoints.
pub compressed: bool,
+ /// `data` has a top-level `href` or `payload`, where `hrefFrom` in `rrweb-types.ts` looks for a page URL on any
+ /// event, so that an inspector reads a large `data` again only when it can hold one.
+ pub may_carry_page_url: bool,
+ /// Where `data.positions` starts in [`Message::inner`]. The reader passes it on its way to `source`, so that an
+ /// inspector reads a mouse move's positions without scanning its `data` again.
+ pub positions_start: Option<u32>,
/// The compressed strings decoded, once per event by [`Message::decode_compressed`]. `None` until then.
pub expansion: Option<Expansion>,
}
@@ -219,6 +225,8 @@ fn read_event(inner: &[u8], span: Span) -> Result<Event, ReadError> {
data: None,
source: None,
compressed: false,
+ may_carry_page_url: false,
+ positions_start: None,
expansion: None,
};
for entry in scan::object_entries(inner, span)? {
@@ -243,8 +251,14 @@ fn read_event(inner: &[u8], span: Span) -> Result<Event, ReadError> {
if let Some(data) = event.data.filter(|&d| scan::is_object(inner, d)) {
for entry in scan::object_entries(inner, data)? {
let entry = entry?;
- if !entry.key_escaped && &inner[entry.key.0..entry.key.1] == b"source" {
- event.source = small_uint(inner, entry.value);
+ if entry.key_escaped {
+ continue;
+ }
+ match &inner[entry.key.0..entry.key.1] {
+ b"source" => event.source = small_uint(inner, entry.value),
+ b"href" | b"payload" => event.may_carry_page_url = true,
+ b"positions" => event.positions_start = u32::try_from(entry.value.0).ok(),
+ _ => {}
}
}
}
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index dab9ac95a210..5dbdd24d6abb 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -98,6 +98,8 @@ pub struct Block {
pub url_limit: Option<UrlLimit>,
/// The image-slot srcset values of the block, which the writer rebuilds from the candidates it keeps.
pub srcsets: Vec<Srcset>,
+ /// The distinct pages of the block's surfacing features, which the writer stores as fingerprints.
+ pub visited_urls: Vec<String>,
/// The earliest Kafka timestamp of the block's messages, for the capture watermark of the index topic.
pub captured_at_ms: Option<i64>,
}
@@ -511,6 +513,7 @@ fn inspect_session(
remote_urls: inspected.remote_urls,
url_limit: inspected.url_limit,
srcsets: inspected.srcsets,
+ visited_urls: inspected.features.visited_urls,
captured_at_ms,
}
})
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 2c93ed8d8736..8fa8e2234e60 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -842,6 +842,7 @@ mod tests {
remote_urls: Vec::new(),
url_limit: None,
srcsets: Vec::new(),
+ visited_urls: Vec::new(),
captured_at_ms: None,
})
};
diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rs
index f930b118644e..535dc2826de0 100644
--- a/rust/ai-research/replay-recorder-v4/src/flush.rs
+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs
@@ -226,6 +226,8 @@ pub async fn write_flush(
stats[index].written_jsonl_bytes += block.counts.jsonl_bytes;
stats[index].max_written_event_bytes = block.counts.max_line_bytes;
stats[index].url_fingerprints = std::mem::take(&mut block.url_fingerprints);
+ stats[index].surfacing.visited_url_fingerprints =
+ std::mem::take(&mut block.visited_url_fingerprints);
}
// Before the index is written: building the jobs counts each block's image URLs into its stats. A dropped block's
// images are not stored, so the fetcher does not fetch them.
diff --git a/rust/ai-research/replay-recorder-v4/src/index.rs b/rust/ai-research/replay-recorder-v4/src/index.rs
index 675742df79a3..28b1c201e866 100644
--- a/rust/ai-research/replay-recorder-v4/src/index.rs
+++ b/rust/ai-research/replay-recorder-v4/src/index.rs
@@ -16,6 +16,7 @@
//! flush is whole when that many distinct `flush_row_index` values are present. Check this before any filter on
//! sessions or teams.
+use replay_core_v4::inspect::features::{SurfacingFeatures, PATH_TOKENS};
use replay_core_v4::inspect::session::Moment;
use replay_core_v4::inspect::stats::BlockStats;
use serde::ser::{SerializeMap, Serializer};
@@ -40,6 +41,10 @@ fn int32(value: u32) -> i32 {
i32::try_from(value).unwrap_or(i32::MAX)
}
+fn int32_count(value: u64) -> i32 {
+ i32::try_from(value).unwrap_or(i32::MAX)
+}
+
fn serialize_int64<S: Serializer>(value: &u64, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_i64(int64(*value))
}
@@ -102,6 +107,7 @@ fn serialize_stats<S: Serializer>(stats: &&BlockStats, serializer: S) -> Result<
touch_moves,
url_fingerprints,
window_ids,
+ surfacing,
} = *stats;
let mut map = serializer.serialize_map(None)?;
macro_rules! columns {
@@ -158,9 +164,134 @@ fn serialize_stats<S: Serializer>(stats: &&BlockStats, serializer: S) -> Result<
image_urls
);
map.serialize_entry("host", host)?;
+ serialize_surfacing(&mut map, surfacing)?;
map.end()
}
+/// The surfacing feature columns, each named for its field after the `surfacing_` prefix.
+fn serialize_surfacing<M: SerializeMap>(
+ map: &mut M,
+ surfacing: &SurfacingFeatures,
+) -> Result<(), M::Error> {
+ // Destructured in full, so a new field does not compile until it has a column here.
+ let SurfacingFeatures {
+ event_count,
+ click_count,
+ keypress_count,
+ mouse_activity_count,
+ rage_click_count,
+ dead_click_count,
+ quick_back_count,
+ page_visit_count,
+ text_selection_count,
+ scroll_event_count,
+ console_error_count,
+ console_error_after_click_count,
+ console_warn_count,
+ network_request_count,
+ network_failed_request_count,
+ network_4xx_count,
+ network_5xx_count,
+ network_request_duration_count,
+ mouse_position_count,
+ mouse_direction_change_count,
+ mouse_velocity_count,
+ scroll_direction_reversal_count,
+ rapid_scroll_reversal_count,
+ scroll_to_top_count,
+ inter_action_gap_count,
+ long_idle_gap_count,
+ backspace_count,
+ mutation_count,
+ viewport_resize_count,
+ touch_event_count,
+ selection_copy_count,
+ path_visit_counts,
+ mouse_sum_x_millipx,
+ mouse_sum_x_squared_px2,
+ mouse_sum_y_millipx,
+ mouse_sum_y_squared_px2,
+ mouse_distance_traveled_millipx,
+ mouse_velocity_sum_px_per_s,
+ mouse_velocity_sum_of_squares_px2_per_s2,
+ total_scroll_magnitude_millipx,
+ inter_action_gap_sum_ms,
+ inter_action_gap_sum_of_squares_ms2,
+ network_request_duration_sum_us,
+ network_request_duration_sum_of_squares_ms2,
+ max_scroll_y_millipx,
+ max_idle_gap_ms,
+ visited_url_fingerprints,
+ click_target_ids,
+ form_field_ids,
+ } = surfacing;
+ macro_rules! columns {
+ ($convert:expr => $($field:ident),*) => {
+ $( map.serialize_entry(concat!("surfacing_", stringify!($field)), &$convert($field))?; )*
+ };
+ }
+ columns!(
+ |v: &u64| int32_count(*v) =>
+ event_count,
+ click_count,
+ keypress_count,
+ mouse_activity_count,
+ rage_click_count,
+ dead_click_count,
+ quick_back_count,
+ page_visit_count,
+ text_selection_count,
+ scroll_event_count,
+ console_error_count,
+ console_error_after_click_count,
+ console_warn_count,
+ network_request_count,
+ network_failed_request_count,
+ network_4xx_count,
+ network_5xx_count,
+ network_request_duration_count,
+ mouse_position_count,
+ mouse_direction_change_count,
+ mouse_velocity_count,
+ scroll_direction_reversal_count,
+ rapid_scroll_reversal_count,
+ scroll_to_top_count,
+ inter_action_gap_count,
+ long_idle_gap_count,
+ backspace_count,
+ mutation_count,
+ viewport_resize_count,
+ touch_event_count,
+ selection_copy_count
+ );
+ for ((_, name), count) in PATH_TOKENS.iter().zip(path_visit_counts) {
+ map.serialize_entry(
+ &format!("surfacing_{name}_path_visit_count"),
+ &int32_count(*count),
+ )?;
+ }
+ columns!(
+ |v: &i64| safe(*v) =>
+ mouse_sum_x_millipx,
+ mouse_sum_x_squared_px2,
+ mouse_sum_y_millipx,
+ mouse_sum_y_squared_px2,
+ mouse_distance_traveled_millipx,
+ mouse_velocity_sum_px_per_s,
+ mouse_velocity_sum_of_squares_px2_per_s2,
+ total_scroll_magnitude_millipx,
+ inter_action_gap_sum_ms,
+ inter_action_gap_sum_of_squares_ms2,
+ network_request_duration_sum_us,
+ network_request_duration_sum_of_squares_ms2,
+ max_scroll_y_millipx,
+ max_idle_gap_ms
+ );
+ columns!(|v: &Vec<String>| v.clone() => visited_url_fingerprints);
+ columns!(|v: &Vec<i64>| v.iter().copied().map(safe).collect::<Vec<_>>() => click_target_ids, form_field_ids);
+ Ok(())
+}
+
fn serialize_moments<S: Serializer>(moments: &&[Moment], serializer: S) -> Result<S::Ok, S::Error> {
#[derive(Serialize)]
struct Entry<'a> {
@@ -367,6 +498,80 @@ mod tests {
("window_ids", "LIST<UTF8>", false),
("urls", "UTF8", true),
("urls_truncated", "BOOLEAN", false),
+ ("surfacing_event_count", "INT32", true),
+ ("surfacing_click_count", "INT32", true),
+ ("surfacing_keypress_count", "INT32", true),
+ ("surfacing_mouse_activity_count", "INT32", true),
+ ("surfacing_rage_click_count", "INT32", true),
+ ("surfacing_dead_click_count", "INT32", true),
+ ("surfacing_quick_back_count", "INT32", true),
+ ("surfacing_page_visit_count", "INT32", true),
+ ("surfacing_text_selection_count", "INT32", true),
+ ("surfacing_scroll_event_count", "INT32", true),
+ ("surfacing_console_error_count", "INT32", true),
+ ("surfacing_console_error_after_click_count", "INT32", true),
+ ("surfacing_console_warn_count", "INT32", true),
+ ("surfacing_network_request_count", "INT32", true),
+ ("surfacing_network_failed_request_count", "INT32", true),
+ ("surfacing_network_4xx_count", "INT32", true),
+ ("surfacing_network_5xx_count", "INT32", true),
+ ("surfacing_network_request_duration_count", "INT32", true),
+ ("surfacing_mouse_position_count", "INT32", true),
+ ("surfacing_mouse_direction_change_count", "INT32", true),
+ ("surfacing_mouse_velocity_count", "INT32", true),
+ ("surfacing_scroll_direction_reversal_count", "INT32", true),
+ ("surfacing_rapid_scroll_reversal_count", "INT32", true),
+ ("surfacing_scroll_to_top_count", "INT32", true),
+ ("surfacing_inter_action_gap_count", "INT32", true),
+ ("surfacing_long_idle_gap_count", "INT32", true),
+ ("surfacing_backspace_count", "INT32", true),
+ ("surfacing_mutation_count", "INT32", true),
+ ("surfacing_viewport_resize_count", "INT32", true),
+ ("surfacing_touch_event_count", "INT32", true),
+ ("surfacing_selection_copy_count", "INT32", true),
+ ("surfacing_login_path_visit_count", "INT32", true),
+ ("surfacing_signup_path_visit_count", "INT32", true),
+ ("surfacing_checkout_path_visit_count", "INT32", true),
+ ("surfacing_cart_path_visit_count", "INT32", true),
+ ("surfacing_billing_path_visit_count", "INT32", true),
+ ("surfacing_settings_path_visit_count", "INT32", true),
+ ("surfacing_account_path_visit_count", "INT32", true),
+ ("surfacing_error_path_visit_count", "INT32", true),
+ ("surfacing_not_found_path_visit_count", "INT32", true),
+ ("surfacing_admin_path_visit_count", "INT32", true),
+ ("surfacing_dashboard_path_visit_count", "INT32", true),
+ ("surfacing_onboarding_path_visit_count", "INT32", true),
+ ("surfacing_cancel_path_visit_count", "INT32", true),
+ ("surfacing_refund_path_visit_count", "INT32", true),
+ ("surfacing_mouse_sum_x_millipx", "INT64", true),
+ ("surfacing_mouse_sum_x_squared_px2", "INT64", true),
+ ("surfacing_mouse_sum_y_millipx", "INT64", true),
+ ("surfacing_mouse_sum_y_squared_px2", "INT64", true),
+ ("surfacing_mouse_distance_traveled_millipx", "INT64", true),
+ ("surfacing_mouse_velocity_sum_px_per_s", "INT64", true),
+ (
+ "surfacing_mouse_velocity_sum_of_squares_px2_per_s2",
+ "INT64",
+ true,
+ ),
+ ("surfacing_total_scroll_magnitude_millipx", "INT64", true),
+ ("surfacing_inter_action_gap_sum_ms", "INT64", true),
+ (
+ "surfacing_inter_action_gap_sum_of_squares_ms2",
+ "INT64",
+ true,
+ ),
+ ("surfacing_network_request_duration_sum_us", "INT64", true),
+ (
+ "surfacing_network_request_duration_sum_of_squares_ms2",
+ "INT64",
+ true,
+ ),
+ ("surfacing_max_scroll_y_millipx", "INT64", true),
+ ("surfacing_max_idle_gap_ms", "INT64", true),
+ ("surfacing_visited_url_fingerprints", "LIST<UTF8>", true),
+ ("surfacing_click_target_ids", "LIST<INT64>", true),
+ ("surfacing_form_field_ids", "LIST<INT64>", true),
];
const IMAGE_COLUMNS: &[(&str, &str, bool)] = &[
@@ -389,6 +594,9 @@ mod tests {
"LIST<UTF8>" => value
.as_array()
.is_some_and(|items| items.iter().all(|item| fits(item, "UTF8"))),
+ "LIST<INT64>" => value
+ .as_array()
+ .is_some_and(|items| items.iter().all(|item| fits(item, "INT64"))),
"MOMENTS" => value.as_array().is_some_and(|moments| {
moments.iter().all(|moment| {
let moment = moment.as_object().unwrap();
@@ -434,6 +642,15 @@ mod tests {
written_jsonl_bytes: u64::MAX,
url_fingerprints: vec!["b08517cee3ff5ecc".to_owned()],
window_ids: vec!["w1".to_owned(), "w2".to_owned()],
+ surfacing: SurfacingFeatures {
+ event_count: u64::MAX,
+ path_visit_counts: std::array::from_fn(|token| token as u64),
+ mouse_sum_x_millipx: i64::MIN,
+ max_idle_gap_ms: 30_001,
+ visited_url_fingerprints: vec!["0f1e2d3c4b5a6978".to_owned()],
+ click_target_ids: vec![-1, 7],
+ ..SurfacingFeatures::default()
+ },
..BlockStats::default()
};
let moments = [Moment {
@@ -500,6 +717,30 @@ mod tests {
&json!(["w1", "w2"]),
]
);
+ assert_eq!(
+ [
+ &value["surfacing_event_count"],
+ &value["surfacing_login_path_visit_count"],
+ &value["surfacing_not_found_path_visit_count"],
+ &value["surfacing_refund_path_visit_count"],
+ &value["surfacing_mouse_sum_x_millipx"],
+ &value["surfacing_max_idle_gap_ms"],
+ &value["surfacing_visited_url_fingerprints"],
+ &value["surfacing_click_target_ids"],
+ &value["surfacing_form_field_ids"],
+ ],
+ [
+ &json!(i32::MAX),
+ &json!(0),
+ &json!(8),
+ &json!(13),
+ &json!(-MAX_SAFE_INTEGER),
+ &json!(30_001),
+ &json!(["0f1e2d3c4b5a6978"]),
+ &json!([-1, 7]),
+ &json!([]),
+ ]
+ );
assert_eq!(
value["moments"],
json!([{"kind": "full_snapshot", "window_id": "w1", "ts_ms": MAX_SAFE_INTEGER, "event_index": i32::MAX}])
diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rs
index d014a5a7ba10..00475a835a1d 100644
--- a/rust/ai-research/replay-recorder-v4/src/urls.rs
+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs
@@ -208,6 +208,7 @@ mod tests {
remote_urls: Vec::new(),
url_limit: None,
srcsets: Vec::new(),
+ visited_urls: Vec::new(),
captured_at_ms: None,
};
let mut stats = BlockStats::default();
diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rs
index 748af3dce228..9d8e9168dcf9 100644
--- a/rust/ai-research/replay-recorder-v4/src/write.rs
+++ b/rust/ai-research/replay-recorder-v4/src/write.rs
@@ -84,6 +84,7 @@ pub struct CompressedBlock {
pub images: Vec<Arc<Image>>,
pub counts: WriterCounts,
pub url_fingerprints: Vec<String>,
+ pub visited_url_fingerprints: Vec<String>,
}
/// A block sealed for one object, with the images it moved out.
@@ -94,6 +95,7 @@ pub struct SealedBlock {
pub images: Vec<Arc<Image>>,
pub counts: WriterCounts,
pub url_fingerprints: Vec<String>,
+ pub visited_url_fingerprints: Vec<String>,
}
fn unique_suffix(at: DateTime<Utc>) -> String {
@@ -320,6 +322,7 @@ pub fn compress_block(
images: images.into_iter().map(Arc::new).collect(),
counts,
url_fingerprints: url_fingerprints(&key.key, &block_urls(block)),
+ visited_url_fingerprints: visited_url_fingerprints(&key.key, &block.visited_urls),
})
}
@@ -342,6 +345,7 @@ pub fn seal_compressed(
images: block.images.clone(),
counts: block.counts,
url_fingerprints: block.url_fingerprints.clone(),
+ visited_url_fingerprints: block.visited_url_fingerprints.clone(),
})
}
@@ -585,6 +589,16 @@ pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String>
fingerprints
}
+/// The block's distinct visited pages for its surfacing features, as fingerprints under the session's fingerprint key,
+/// sorted.
+pub fn visited_url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String> {
+ let key = url_fingerprint_key(session_key);
+ let mut fingerprints: Vec<String> = urls.iter().map(|url| url_fingerprint(&key, url)).collect();
+ fingerprints.sort_unstable();
+ fingerprints.dedup();
+ fingerprints
+}
+
/// Numbers the rows of each partition in order and gives each row the count of its partition.
fn number_rows(rows: &mut [BlockIndexRow<'_>]) {
let mut counts: HashMap<(&str, i32), u32> = HashMap::new();
@@ -771,6 +785,7 @@ mod tests {
remote_urls: Vec::new(),
url_limit: None,
srcsets: Vec::new(),
+ visited_urls: Vec::new(),
captured_at_ms: None,
}
}
@@ -1222,6 +1237,9 @@ mod tests {
{"type": 4, "timestamp": 4, "data": {"href": "https://app.example.com/a"}},
]);
let mut blocks = [block("kept", events.clone()), block("again", events)];
+ blocks[0].visited_urls = ["/b", "/a"]
+ .map(|path| format!("https://app.example.com{path}"))
+ .to_vec();
let mut keys = FlushKeys::default();
for session in ["kept", "again"] {
let (id, key) = session_key(session);
@@ -1265,6 +1283,9 @@ mod tests {
sealed[0].url_fingerprints,
[fingerprint("/a"), fingerprint("/b")]
);
+ let mut visited = [fingerprint("/a"), fingerprint("/b")];
+ visited.sort_unstable();
+ assert_eq!(sealed[0].visited_url_fingerprints, visited);
let session_frames = session_keyed_frames(&sealed);
let memory = Arc::new(MemoryStore::default());