:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai-research): fetch web fonts for offline v4 replay renders (#126401)
Robbiece0a2458e48c
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.tsindex 166725945ca6..e3041b58f5f9 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/fetch-runner.ts@@ -29,7 +29,7 @@ import { ImageFetchProcessingMetrics } from './processing-metrics'import type { ResponseUrlBinder } from './response-url-binding'import { ImageFetchTopHogMetrics } from './tophog-metrics'import { urlHistoryExpiresAtMs, urlHistoryStorageExpiresAtMs } from './url-history-expiry'-import { canonicalizeUrl } from './v4-native'+import { canonicalizeUrl, urlAssetOf } from './v4-native'export type ShedReason =| 'breaker_open'@@ -558,6 +558,7 @@ export class FetchRunner implements FetchPass {: undefinedconst fetched = await ImageFetchProcessingMetrics.measure('candidate_fetch', () =>this.fetcher.fetch(candidate.currentUrl, {+ asset: urlAssetOf(candidate.originalRef),sourcePartitions: candidate.sourcePartitions,maxBytes: this.options.maxBytes,timeoutMs: this.options.requestTimeoutMs,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.tsindex a78bb573236a..3cf178ff398e 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts@@ -136,9 +136,13 @@ describe('FrontierPublisher', () => {})})- it('groups delayed jobs by destination topic and registrable domain', async () => {+ it('groups delayed jobs by destination topic, registrable domain and asset', async () => {const { batch, sent } = build()const second = candidate({ originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}` })+ const font = candidate({+ originalRef: `fonturl:v4:7:2026-10:${'d'.repeat(22)}`,+ currentUrl: 'https://cdn.example.com/a.woff2',+ })const other = candidate({originalRef: `imageurl:v4:7:2026-10:${'c'.repeat(22)}`,currentUrl: 'https://img.other.net/c.png',@@ -148,17 +152,21 @@ describe('FrontierPublisher', () => {})await batch.republish(candidate(), candidate(), 'retry', 30_000)+ await batch.republish(font, font, 'retry', 30_000)await batch.republish(second, second, 'not_ready', 30_000)await batch.republish(other, other, 'retry', 30_000)await batch.flush()- expect(sent).toHaveLength(2)- const example = sent.find((item) => item.key === 'example.com')!- expect(example.topic).toBe('retry_1m')- expect(parseCollectedUrlsRecord(example.value, 'example.com')).toMatchObject({- ok: true,- candidates: [{ originalRef: candidate().originalRef }, { originalRef: second.originalRef }],- })+ expect(sent).toHaveLength(3)+ const example = sent.filter((item) => item.key === 'example.com')+ expect(example.map((item) => item.topic)).toEqual(['retry_1m', 'retry_1m'])+ expect(example.map((item) => parseCollectedUrlsRecord(item.value, 'example.com'))).toMatchObject([+ {+ ok: true,+ candidates: [{ originalRef: candidate().originalRef }, { originalRef: second.originalRef }],+ },+ { ok: true, candidates: [{ originalRef: font.originalRef }] },+ ])})it('keeps configured destination topics separate when they share a metric class', async () => {diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.tsindex 8bd4866ae4a8..da6817af55a3 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts@@ -11,6 +11,7 @@ import {serializeFrontierRecord,} from './collected-urls-record'import { ImageFetchRequestMetrics, RepublishTopic } from './metrics'+import { urlAssetOf } from './v4-native'export interface DelayTier {topic: string@@ -207,7 +208,8 @@ class BufferedRepublishBatch implements RepublishBatch {private planMessages(): PlannedRepublishMessage[] {const groups = new Map<string, PendingRepublish[]>()for (const item of this.pending) {- const key = `${item.destination.topic}\0${item.candidate.registrableDomain}`+ // An older fetcher dead-letters a whole record when it cannot parse one reference, so font and image jobs never share a record.+ const key = `${item.destination.topic}\0${item.candidate.registrableDomain}\0${urlAssetOf(item.candidate.originalRef)}`const group = groups.get(key)if (group) {group.push(item)diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.tsindex a4b51491583f..d49d64539eac 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.test.ts@@ -13,8 +13,10 @@ const fetchStreamedMock = fetchStreamed as jest.MockedFunction<typeof fetchStreaconst PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00])const GIF = Buffer.from('GIF89a-and-then-some')+const WOFF2 = Buffer.from('wOF2-and-then-some')const OPTIONS: ImageFetchOptions = {+ asset: 'image',maxBytes: 1000,timeoutMs: 5000,maxRedirects: 3,@@ -153,16 +155,20 @@ describe('HttpImageFetcher', () => {})it.each([- ['a type that is not an image', 'text/html', PNG, 'not_image'],- ['an SVG, which the store validates as a document', 'image/svg+xml', PNG, 'ok'],- ['the BMP format', 'image/bmp', Buffer.from('BM'), 'not_image'],- ['the AVIF format', 'image/avif', Buffer.from('\x00\x00\x00\x18ftypavif', 'binary'), 'not_image'],- ['a payload that is not the declared format', 'image/gif', PNG, 'ok'],- ['a payload of the wrong raster format', 'image/png', GIF, 'ok'],- ])('handles %s', async (_name, contentType, bytes, outcome) => {+ ['a type that is not an image', 'image', 'text/html', PNG, 'not_image'],+ ['an SVG, which the store validates as a document', 'image', 'image/svg+xml', PNG, 'ok'],+ ['the BMP format', 'image', 'image/bmp', Buffer.from('BM'), 'not_image'],+ ['the AVIF format', 'image', 'image/avif', Buffer.from('\x00\x00\x00\x18ftypavif', 'binary'), 'not_image'],+ ['a payload that is not the declared format', 'image', 'image/gif', PNG, 'ok'],+ ['a payload of the wrong raster format', 'image', 'image/png', GIF, 'ok'],+ ['a generic binary type for an image', 'image', 'application/octet-stream', PNG, 'not_image'],+ ['a generic binary type for a font', 'font', 'application/octet-stream', WOFF2, 'ok'],+ ['a legacy font type', 'font', 'application/x-font-woff', WOFF2, 'ok'],+ ['an image where a font is expected', 'font', 'image/png', PNG, 'not_image'],+ ] as const)('handles %s', async (_name, asset, contentType, bytes, outcome) => {fetchStreamedMock.mockResolvedValue(image(bytes, contentType))- const result = await fetcher().fetch('https://cdn.example.com/a', OPTIONS)+ const result = await fetcher().fetch('https://cdn.example.com/a', { ...OPTIONS, asset })expect(result).toMatchObject({ outcome })})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.tsindex 8995c977ffbc..8f85a4148535 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-fetcher.ts@@ -5,20 +5,11 @@ import { OriginPolicyReason, ResponseOptOutReason, responseOptOutReason } from 'import { HttpCacheMetadata } from './crawl-history'import { ImageFetchRequestMetrics } from './metrics'import { REQUEST_IDENTITY_HEADERS } from './request-identity'+import { URL_ASSET_RULES } from './url-asset-rules'import { canonicalizeUrl } from './v4-native'+import type { V4UrlAsset } from './v4-native'import { WebBotAuthRequestSigner } from './web-bot-auth'-/**- * The formats that the fetch lane accepts. The store keeps the bytes as fetched, so an SVG stays an SVG document. An SVG- * is text that can carry the page's own data, and redaction downstream of the dataset handles it.- *- * AVIF is absent because the image store validates only the formats in `SUPPORTED_IMAGE_MEDIA_TYPES`- * (`image-transport.ts`), so it would reject every AVIF image that this lane fetched.- */-const ALLOWED_CONTENT_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/svg+xml'] as const--export type ImageContentType = (typeof ALLOWED_CONTENT_TYPES)[number]-export type FetchOutcome =| 'ok'| 'not_found'@@ -49,7 +40,7 @@ export interface ImageFetchResult {currentUrl: stringstatus?: numberbytes?: Buffer- contentType?: ImageContentType+ contentType?: stringcontentEncoding?: stringcache?: HttpCacheMetadatarefusalReason?: FetchRefusalReason@@ -64,6 +55,7 @@ export interface ImageFetchResult {}export interface ImageFetchOptions {+ asset: V4UrlAssetsourcePartitions?: readonly number[]maxBytes: number/** Covers the redirect chain as a whole, so a chain of slow hops cannot outlive one hop's budget. */@@ -108,7 +100,6 @@ export interface ImageFetcher {const REQUEST_HEADERS: Record<string, string> = {...REQUEST_IDENTITY_HEADERS,- accept: 'image/*','accept-encoding': 'gzip, deflate, br, zstd',}@@ -151,6 +142,7 @@ export class HttpImageFetcher implements ImageFetcher {scheduled = await options.scheduleRequest(new URL(target), deadlineMs, () =>this.hop(target,+ options.asset,Math.max(1, deadlineMs - Date.now()),options.maxBytes,currentCache,@@ -219,6 +211,7 @@ export class HttpImageFetcher implements ImageFetcher {private async hop(url: string,+ asset: V4UrlAsset,timeoutMs: number,maxBytes: number,previousCache: HttpCacheMetadata | undefined,@@ -228,7 +221,12 @@ export class HttpImageFetcher implements ImageFetcher {const requestTimeMs = Date.now()const canonical = canonicalizeUrl(url)let requestOutcome = ImageFetchRequestMetrics.outcomeForHttpStatus()- const headers: Record<string, string> = { ...REQUEST_HEADERS, ...this.webBotAuthSigner.headersForGet(url) }+ const rules = URL_ASSET_RULES[asset]+ const headers: Record<string, string> = {+ ...REQUEST_HEADERS,+ accept: rules.accept,+ ...this.webBotAuthSigner.headersForGet(url),+ }if (previousCache?.etag) {headers['if-none-match'] = previousCache.etag} else if (previousCache?.lastModified) {@@ -267,7 +265,9 @@ export class HttpImageFetcher implements ImageFetcher {}}- const contentTypes = headerValues(response.headerLines, 'content-type').map(normalizeContentType)+ const contentTypes = headerValues(response.headerLines, 'content-type').map((header) =>+ normalizeContentType(header, rules.contentTypes)+ )const contentType = contentTypes[0]if (!contentType || contentTypes.some((candidate) => candidate !== contentType)) {response.discard()@@ -402,12 +402,12 @@ function resolveRedirect(from: string, location: string, policy: RedirectPolicy)return new URL(canonical.fetch)}-function normalizeContentType(header: string | undefined): ImageContentType | undefined {+function normalizeContentType(header: string | undefined, allowed: readonly string[]): string | undefined {if (!header) {return undefined}const type = header.split(';')[0].trim().toLowerCase()- return ALLOWED_CONTENT_TYPES.find((allowed) => allowed === type)+ return allowed.find((candidate) => candidate === type)}const ALLOWED_CONTENT_ENCODINGS = new Set(['gzip', 'deflate', 'br', 'zstd'])diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-transport.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-transport.test.tsindex 3dd9efcdefa5..ac0bc70dd84b 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-transport.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-transport.test.ts@@ -4,6 +4,7 @@ import {MAX_UNCOMPRESSED_IMAGE_BYTES,SUPPORTED_IMAGE_MEDIA_TYPES,imageBytesMatchMediaType,+ prepareFetchedFont,prepareFetchedImage,} from './image-transport'@@ -143,4 +144,33 @@ describe('image transport', () => {] as const)('labels %s', async (_case, bytes, contentType, reason) => {await expect(prepareFetchedImage(bytes, contentType, undefined)).rejects.toMatchObject({ reason })})++ it.each([+ ['WOFF2', Buffer.from('wOF2body'), 'font/woff2', null],+ [+ 'TrueType with a generic binary type',+ Buffer.from([0x00, 0x01, 0x00, 0x00, 0x00]),+ 'binary/octet-stream',+ null,+ ],+ ['OpenType', Buffer.from('OTTObody'), 'font/otf', null],+ [+ 'an HTML error page with a generic binary type',+ Buffer.from('<html>'),+ 'application/octet-stream',+ 'content_type_mismatch',+ ],+ [+ 'an Embedded OpenType font',+ Buffer.from('\x00\x00\x01\x00LP', 'binary'),+ 'application/vnd.ms-fontobject',+ 'unsupported_content_type',+ ],+ ['a font with a text type', Buffer.from('wOF2body'), 'text/plain', 'unsupported_content_type'],+ ] as const)('takes or refuses %s as a font', async (_case, bytes, contentType, reason) => {+ const prepared = prepareFetchedFont(gzipSync(bytes), contentType, 'gzip')+ await (reason === null+ ? expect(prepared).resolves.toEqual(bytes)+ : expect(prepared).rejects.toMatchObject({ reason }))+ })})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-transport.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-transport.tsindex adf3d0641a5c..a0373c944de6 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-transport.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/image-transport.ts@@ -18,6 +18,27 @@ export const SUPPORTED_IMAGE_MEDIA_TYPES = [export type SupportedImageMediaType = (typeof SUPPORTED_IMAGE_MEDIA_TYPES)[number]+/** Many servers label a font with a legacy or generic binary type, so the bytes decide what a font is. */+export const FONT_MEDIA_TYPES = [+ 'font/woff2',+ 'font/woff',+ 'font/ttf',+ 'font/otf',+ 'font/sfnt',+ 'font/collection',+ 'application/font-woff2',+ 'application/font-woff',+ 'application/x-font-woff',+ 'application/font-sfnt',+ 'application/x-font-ttf',+ 'application/x-font-truetype',+ 'application/x-font-otf',+ 'application/x-font-opentype',+ 'application/vnd.ms-opentype',+ 'application/octet-stream',+ 'binary/octet-stream',+] as const+export type ImageTransportRejectionReason =| 'content_encoding_too_deep'| 'malformed_content_encoding'@@ -235,7 +256,29 @@ function isSvgDocument(bytes: Buffer): boolean {return new RegExp(`\\sxmlns:${prefix}\\s*=\\s*["']http://www\\.w3\\.org/2000/svg["']`).test(tag)}-export async function prepareFetchedImage(+/** WOFF2, WOFF, OpenType with CFF outlines, TrueType (two signatures) and a TrueType or OpenType collection. */+const FONT_SIGNATURES = ['wOF2', 'wOFF', 'OTTO', '\x00\x01\x00\x00', 'true', 'ttcf']++function isFontFile(bytes: Buffer): boolean {+ return FONT_SIGNATURES.includes(bytes.toString('latin1', 0, 4))+}++async function decodeBody(bytes: Buffer, contentEncoding: string | undefined): Promise<Buffer> {+ if (bytes.length > MAX_UNCOMPRESSED_IMAGE_BYTES) {+ throw new InvalidImageTransportError(+ 'decoded_too_large',+ `decoded image exceeds ${MAX_UNCOMPRESSED_IMAGE_BYTES} bytes`+ )+ }+ let decoded = bytes+ const encodings = parseContentEncodings(contentEncoding)+ for (const encoding of encodings.reverse()) {+ decoded = await decodeWithLimit(decoded, encoding)+ }+ return decoded+}++export async function prepareFetchedFont(bytes: Buffer,contentType: string | undefined,contentEncoding: string | undefined@@ -243,21 +286,28 @@ export async function prepareFetchedImage(if (!contentType) {throw new InvalidImageTransportError('missing_content_type', 'unsupported content-type: missing')}- if (!isSupportedImageMediaType(contentType)) {+ if (!(FONT_MEDIA_TYPES as readonly string[]).includes(contentType)) {throw new InvalidImageTransportError('unsupported_content_type', `unsupported content-type: ${contentType}`)}- if (bytes.length > MAX_UNCOMPRESSED_IMAGE_BYTES) {- throw new InvalidImageTransportError(- 'decoded_too_large',- `decoded image exceeds ${MAX_UNCOMPRESSED_IMAGE_BYTES} bytes`- )+ const decoded = await decodeBody(bytes, contentEncoding)+ if (!isFontFile(decoded)) {+ throw new InvalidImageTransportError('content_type_mismatch', 'font bytes have no font signature')}+ return decoded+}- let decoded = bytes- const encodings = parseContentEncodings(contentEncoding)- for (const encoding of encodings.reverse()) {- decoded = await decodeWithLimit(decoded, encoding)+export async function prepareFetchedImage(+ bytes: Buffer,+ contentType: string | undefined,+ contentEncoding: string | undefined+): Promise<Buffer> {+ if (!contentType) {+ throw new InvalidImageTransportError('missing_content_type', 'unsupported content-type: missing')+ }+ if (!isSupportedImageMediaType(contentType)) {+ throw new InvalidImageTransportError('unsupported_content_type', `unsupported content-type: ${contentType}`)}+ const decoded = await decodeBody(bytes, contentEncoding)if (!imageBytesMatchMediaType(decoded, contentType)) {throw new InvalidImageTransportError('content_type_mismatch',diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.tsindex d373e12b816e..1ec711d950c9 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/job-admission.ts@@ -37,7 +37,7 @@ const DEFERRED = Symbol('deferred')/*** Drops a v4 job before any request when nothing could store its image: its team month has no live key, because the* key row expired, is missing or is a tombstone. The recorder stores the row before it sends a URL, so a missing row- * means it expired.+ * means it expired. A public font is stored for no team, so it needs no key.*/export class V4JobAdmission {private readonly keyLookups = pLimit(KEY_LOOKUP_CONCURRENCY)@@ -76,13 +76,17 @@ export class V4JobAdmission {budget: AbortSignal): Promise<FetchCandidate[]> {const keys = new Map<string, Promise<TeamMonthKey | typeof DEFERRED>>()- const live: { candidate: FetchCandidate; teamMonth: string }[] = []+ const live: { candidate: FetchCandidate; teamMonth?: string }[] = []for (const candidate of candidates) {const ref = parseV4UrlRef(candidate.originalRef)if (!ref) {droppedJobs.labels('foreign_ref').inc()continue}+ if (ref.scope === 'public') {+ live.push({ candidate })+ continue+ }// DynamoDB removes an expired row some time after `expires_at`, so the row may still be there.if (teamMonthKeyExpiresAtMs(ref.keyMonth) <= nowMs) {droppedJobs.labels('expired').inc()@@ -103,8 +107,10 @@ export class V4JobAdmission {const admitted: FetchCandidate[] = []let deferred = 0for (const { candidate, teamMonth } of live) {- const key = resolved.get(teamMonth)!- if (key === DEFERRED) {+ const key = teamMonth === undefined ? undefined : resolved.get(teamMonth)!+ if (key === undefined) {+ admitted.push(candidate)+ } else if (key === DEFERRED) {deferred++admitted.push({...candidate,diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.tsindex 9c0f5e5b94af..30b158ed7d16 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.test.ts@@ -9,7 +9,11 @@ import type { ImageFetchResult } from './image-fetcher'import { RawImageStore } from './raw-image-store'const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.from('png body')])+const WOFF2 = Buffer.from('wOF2 body')const REF = 'imageurl:v4:7:2026-10:abcdefghijklmnopqrstuv'+const FONT_REF = 'fonturl:v4:7:2026-10:abcdefghijklmnopqrstuv'+const PUBLIC_FONT_URL = 'https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu4mxK.woff2'+const PUBLIC_FONT_REF = 'fonturl:v4:public:I6bjVBXgjPcaYLPITibNM4'const KEY = Buffer.alloc(32, 9)const STORED: TeamMonthKey = { status: 'ok', key: KEY }@@ -53,45 +57,69 @@ const candidate = (ref = REF): FetchCandidate => ({republishCount: 0,lastRepublishReason: null,})-const fetched = (bytes = PNG): ImageFetchResult => ({+const fetched = (bytes = PNG, contentType = 'image/png'): ImageFetchResult => ({outcome: 'ok',redirects: 0,currentUrl: IMAGE_URL,status: 200,bytes,- contentType: 'image/png',+ contentType,})describe('RawImageStore', () => {- it('stores the image sealed under its team month key, over any earlier object of its reference', async () => {- const { sink, puts } = store(STORED)- await sink.publishImage(candidate(), fetched())+ it.each([+ ['image', REF, fetched(), PNG, 'images/url', 'image-url'],+ ['font', FONT_REF, fetched(WOFF2, 'font/woff2'), WOFF2, 'fonts/url', 'font-url'],+ ])(+ 'stores the %s sealed under its team month key, over any earlier object of its reference',+ async (_asset, ref, result, bytes, prefix, kind) => {+ const { sink, puts } = store(STORED)+ await sink.publishImage(candidate(ref), result)++ expect(puts).toHaveLength(1)+ expect(puts[0]).toBeInstanceOf(PutObjectCommand)+ const { Bucket, Key, IfNoneMatch, Body, ContentType } = (puts[0] as PutObjectCommand).input+ expect([Bucket, Key, IfNoneMatch]).toEqual([+ 'images-v4',+ `${prefix}/month=2026-10/7/abcdefghijklmnopqrstuv`,+ undefined,+ ])+ const { aad, body } = open(Body as Buffer)+ expect(body).toEqual(bytes)+ expect(aad).toEqual({+ v: 4,+ context: { teamId: 7, keyMonth: '2026-10', kind, codec: 'none', ref },+ })++ // The refresh copies the object onto itself, which resets the creation time the lifecycle counts from.+ await expect(sink.refreshImage(candidate(ref))).resolves.toBe(true)+ expect(puts[1]).toBeInstanceOf(CopyObjectCommand)+ expect((puts[1] as CopyObjectCommand).input).toEqual({+ Bucket,+ Key,+ CopySource: `images-v4/${prefix}/month%3D2026-10/7/abcdefghijklmnopqrstuv`,+ MetadataDirective: 'REPLACE',+ ContentType,+ })+ }+ )- expect(puts).toHaveLength(1)- expect(puts[0]).toBeInstanceOf(PutObjectCommand)- const { Bucket, Key, IfNoneMatch, Body, ContentType } = (puts[0] as PutObjectCommand).input- expect([Bucket, Key, IfNoneMatch]).toEqual([- 'images-v4',- 'images/url/month=2026-10/7/abcdefghijklmnopqrstuv',- undefined,+ it('stores a public font once and unsealed, with no team key, and refreshes that one object', async () => {+ const keyReads = jest.fn(() => Promise.resolve(STORED))+ const puts: (PutObjectCommand | CopyObjectCommand)[] = []+ const s3 = { send: jest.fn((command: PutObjectCommand | CopyObjectCommand) => puts.push(command)) }+ const sink = new RawImageStore({ get: keyReads }, s3 as any, 'images-v4', 10)+ const job = { ...candidate(PUBLIC_FONT_REF), currentUrl: PUBLIC_FONT_URL }++ await sink.publishImage(job, fetched(WOFF2, 'font/woff2'))+ await expect(sink.refreshImage(job)).resolves.toBe(true)++ expect(keyReads).not.toHaveBeenCalled()+ expect(puts.map((command) => [command.constructor.name, command.input.Key])).toEqual([+ ['PutObjectCommand', 'fonts/url/public/I6bjVBXgjPcaYLPITibNM4'],+ ['CopyObjectCommand', 'fonts/url/public/I6bjVBXgjPcaYLPITibNM4'],])- const { aad, body } = open(Body as Buffer)- expect(body).toEqual(PNG)- expect(aad).toEqual({- v: 4,- context: { teamId: 7, keyMonth: '2026-10', kind: 'image-url', codec: 'none', ref: REF },- })-- // The refresh copies the object onto itself, which resets the creation time the lifecycle counts from.- await expect(sink.refreshImage(candidate())).resolves.toBe(true)- expect(puts[1]).toBeInstanceOf(CopyObjectCommand)- expect((puts[1] as CopyObjectCommand).input).toEqual({- Bucket,- Key,- CopySource: 'images-v4/images/url/month%3D2026-10/7/abcdefghijklmnopqrstuv',- MetadataDirective: 'REPLACE',- ContentType,- })+ expect((puts[0] as PutObjectCommand).input.Body).toEqual(WOFF2)})it('resolves to false when it refreshes a reference that has no object', async () => {@@ -110,6 +138,12 @@ describe('RawImageStore', () => {candidate('imageurl:v3:7:2026-10:abcdefghijklmnopqrstuv'),fetched(),],+ [+ 'a public font reference whose job ended on another URL, such as a redirect target',+ STORED,+ candidate(PUBLIC_FONT_REF),+ fetched(WOFF2, 'font/woff2'),+ ],] as const)('resolves without a new object for %s', async (_name, key, job, result) => {const { sink, puts } = store(key)await expect(sink.publishImage(job, result)).resolves.toBeUndefined()diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.tsindex 3d8fcd5e48dd..5e616ccb363c 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/raw-image-store.ts@@ -7,22 +7,24 @@ import type { TeamMonthKeys } from '~/ingestion/pipelines/sessionreplay/ai-reseaimport type { FetchCandidate } from './collected-urls-record'import type { ImageFetchResult } from './image-fetcher'import type { RefreshingImageSink } from './image-sink'-import { InvalidImageTransportError, prepareFetchedImage } from './image-transport'+import { InvalidImageTransportError } from './image-transport'import { ImageFetchProcessingMetrics } from './processing-metrics'-import { parseV4UrlRef, sealFetchedImage } from './v4-native'+import { URL_ASSET_RULES } from './url-asset-rules'+import { V4UrlAsset, V4UrlRef, parseV4UrlRef, publicFontRefOf, sealFetchedImage } from './v4-native'const S3_WRITE_TIMEOUT_MS = 30_000const CONTENT_TYPE = 'application/octet-stream'const storedImages = new Counter({name: 'ai_research_v4_image_fetch_stored_images_total',- help: 'Fetched images the v4 lane handled, by outcome.',- labelNames: ['outcome'],+ help: 'Fetched images and fonts the v4 lane handled, by asset and outcome.',+ labelNames: ['asset', 'outcome'],})/*** Stores each fetched image sealed and unscrubbed in `images-v4`, under the team month key of its reference. The v4* dataset is anonymized downstream, so unlike the v3 lane this sink never sends an image to the scrub lane.+ * A public font is the same public file for every team, so it is stored once and unsealed.** S3 expires an object by its creation time, and blocks that reference an image are written all month. So a new fetch* overwrites the object of its reference, and `refreshImage` copies it onto itself to reset its age.@@ -53,7 +55,7 @@ export class RawImageStore implements RefreshingImageSink {public async refreshImage(candidate: FetchCandidate): Promise<boolean> {const ref = parseV4UrlRef(candidate.originalRef)if (!ref) {- storedImages.labels('foreign_ref').inc()+ storedImages.labels('unknown', 'foreign_ref').inc()return false}return await ImageFetchProcessingMetrics.runLimited(@@ -62,12 +64,12 @@ export class RawImageStore implements RefreshingImageSink {'image_publish_delivery',{debugTag: candidate.registrableDomain,- fn: () => this.copyOntoItself(imageObjectKey(ref)),+ fn: () => this.copyOntoItself(ref.asset, objectKey(ref)),})}- private async copyOntoItself(objectKey: string): Promise<boolean> {+ private async copyOntoItself(asset: V4UrlAsset, objectKey: string): Promise<boolean> {try {await this.s3.send(new CopyObjectCommand({@@ -83,12 +85,12 @@ export class RawImageStore implements RefreshingImageSink {} catch (error) {// The fetch stored nothing for this reference, for example because the body was not an image.if (error instanceof S3ServiceException && error.$metadata.httpStatusCode === 404) {- storedImages.labels('refresh_not_stored').inc()+ storedImages.labels(asset, 'refresh_not_stored').inc()return false}throw error}- storedImages.labels('refreshed').inc()+ storedImages.labels(asset, 'refreshed').inc()return true}@@ -96,45 +98,58 @@ export class RawImageStore implements RefreshingImageSink {const ref = parseV4UrlRef(candidate.originalRef)// A throw keeps the URL unfinished, and these inputs come back the same on every fetch.if (!ref) {- storedImages.labels('foreign_ref').inc()+ storedImages.labels('unknown', 'foreign_ref').inc()return}+ const { asset } = refif (!result.bytes) {- storedImages.labels('invalid').inc()+ storedImages.labels(asset, 'invalid').inc()return}- const { teamId, keyMonth } = ref- let image: Buffer+ // Every team reads this one object, so it must hold the file its reference names and not a redirect target.+ if (ref.scope === 'public' && publicFontRefOf(candidate.currentUrl) !== candidate.originalRef) {+ storedImages.labels(asset, 'not_public').inc()+ return+ }+ const rules = URL_ASSET_RULES[asset]+ let body: Buffertry {- image = await prepareFetchedImage(result.bytes, result.contentType, result.contentEncoding)+ body = await rules.prepare(result.bytes, result.contentType, result.contentEncoding)} catch (error) {// A body that is not the image it claims to be is handled: fetching it again returns the same body.if (error instanceof InvalidImageTransportError) {- storedImages.labels('invalid').inc()+ storedImages.labels(asset, 'invalid').inc()return}throw error}- const key = await this.keys.get(teamId, keyMonth)- if (key.status !== 'ok') {- storedImages.labels(key.status === 'deleted' ? 'deleted' : 'missing_key').inc()- return+ let stored = body+ if (ref.scope === 'team') {+ const { teamId, keyMonth } = ref+ const key = await this.keys.get(teamId, keyMonth)+ if (key.status !== 'ok') {+ storedImages.labels(asset, key.status === 'deleted' ? 'deleted' : 'missing_key').inc()+ return+ }+ stored = await sealFetchedImage(key.key, candidate.originalRef, body)}- const frame = await sealFetchedImage(key.key, candidate.originalRef, image)await this.s3.send(new PutObjectCommand({Bucket: this.bucket,- Key: imageObjectKey(ref),- Body: frame,+ Key: objectKey(ref),+ Body: stored,ContentType: CONTENT_TYPE,}),{ abortSignal: AbortSignal.timeout(S3_WRITE_TIMEOUT_MS) })- storedImages.labels('stored').inc()+ storedImages.labels(asset, 'stored').inc()}}/** One object per reference. The month is the reference's key month, not the day it was fetched. */-function imageObjectKey(ref: { teamId: number; keyMonth: string; hash: string }): string {- return `images/url/month=${ref.keyMonth}/${ref.teamId}/${ref.hash}`+function objectKey(ref: V4UrlRef): string {+ const prefix = URL_ASSET_RULES[ref.asset].objectKeyPrefix+ return ref.scope === 'public'+ ? `${prefix}/public/${ref.hash}`+ : `${prefix}/month=${ref.keyMonth}/${ref.teamId}/${ref.hash}`}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.tsindex 0cf0076f838c..18f1aafa7804 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/response-url-binding.ts@@ -3,7 +3,7 @@ import type {TeamMonthKeys,} from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/team-month-keys'-import { imageUrlHash, parseV4UrlRef } from './v4-native'+import { imageUrlHash, parseV4UrlRef, publicUrlHash } from './v4-native'/** The keyed hash that the crawl history stores in place of a response URL. */export type ResponseUrlBinding = (url: string) => string@@ -16,6 +16,7 @@ export interface ResponseUrlBinder {* Binds cached validators to the URL that issued them without storing that URL, which can hold personal data. The* binding is the HMAC of the URL under the image URL key of the reference's team month, so it compares only within* that team month. A reference with no live key has no binding, and its fetch sends no validators.+ * A public font's URL holds no personal data, so its binding is the unkeyed hash that its reference uses.*/export class TeamMonthResponseUrlBinder implements ResponseUrlBinder {constructor(private readonly keys: Pick<TeamMonthKeys, 'get'>) {}@@ -25,6 +26,9 @@ export class TeamMonthResponseUrlBinder implements ResponseUrlBinder {if (!ref) {return undefined}+ if (ref.scope === 'public') {+ return publicUrlHash+ }let key: TeamMonthKeytry {key = await this.keys.get(ref.teamId, ref.keyMonth, signal)diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.tsindex 37baab86bd02..3c1f9a993e55 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.test.ts@@ -9,25 +9,28 @@ const OK: ImageFetchResult = {bytes: Buffer.from('png'),contentType: 'image/png',}-const options = (tdmrepReservation: boolean) => ({ tdmrepReservation }) as ImageFetchOptions+const options = (tdmrepReservation: boolean, asset: ImageFetchOptions['asset'] = 'image') =>+ ({ tdmrepReservation, asset }) as ImageFetchOptionsdescribe('SharedImageFetcher', () => {it.each([- ['one after the other', false],- ['at the same time', true],- ])('fetches again for a job whose TDM reservation differs, %s', async (_name, concurrent) => {+ ['TDM reservation', 'one after the other', options(false), options(true)],+ ['TDM reservation', 'at the same time', options(false), options(true)],+ ['asset', 'one after the other', options(false), options(false, 'font')],+ ['asset', 'at the same time', options(false), options(false, 'font')],+ ])('fetches again for a job whose %s differs, %s', async (_field, timing, first, second) => {const inner = { fetch: jest.fn((_url: string, _options: ImageFetchOptions) => Promise.resolve(OK)) }const fetcher = new SharedImageFetcher(inner)- if (concurrent) {- await Promise.all([fetcher.fetch(IMAGE_URL, options(false)), fetcher.fetch(IMAGE_URL, options(true))])+ if (timing === 'at the same time') {+ await Promise.all([fetcher.fetch(IMAGE_URL, first), fetcher.fetch(IMAGE_URL, second)])} else {- await fetcher.fetch(IMAGE_URL, options(false))- await fetcher.fetch(IMAGE_URL, options(true))+ await fetcher.fetch(IMAGE_URL, first)+ await fetcher.fetch(IMAGE_URL, second)}- await fetcher.fetch(IMAGE_URL, options(true))+ await fetcher.fetch(IMAGE_URL, second)- expect(inner.fetch.mock.calls.map(([, used]) => used.tdmrepReservation)).toEqual([false, true])+ expect(inner.fetch.mock.calls.map(([, used]) => used)).toEqual([first, second])})it.each([diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.tsindex b1a64ad52458..76a5310afed7 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/shared-image-fetcher.ts@@ -37,8 +37,8 @@ export class SharedImageFetcher implements ImageFetcher {}public async fetch(url: string, options: ImageFetchOptions): Promise<ImageFetchResult> {- // The TDM reservation decides if a response is refused, so only jobs with the same reservation share a response.- const id = `${options.tdmrepReservation ? 'reserved' : 'open'} ${url}`+ // The TDM reservation decides if a response is refused and the asset decides its allowed types, so only jobs that agree on both share a response.+ const id = `${options.asset} ${options.tdmrepReservation ? 'reserved' : 'open'} ${url}`const recent = this.recent.get(id)if (recent) {sharedFetches.inc()diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-asset-rules.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-asset-rules.tsnew file mode 100644index 000000000000..e4a2825eded4--- /dev/null+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-asset-rules.ts@@ -0,0 +1,30 @@+import {+ FONT_MEDIA_TYPES,+ SUPPORTED_IMAGE_MEDIA_TYPES,+ prepareFetchedFont,+ prepareFetchedImage,+} from './image-transport'+import type { V4UrlAsset } from './v4-native'++export interface UrlAssetRules {+ accept: string+ contentTypes: readonly string[]+ prepare: (bytes: Buffer, contentType: string | undefined, contentEncoding: string | undefined) => Promise<Buffer>+ /** Each prefix needs a write grant for the fetcher's role in posthog-cloud-infra (`v4-staging.hcl`). */+ objectKeyPrefix: 'images/url' | 'fonts/url'+}++export const URL_ASSET_RULES: Record<V4UrlAsset, UrlAssetRules> = {+ image: {+ accept: 'image/*',+ contentTypes: SUPPORTED_IMAGE_MEDIA_TYPES,+ prepare: prepareFetchedImage,+ objectKeyPrefix: 'images/url',+ },+ font: {+ accept: 'font/woff2, font/woff, font/ttf, font/otf, */*;q=0.1',+ contentTypes: FONT_MEDIA_TYPES,+ prepare: prepareFetchedFont,+ objectKeyPrefix: 'fonts/url',+ },+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.tsindex f2e01a3ae899..203df8054bac 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/url-history-expiry.ts@@ -16,10 +16,11 @@ const MONTH_GRACE_DAYS = 8* When the history entry of a reference may go. A v4 reference carries the month it was written in, so the recorder* sends no job for it after that month. Its entry stays 8 more days for jobs that are still on their way, and a later* job finds no entry and fetches the image again.+ * A public font reference has no month, so every team's jobs share its entry until its next fetch.*/export function urlHistoryStorageExpiresAtMs(ref: string, nextFetchAtMs: number): number {const v4 = parseV4UrlRef(ref)- if (!v4) {+ if (!v4 || v4.scope === 'public') {return nextFetchAtMs}const [year, month] = v4.keyMonth.split('-').map(Number)diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.tsindex 4f5dde7d842f..beea9dd80901 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-lane.test.ts@@ -30,6 +30,7 @@ const fetchStreamedMock = fetchStreamed as jest.MockedFunction<typeof fetchStreaconst NOW_MS = Date.parse('2026-10-15T12:00:00Z')const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.from('png body')])+const WOFF2 = Buffer.from('wOF2 body')const TEAM_KEYS: Record<string, Buffer> = {'team:7': Buffer.from(vector.teamMonthKey, 'base64'),'team:8': Buffer.alloc(32, 8),@@ -49,7 +50,7 @@ function respond(status: number, headers: Record<string, string> = {}, bytes = B}/** A record in the shape `replay-recorder-v4/src/urls.rs` produces: keyed by registrable domain, with no headers. */-function recorderRecord(jobs: { ref: string; url: string }[]): Message {+function recorderRecord(jobs: { ref: string; url: string }[], registrableDomain = 'example.com'): Message {const value = Buffer.from(JSON.stringify({v: 2,@@ -66,7 +67,7 @@ function recorderRecord(jobs: { ref: string; url: string }[]): Message {})),}))- return { value, key: Buffer.from('example.com'), size: value.length, topic: 'v4', partition: 0, offset: 0 }+ return { value, key: Buffer.from(registrableDomain), size: value.length, topic: 'v4', partition: 0, offset: 0 }}function openFrame(key: Buffer, frame: Buffer): { context: Record<string, unknown>; body: Buffer } {@@ -81,25 +82,27 @@ function openFrame(key: Buffer, frame: Buffer): { context: Record<string, unknowreturn { context: parseJSON(aad.toString()).context, body }}-/** The origin's robots.txt and tdmrep.json are known for a year, so the jobs do not wait on one configuration fetch. */+/** The origins' robots.txt and tdmrep.json are known for a year, so the jobs do not wait on one configuration fetch. */class MemoryCrawlHistory implements CrawlHistoryStore {public readonly items = new Map<string, CrawlHistoryItem>(- (['robots', 'tdmrep'] as const).map((kind) => {- const origin = new URL(vector.url.url).origin- const nowMs = Date.now()- const item: CrawlHistoryItem = {- kind,- key: configurationCacheKey(origin, kind),- origin,- status: 'absent',- fetchedAtMs: nowMs,- refreshAtMs: nowMs + 365 * DAY_MS,- freshUntilMs: nowMs + 365 * DAY_MS,- retryAtMs: nowMs,- storageExpiresAtMs: nowMs + 365 * DAY_MS,- }- return [item.key, item]- })+ [vector.url.url, vector.publicFontUrl.url].flatMap((url) =>+ (['robots', 'tdmrep'] as const).map((kind) => {+ const origin = new URL(url).origin+ const nowMs = Date.now()+ const item: CrawlHistoryItem = {+ kind,+ key: configurationCacheKey(origin, kind),+ origin,+ status: 'absent',+ fetchedAtMs: nowMs,+ refreshAtMs: nowMs + 365 * DAY_MS,+ freshUntilMs: nowMs + 365 * DAY_MS,+ retryAtMs: nowMs,+ storageExpiresAtMs: nowMs + 365 * DAY_MS,+ }+ return [item.key, item] as const+ })+ ))read(keys: string[]): Promise<Map<string, CrawlHistoryItem>> {return Promise.resolve(@@ -259,6 +262,25 @@ describe('v4 image fetch lane', () => {expect(JSON.stringify(urlHistory)).not.toContain(new URL(vector.url.url).host)})+ it('stores a public font once and unsealed with no team key, so a later job sends no request', async () => {+ const harness = lane({})+ runners.push(harness.runner)+ fetchStreamedMock.mockImplementation(() =>+ Promise.resolve(respond(200, { 'content-type': 'font/woff2' }, WOFF2))+ )+ const font = { ref: vector.publicFontUrl.ref, url: vector.publicFontUrl.url }++ await harness.consumer.handleBatch([recorderRecord([font], 'gstatic.com')], NOW_MS)+ await harness.consumer.handleBatch([recorderRecord([font], 'gstatic.com')], NOW_MS + DAY_MS)++ expect(harness.deadLetters.park).not.toHaveBeenCalled()+ expect(harness.dynamo.send).not.toHaveBeenCalled()+ expect(fetchStreamedMock.mock.calls.map(([url]) => url)).toEqual([font.url])+ expect(harness.puts.map((put) => [put.input.Key, put.input.Body])).toEqual([+ [`fonts/url/public/${font.ref.split(':').pop()}`, WOFF2],+ ])+ })+it.each([// The row of 2026-05 expired on 2026-10-01, and DynamoDB may not have removed it yet.['a month whose key expired', row(7, '2026-05'), `imageurl:v4:7:2026-05:${OTHER_HASH}`],diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.test.tsindex 93d5e24e3855..c9dd85bcb8dd 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.test.ts@@ -1,4 +1,13 @@-import { isPublicHost, parseV4UrlRef, politenessKey, tryCanonicalizeUrl } from '@posthog/ai-research-v4-native'+import {+ isPublicHost,+ parseV4UrlRef,+ politenessKey,+ publicFontRefOf,+ tryCanonicalizeUrl,+} from '@posthog/ai-research-v4-native'++import vector from '~/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json'+import { parseImageRef } from '~/ingestion/pipelines/sessionreplay/ml-mirror-image-scrub/content-ref'/*** The crate owns the rule and tests it. This file tests only that the fetch lane reaches that@@ -48,14 +57,39 @@ describe('the v4 url policy addon', () => {})})- it('parses only an image URL reference whose team id a number holds exactly', () => {+ describe('parseV4UrlRef', () => {const hash = 'a'.repeat(22)- expect(parseV4UrlRef(`imageurl:v4:7:2026-10:${hash}`)).toEqual({ teamId: 7, keyMonth: '2026-10', hash })- expect(parseV4UrlRef(`image:v4:7:2026-10:${hash}`)).toBeNull()- expect(parseV4UrlRef(`imageurl:v4:9007199254740993:2026-10:${hash}`)).toBeNull()++ it.each([+ ['imageurl', 'image'],+ ['fonturl', 'font'],+ ])('parses a month-scoped v4 %s ref that no v2 or v3 reader takes', (prefix, asset) => {+ const ref = `${prefix}:v4:7:2026-10:${hash}`+ expect(parseV4UrlRef(ref)).toEqual({ scope: 'team', asset, teamId: 7, keyMonth: '2026-10', hash })+ expect(parseImageRef(ref)).toBeNull()+ })++ it('names a public font file by the same reference as the recorder, and no other URL', () => {+ const ref = publicFontRefOf(vector.publicFontUrl.url)+ expect(ref).toBe(vector.publicFontUrl.ref)+ expect(parseV4UrlRef(ref!)).toEqual({ scope: 'public', asset: 'font', hash: ref!.split(':').pop() })+ expect(parseImageRef(ref!)).toBeNull()+ expect(vector.publicFontUrl.teamUrls.map(publicFontRefOf)).toEqual(+ vector.publicFontUrl.teamUrls.map(() => null)+ )+ })++ it.each([+ ['a content ref', `image:v4:7:2026-10:${hash}`],+ ['a team id that a number cannot hold exactly', `imageurl:v4:9007199254740993:2026-10:${hash}`],+ ['a bundled font marker, which names no URL', 'fonturl:v4:bundled:open-sans'],+ ['a public image ref, which the recorder never writes', `imageurl:v4:public:${hash}`],+ ])('refuses %s', (_name, ref) => {+ expect(parseV4UrlRef(ref)).toBeNull()+ })})- it.each(['urlJob', 'isUrlHistoryValue', 'parseV4UrlRef', 'sealFetchedImage', 'imageUrlHash'])(+ it.each(['urlJob', 'isUrlHistoryValue', 'parseV4UrlRef', 'sealFetchedImage', 'imageUrlHash', 'publicUrlHash'])('stops the fetch pod at startup when index.node has no %s',(missing) => {const nativePath = require.resolve('@posthog/ai-research-v4-native/index.node')diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.tsindex 5efa35be2e42..b37ecb05298c 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/v4-native.ts@@ -18,7 +18,9 @@ function addon(): typeof import('@posthog/ai-research-v4-native') {typeof loaded.isUrlHistoryValue !== 'function' ||typeof loaded.parseV4UrlRef !== 'function' ||typeof loaded.sealFetchedImage !== 'function' ||- typeof loaded.imageUrlHash !== 'function'+ typeof loaded.imageUrlHash !== 'function' ||+ typeof loaded.publicFontRefOf !== 'function' ||+ typeof loaded.publicUrlHash !== 'function') {throw new Error('the ai-research-v4-native addon is missing a shared rule: rebuild index.node')}@@ -51,8 +53,9 @@ function runsSharedRules(): boolean {return (addon().urlJob(1, undefined, 0) === 'drop' &&addon().isUrlHistoryValue('probe', entry) &&- addon().parseV4UrlRef(PROBE_REF)?.teamId === 1 &&- addon().imageUrlHash(key, 'https://example.com/probe.png').length === 22+ addon().parseV4UrlRef(PROBE_REF)?.hash === 'AAAAAAAAAAAAAAAAAAAAAA' &&+ addon().imageUrlHash(key, 'https://example.com/probe.png').length === 22 &&+ addon().publicUrlHash('https://example.com/probe.png').length === 22)} catch {return false@@ -94,10 +97,16 @@ export function isUrlHistoryValue(key: string, value: string): boolean {return addon().isUrlHistoryValue(key, value)}-export function parseV4UrlRef(reference: string): { teamId: number; keyMonth: string; hash: string } | null {+export type { V4UrlAsset, V4UrlRef } from '@posthog/ai-research-v4-native'++export function parseV4UrlRef(reference: string): import('@posthog/ai-research-v4-native').V4UrlRef | null {return addon().parseV4UrlRef(reference)}+export function urlAssetOf(originalRef: string): import('@posthog/ai-research-v4-native').V4UrlAsset {+ return originalRef.startsWith('fonturl:') ? 'font' : 'image'+}+export function sealFetchedImage(teamMonthKey: Buffer, reference: string, image: Buffer): Promise<Buffer> {return addon().sealFetchedImage(teamMonthKey, reference, image)}@@ -105,3 +114,11 @@ export function sealFetchedImage(teamMonthKey: Buffer, reference: string, image:export function imageUrlHash(teamMonthKey: Buffer, url: string): string {return addon().imageUrlHash(teamMonthKey, url)}++export function publicFontRefOf(fetchUrl: string): string | null {+ return addon().publicFontRefOf(fetchUrl)+}++export function publicUrlHash(url: string): string {+ return addon().publicUrlHash(url)+}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.jsonindex ccd6033b918c..43229967c958 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json@@ -56,6 +56,32 @@"nonce": "BgYGBgYGBgYGBgYG","frame": "QUlTUjAzAIp7ImNvbnRleHQiOnsiY29kZWMiOiJub25lIiwia2V5TW9udGgiOiIyMDI2LTEwIiwia2luZCI6ImltYWdlLXVybCIsInJlZiI6ImltYWdldXJsOnY0Ojc6MjAyNi0xMDpURVgwYWpDMFVrYU9DOWpzNWM3b3FyIiwidGVhbUlkIjo3fSwidiI6NH0GBgYGBgYGBgYGBga8E3CK+Pm3JojWgEgphrSGIg8HFZFiLLkBK3fy+X8ag5A8Kzp6xVYHOyi6K6kvXaY="},+ "fontUrl": {+ "url": "https://fonts.example.com/brand.woff2",+ "bytes": "c3ludGhldGljIHY0IGZldGNoZWQgZm9udCBieXRlcw==",+ "ref": "fonturl:v4:7:2026-10:Bwu1146AUxxsfd2I6Co_JI",+ "context": {+ "teamId": 7,+ "keyMonth": "2026-10",+ "kind": "font-url",+ "codec": "none",+ "ref": "fonturl:v4:7:2026-10:Bwu1146AUxxsfd2I6Co_JI"+ },+ "nonce": "BwcHBwcHBwcHBwcH",+ "frame": "QUlTUjAzAIh7ImNvbnRleHQiOnsiY29kZWMiOiJub25lIiwia2V5TW9udGgiOiIyMDI2LTEwIiwia2luZCI6ImZvbnQtdXJsIiwicmVmIjoiZm9udHVybDp2NDo3OjIwMjYtMTA6Qnd1MTE0NkFVeHhzZmQySTZDb19KSSIsInRlYW1JZCI6N30sInYiOjR9BwcHBwcHBwcHBwcHKsLj6wAUi3AUyiG5tqQWkl5QYy6gap0r6q5z6YcT8NO5Z029g474FZHmPA8DY1o="+ },+ "publicFontUrl": {+ "url": "https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu4mxK.woff2",+ "ref": "fonturl:v4:public:I6bjVBXgjPcaYLPITibNM4",+ "teamUrls": [+ "https://fonts.gstatic.com/l/font?kit=example-subset&skey=0000&v=v30",+ "http://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu4mxK.woff2",+ "https://fonts.gstatic.com:8443/s/roboto/v30/KFOmCnqEu92Fr1Mu4mxK.woff2",+ "https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu4mxK.woff2?v=1",+ "https://fonts.gstatic.com/roboto/v30/KFOmCnqEu92Fr1Mu4mxK.woff2",+ "https://cdn.example.com/s/roboto/v30/KFOmCnqEu92Fr1Mu4mxK.woff2"+ ]+ },"url": {"url": "https://images.example.com/logo.png","ref": "imageurl:v4:7:2026-10:TEX0ajC0UkaOC9js5c7oqr"diff --git a/products/ai_training/docs/replay-data.md b/products/ai_training/docs/replay-data.mdindex 2bf7a3ccf3e2..b7979e310896 100644--- a/products/ai_training/docs/replay-data.md+++ b/products/ai_training/docs/replay-data.md@@ -404,6 +404,7 @@ UpdateItem SET deleted = :true, expires_at = if_not_exists(expires_at, :expiry)| Session, and person through its sessions | The session row | The session's blocks and its sealed index values. Its `url_fingerprints` can no longer be linked to a URL || Team | The team month rows of the previous, current and next UTC month, then every `month:` row of the team, then every session row in the 32 shards | Everything of the team: blocks, inline images, fetched images and sealed index values |+- A public font file (see [Web fonts](#web-fonts)) belongs to no team, so no deletion reaches it. It is a file that Google serves to anyone, and the blocks that say where a session used it are sealed.- A session tombstone does not reach inline or fetched images. Both are sealed under the team month key, and other sessions of the team can refer to the same image. They stay readable with that key until they expire (see [Lifecycle](#lifecycle)), but the deleted session's blocks, which say where the session used them, cannot be read.- A tombstone the worker creates for a row that does not exist yet gets an `expires_at` by the key rule, from the row's month for a team month row and from the current month for a session row. A tombstone that replaces a key keeps the key's expiry.- The worker reads only `pk` and `sk` and has no KMS access. Its role has Query and UpdateItem on the v4 key table.@@ -416,19 +417,20 @@ UpdateItem SET deleted = :true, expires_at = if_not_exists(expires_at, :expiry)Nothing in v4 is kept for good. Each kind of data has a fixed lifetime, and a deletion only makes data unreadable sooner.-| Data | Where | Lifetime | Removed by |-| ------------------------------ | ---------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- |-| Session blocks | `rrweb-v4` `blocks/` | 30 days after the write | S3 expiration |-| Block and image index | `rrweb-v4` `index/` | 30 days after the write | S3 expiration |-| Inline image packs | `images-v4` `images/packs/` | 38 days after the write | S3 expiration |-| Fetched images | `images-v4` `images/url/` | 38 days after the last write. The fetcher copies an image onto itself when a job refers to it and the copy is more than 7 days old | S3 expiration |-| Team month and session keys | `ai-research-replay-v4-keys` | Until the start of the fifth month after `key_month` | DynamoDB TTL |-| Tombstones | `ai-research-replay-v4-keys` | The same expiry as the row they replace | DynamoDB TTL |-| Fetch history and robots cache | `ai-research-v4-image-fetch-history` | URL history: 8 days after the end of its month. Robots and tdmrep cache: at least 30 days | DynamoDB TTL |-| Index records | Kafka `ai_research_v4_index` | 7 days | Topic retention |-| Fetch jobs, with raw URLs | Kafka `ai_research_v4_image_fetch`, its retry topics and its DLQ | 24 hours | Topic retention |-| Seen and blocked sessions | `ai-research-v4` Redis, `@posthog/ai-research-v4/` | The main lane's expiry, or earlier under memory pressure | Redis expiry and LRU eviction |-| Held keys | Recorder and fetcher memory | 5 minutes for a session row, one hour for a team month row | Cache expiry |+| Data | Where | Lifetime | Removed by |+| ------------------------------ | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- |+| Session blocks | `rrweb-v4` `blocks/` | 30 days after the write | S3 expiration |+| Block and image index | `rrweb-v4` `index/` | 30 days after the write | S3 expiration |+| Inline image packs | `images-v4` `images/packs/` | 38 days after the write | S3 expiration |+| Fetched images | `images-v4` `images/url/` | 38 days after the last write. The fetcher copies an image onto itself when a job refers to it and the copy is more than 7 days old | S3 expiration |+| Fetched fonts | `images-v4` `fonts/url/` | The same as fetched images | S3 expiration |+| Team month and session keys | `ai-research-replay-v4-keys` | Until the start of the fifth month after `key_month` | DynamoDB TTL |+| Tombstones | `ai-research-replay-v4-keys` | The same expiry as the row they replace | DynamoDB TTL |+| Fetch history and robots cache | `ai-research-v4-image-fetch-history` | URL history: 8 days after the end of its month, and a public font's history until its next fetch. Robots and tdmrep cache: at least 30 days | DynamoDB TTL |+| Index records | Kafka `ai_research_v4_index` | 7 days | Topic retention |+| Fetch jobs, with raw URLs | Kafka `ai_research_v4_image_fetch`, its retry topics and its DLQ | 24 hours | Topic retention |+| Seen and blocked sessions | `ai-research-v4` Redis, `@posthog/ai-research-v4/` | The main lane's expiry, or earlier under memory pressure | Redis expiry and LRU eviction |+| Held keys | Recorder and fetcher memory | 5 minutes for a session row, one hour for a team month row | Cache expiry |- S3 expiration counts from the time an object was written, rounds up to the next midnight UTC and can run late, so plan for objects to stay for up to 2 days more.- Blocks expire by write time, not by session start, so a session whose blocks span several write days loses its first blocks first. Nothing marks the rest as partial, and `cold_start` does not cover it. Readers that need whole sessions follow the rule in [Block index](#block-index).@@ -449,6 +451,7 @@ The additional authenticated data is the canonical JSON of `{"v": 4, "context":| Session block | session key | `teamId`, `sessionId`, `keyMonth`, `kind: "rrweb"`, `codec: "brotli"`, `ref` (the block object key) || Inline image in a pack | team month key | `teamId`, `keyMonth`, `kind: "image"`, `codec: "none"`, `ref` (the image reference) || Fetched image | team month key | `teamId`, `keyMonth`, `kind: "image-url"`, `codec: "none"`, `ref` (the image URL reference) |+| Fetched font | team month key | `teamId`, `keyMonth`, `kind: "font-url"`, `codec: "none"`, `ref` (the font URL reference) || Sealed index value | session key | `teamId`, `sessionId`, `keyMonth`, `kind: "index"`, `ref` (the column name) |A sealed index value uses the v3 JSON envelope with `"v": 4`.@@ -471,7 +474,8 @@ The recorder streams each block into the encoder. Read a block as a stream: a Br- The recorder reads every argument of a canvas mutation, not only the frames of the posthog-js canvas recorder. A buffer becomes a reference when its bytes are PNG, JPEG, GIF or WebP, and a marker when they are not. The pixels of an `ImageData`, and the numbers of a `Uint8Array` or a `Uint8ClampedArray`, become `image:v4:declined:unsupported`. A data URI in the `src` of an `HTMLImageElement` moves out like any data URI, and a remote URL there stays as it is.- A remote image URL becomes `imageurl:v4:{team_id}:{YYYY-MM}:{hash}` in the block and in its fetch job, where the month is the UTC month of the write. The block and the job always carry the same reference, so a reader opens the fetched image directly.- A URL that the URL policy refuses becomes `imageurl:v4:declined:{reason}` in the block, for example `declined:credential`, `declined:non_public_host` or `declined:tracking_beacon`. The original URL cannot be recovered. The recorder counts rewrites and refusals in `ai_research_v4_recorder_image_url_rewrites_total`.-- The `v4_image_refs` module of the `posthog-replay-anonymizer` crate defines each reference and marker shape. The recorder writes them with it. A reader that must tell them apart from page text uses its `parse` function, which accepts only the exact strings the recorder writes.+- Each `src` URL of an `@font-face` rule becomes a font reference in the same way, as [Web fonts](#web-fonts) describes. `ai_research_v4_recorder_image_url_rewrites_total` counts both, by `kind`: `image`, `font` or `bundled_font`.+- The `v4_image_refs` module of the `posthog-replay-anonymizer` crate defines each reference and marker shape, for images and for fonts. The recorder writes them with it. A reader that must tell them apart from page text uses its `parse` function, which accepts only the exact strings the recorder writes.- The rewrite covers every remote URL that the recorder reads from an image slot: image sources, `srcset`, SVG image links, video posters, `<picture>` sources and CSS `url(...)` in style attributes, style sheets and style rules, in custom properties too, because a page passes a custom property to an image property through `var()`. It also covers `src`, `srcset`, `href` and `poster` in an attribute change on a node the recorder does not know, for example after a restart. `srcset` keeps its descriptors and CSS keeps its `url(...)` syntax. A URL is also replaced where a selector holds it with CSSOM's escapes, by the reference with the same escapes, so a class such as `bg-[url(https://cdn.example.com/a.png)]` and its rule `.bg-\[url\(https\:\/\/cdn\.example\.com\/a\.png\)\]` still match within a block. A reader that replaces references by text must replace the escaped form as well. It is not limited by the 512 URLs per message that get fetch jobs.- A srcset in an image slot keeps one candidate: the one that Chromium shows at a device pixel ratio of 1, which is the ratio the screenshotter renders at. Of `x` descriptors, that is the smallest density at or above 1x, or the largest density when all are below 1x. A `w` descriptor gives a density only against the layout width, which the recorder does not know, so of `w` descriptors the recorder keeps the v3 choice: the largest at or below 1024w, or the smallest when all are wider. The first candidate wins a tie. Only that candidate gets a fetch job, because the renderer has no network access and shows only fetched images. A srcset that mixes `w` and `x` descriptors, or that v3 would read as malformed, keeps no candidate.- The kept candidate stays only when it can show offline: a URL that the URL policy accepts, or an inline image that the image scan keeps. Otherwise the srcset is empty, and no other candidate takes its place, because no other candidate has a fetch job. A candidate that leaves becomes no `declined` marker.@@ -484,20 +488,23 @@ The recorder streams each block into the encoder. Read a block as a stream: a Br- A `<picture>` source whose `type` names a media type the image fetcher does not store, such as `image/avif`, gets no fetch job and loses its remote candidates. Its kept candidate stays only when it is inline, such as an SVG data URI. Srcsets with the same remote URLs in the same order share one decision, so such a source keeps the candidate of a source of a stored type with the same remote URLs, and an `img` that shows its `src` keeps the candidate of an `img` that shows the candidate. That `img` still shows its `src`. The writer rebuilds each srcset that the inspector read in an image slot, in the bytes it stores, after the inline images left and the escapes were written again. A `srcset` key elsewhere, for example in a custom event's payload, stays as it is. A browser skips a source with no candidates, so the replay falls back to the next source or the `img`. The recorder counts these as `ai_research_v4_recorder_srcset_rewrites_total{outcome="trimmed"|"emptied"}`.- The writer rewrites CSS text before it replaces the URLs, so that an offline render loads only images the block refers to. An `image-set()` keeps one candidate, because a renderer picks the candidate that fits its pixel ratio and does not fall back when that candidate fails. It keeps the highest resolution of a type the image fetcher stores among the candidates that can load: inline, accepted by the URL policy, or root-relative on a known page. Only that candidate gets a fetch job, and the other remote candidates get references without one, in case the writer does not find the set's text. A set that holds an inline image keeps all its candidates, because the writer moves the inline image out first.- A root-relative CSS `url(/…)` resolves against the origin of its window's latest page URL, from a Meta event or a custom event that names a page, such as `$url_changed`, with `https` in place of the page's scheme. It gets the reference of the absolute URL, or a `declined` marker. A path-relative `url(…)` stays as the page wrote it, so that no part of a page path, which can hold a token, reaches a fetch. rrweb writes snapshot CSS with absolute URLs, but sends rule text, adopted style sheets and style declarations as the page wrote them. A text that resolves two ways in one block, as in windows on two origins, or that one window resolves and another cannot, keeps no rewrite. An event resolves at most 1,024 relative URLs, a flush keeps at most 64 MiB of resolved URLs, and a page URL longer than 2,048 characters resolves none.++- The recorder reads CSS with `cssparser`, which tokenizes as CSS Syntax 3 does, so names, escapes, comments, strings and bad URLs read as a browser reads them. So `background-\69mage`, `\75 rl(…)` and `background/**/:url("…"/**/)` count, and so does a `url(` that the end of the text closes. A bad URL such as `url(a(1).png)`, which no browser loads, does not count. An image URL that itself holds an escape stays as the page wrote it, as v3 left it.- The writer rewrites only whole `url()` and `image-set()` functions, found as the inspector finds them, inside JSON strings that hold CSS: the values of `_cssText`, `style` (style diffs included), `textContent`, `rule`, `replace`, `replaceSync` and `value`. So a class name such as `bg-[url(/a.png)]`, a quoted string or a comment keeps its text, and the class's rule still matches. A text node that shows the same function text gets the rewrite too. The recorder counts these as `ai_research_v4_recorder_css_rewrites_total{kind="image_set"|"relative_url", outcome="rewritten"|"unfound"}`.- The recorder reads a protocol-relative URL, such as `//cdn.example.com/a.png`, as `https`. It gets the same reference or `declined` marker as the `https` form, and the rewrite replaces the text as the block holds it. A browser reads a backslash as a slash there, so `\\cdn.example.com/a.png` counts too, and so does a URL such as `https:\\cdn.example.com/a.png` with any mix of slashes and backslashes after the scheme, or none: on a page of another scheme the browser loads `https:cdn.example.com/a.png` from that host.- Some URLs stay as they arrived: relative URLs outside CSS, path-relative CSS URLs, root-relative CSS URLs that resolve two ways in a block, go past a budget, or come before this process saw their window's page URL (after a restart or an eviction), the candidates of an `image-set()` that does not parse, remote URLs outside image slots (links, frames, text and CSS properties that do not show an image), URLs inside a compressed value that did not decode or that the decode budget skipped, and URL text written in another form than the recorder read it, for example with a CSS hex escape. The recorder counts the last kind as `outcome="unrewritten"`.- Before the rewrite, the recorder writes again any JSON string that holds an escape capture would not write, for example `\/`, so an escaped URL is found too. A scheme matches in any letter case.-- If a refused URL is still in a block after the rewrite, the recorder writes the block again instead of dropping it. The second pass replaces the refused URL wherever its bytes are, also inside a longer URL. For a refused URL the stored bytes do not hold as the recorder read it, for example behind a CSS escape, it replaces every URL token that holds the URL's host with `imageurl:v4:declined:unchecked`. The recorder counts this as `url_fallbacks_total{reason="declined_kept"|"declined_unfound"}`. It drops the block, as `dropped_blocks_total{outcome="declined_unrewritten"}`, only when the second pass still leaves a refused URL, so no refused URL is stored.+- If a refused URL is still in a block after the rewrite, the recorder writes the block again instead of dropping it. The second pass replaces the refused URL wherever its bytes are, also inside a longer URL. For a refused URL the stored bytes do not hold as the recorder read it, for example behind a CSS escape, it replaces every URL token that holds the URL's host with `imageurl:v4:declined:unchecked`, or with `fonturl:v4:declined:unchecked` in a `url()` of an `@font-face` `src`. The recorder counts this as `url_fallbacks_total{reason="declined_kept"|"declined_unfound"}`. It drops the block, as `dropped_blocks_total{outcome="declined_unrewritten"}`, only when the second pass still leaves a refused URL, so no refused URL is stored.- The recorder reads object keys as the player does: it writes an escaped key, such as `"\u0073rc"`, in its plain form before it inspects the event, and it takes the last copy of a repeated key.-- A block that the recorder cannot check in full is kept. Past 10,000 image-slot URLs (`too_many_urls`), 2 MiB of them (`too_many_url_bytes`) or 200,000 attribute changes in a mutation (`too_many_attribute_items`), the recorder names the URLs it read, and replaces every other remote URL token in the block, links and text included, with `imageurl:v4:declined:unchecked`. A replaced token that starts after `(`, `'` or `"` runs to the `)`, `'` or `"` that ends it, a token that starts its JSON string runs to the end of the string, because a space does not end an attribute's URL, and any other token runs to whitespace, so the scrub never splits an escape. It counts these as `url_fallbacks_total{reason}` and the replaced tokens as `scrubbed_urls_total`. A URL of any length is named; one longer than the fetcher takes becomes `imageurl:v4:declined:too_long`. Only a block whose sealing fails is dropped.+- A block that the recorder cannot check in full is kept. Past 10,000 image-slot URLs (`too_many_urls`), 2 MiB of them (`too_many_url_bytes`) or 200,000 attribute changes in a mutation (`too_many_attribute_items`), the recorder names the URLs it read, and replaces every other remote URL token in the block, links and text included, with `imageurl:v4:declined:unchecked`. A `url()` in the `src` of an `@font-face` rule becomes `fonturl:v4:declined:unchecked` instead, so the renderer reads it as a font it does not have. A replaced token that starts after `(`, `'` or `"` runs to the `)`, `'` or `"` that ends it, a token that starts its JSON string runs to the end of the string, because a space does not end an attribute's URL, and any other token runs to whitespace, so the scrub never splits an escape. It counts these as `url_fallbacks_total{reason}` and the replaced tokens as `scrubbed_urls_total`. A URL of any length is named; one longer than the fetcher takes becomes `imageurl:v4:declined:too_long`. Only a block whose sealing fails is dropped.+- CSS nesting is the exception. A block whose CSS nests rules or functions 75 deep is dropped, as `dropped_blocks_total{outcome="too_many_nested_blocks"}`. The recorder cannot read CSS past that depth, and the scrub replaces only the URL tokens that it finds, so a scrub could keep a URL that nobody checked.- When the text of a rewritten URL also appears outside an image slot in the same block, it is replaced there too.- `hash` is the first 22 characters of the base64url HMAC-SHA256, as in v3.### LayoutEvery prefix that the recorder writes is partitioned by the UTC day of the write.-Fetched images are partitioned by the key month of their reference.+Fetched images and fonts are partitioned by the key month of their reference. A public font file has no month.| Bucket | Prefix | Contents || ----------- | --------------------------------------------- | ------------------------------------------------------------------------------- |@@ -508,6 +515,8 @@ Fetched images are partitioned by the key month of their reference.| `rrweb-v4` | `index/images/_watermarks/` | JSON, one object per Kafka partition: how far the image index is complete || `images-v4` | `images/packs/dt={day}/{team_id}/` | One object per recorder flush and team: sealed images, each found by byte range || `images-v4` | `images/url/month={YYYY-MM}/{team_id}/{hash}` | One sealed fetched image per URL reference |+| `images-v4` | `fonts/url/month={YYYY-MM}/{team_id}/{hash}` | One sealed fetched font per font URL reference |+| `images-v4` | `fonts/url/public/{hash}` | One unsealed font file per public font reference, for every team |S3 expires every object a fixed time after it was written: 30 days in `rrweb-v4` and 38 days in `images-v4`. [Lifecycle](#lifecycle) lists every lifetime and why they fit together.@@ -683,13 +692,13 @@ The fetcher keeps a crawl history per URL reference, so per team and month, in i- The history holds no URL in plaintext. It binds the stored validators to an HMAC of the response URL under the team month URL key, and sends them only when the job's URL has the same HMAC.A URL is therefore fetched from its origin about once per team and month, as v3 fetches it about once in 30 days.-Unlike v3, the history is per team: a sealed image is readable only with its team's key, so two teams that refer to one URL each fetch it.+Unlike v3, the history is per team: a sealed image is readable only with its team's key, so two teams that refer to one URL each fetch it. A public font file is the exception, as [Web fonts](#web-fonts) says.The recorder sends a job for each referenced URL at least once per day, unless the fetcher's history shows that the fetcher would drop it (see below). So the fetcher sees a reference in every 7-day refresh interval.-The fetcher drops a job whose team month row is missing, expired or deleted.+The fetcher drops a job whose team month row is missing, expired or deleted. A public font job needs no row.Each recorder pod sends a reference at most once per UTC day. When `AI_RESEARCH_V4_IMAGE_FETCH_HISTORY_TABLE` is set, the recorder also reads the fetcher's history, as the v3 mirror did:-- It reads only the references of the flush that it did not send or mark that day, each one once. The read starts with the flush and runs while the flush seals its blocks.+- It reads only the image and font references of the flush that it did not send or mark that day, each one once. The read starts with the flush and runs while the flush seals its blocks.- It does not send a job that the fetcher would still drop at the end of the day. That is a job before the refetch time of its reference, when the reference has no stored copy or a stored copy less than 7 days old. It marks that reference as sent for the day.- It sends every other job, so the fetcher still refreshes each stored copy within its interval.- It parses a history row with the same code as the fetcher. A row that the fetcher reads as a miss is a miss for the recorder too.@@ -697,3 +706,40 @@ Each recorder pod sends a reference at most once per UTC day. When `AI_RESEARCH_- `ai_research_v4_recorder_image_fetch_history_total` counts the references it read by `outcome`: `fresh` (marked), `due` (a refresh or refetch is due today), `absent`, `invalid` and `error`.`replay_core_v4::fetch_history` holds the drop rule, the 7-day refresh interval and the row parser. The recorder calls it directly, and the fetcher calls it through `@posthog/ai-research-v4-native`.++### Web fonts++A renderer that has no network needs the page's fonts. rrweb keeps the `@font-face` rules of a page in its style text, but it does not record font files, so the v4 lane fetches them as it fetches images.++The recorder reads the `src` URLs of `@font-face` rules in `<style>` text, `_cssText`, style sheet rules and adopted style sheets. Each URL becomes one of these:++| The rule's family | In the block | Fetch job |+| ------------------------------------------------------- | --------------------------------------- | --------- |+| A family in `replay-core-v4/src/bundled_fonts.txt` | `fonturl:v4:bundled:{slug}` | None |+| Any other family, for a public font file | `fonturl:v4:public:{hash}` | Yes |+| Any other family | `fonturl:v4:{team_id}:{YYYY-MM}:{hash}` | Yes |+| Any other family, for a URL that the URL policy refuses | `fonturl:v4:declined:{reason}` | None |+| Any other family, for a URL the recorder did not check | `fonturl:v4:declined:unchecked` | None |++- The renderer ships every family in `bundled_fonts.txt`. A reader replaces `fonturl:v4:bundled:{slug}` with its own copy of that family, and picks the face from the rule's `font-weight` and `font-style` descriptors. The slug is the family name in lowercase, with `-` for each space.+- The list holds only families whose licence lets the renderer carry them. The renderer ships the full font of each family, which holds every glyph of a subset that a page asks for, such as a Google Fonts `text=` or `icon_names=` subset.+- The list holds Google's icon families, Material Icons and Material Symbols, because Google never moves an icon to another code point or ligature. It holds no site's own icon font. Many sites share a generated name such as `icomoon`, and an icon font generator can number the glyphs again on each build, so the fetcher fetches those.+- A page can ask Google for one fixed fill or grade of Material Symbols, such as `FILL@1`. The bundled font has every fill and grade, and shows the default unless the page's CSS sets `font-variation-settings`. So such a page shows outlined icons where it showed filled ones. The `font-weight` and `font-size` of the page still set the weight and optical size.+- The recorder compares family names without case, and reads a run of spaces as one space. A family it does not know, for example a name that a build tool made, is fetched.+- A team font reference uses the same hash, image URL key and URL policy as an image URL reference. A public font reference uses the same URL policy. A font job goes to the `ai_research_v4_image_fetch` topic in the same shape, keyed by registrable domain, so fonts share the image jobs' partitions, politeness limits and crawl history rules. A record holds only font jobs or only image jobs.+- An Embedded OpenType or SVG font file gets a reference but no fetch job, because Chromium loads neither format.+- A root-relative font URL, such as `src:url(/fonts/a.woff2)`, resolves as a root-relative image URL does, and gets the font reference of the absolute URL. A path-relative font URL stays as the page wrote it.+- The recorder reads CSS escapes as a browser does, in the `@font-face` keyword, the `src` and `font-family` descriptors, the `url()` name and the URL itself. So `@\66 ont-face`, `\73 rc` and `url(https://cdn.example.com/a\2e woff2)` count, and an escaped protocol-relative URL reads as `https`. A quoted URL drops a backslash before a newline, as a line continuation. Comments around a descriptor name, as in `src/**/:`, do not hide it. A font URL that holds an escape gets a rewrite of its whole `url()` to its font reference, as a root-relative font URL does.+- When one URL is both an image and a font, the block holds its image reference.+- Font URLs have their own budget per block: 4,096 URLs and 256 KiB. Font URLs never count toward the image URL limits above. Past the font budget, the recorder still names every other URL it reads. It replaces each remote URL in an `@font-face` `src` that it did not name with `fonturl:v4:declined:unchecked`, and leaves links and text as they are. It does the same when a block holds more font URLs with an escape than its CSS rewrites can hold, 10,000 of them or 2 MiB. It counts these as `url_fallbacks_total{reason="too_many_font_urls"|"too_many_font_url_bytes"|"too_many_font_rewrites"}`.++The fetcher reads the asset from the reference prefix. For a font it sends `Accept: font/woff2, font/woff, font/ttf, font/otf, */*;q=0.1`. It takes a response whose media type is a `font/*` type, a legacy font type such as `application/x-font-woff`, or `application/octet-stream`, because many servers label fonts with a generic type. The decoded bytes must start with a WOFF2, WOFF, OpenType, TrueType or font collection signature. It stores the font as fetched under `fonts/url/`, sealed under the team month key with `kind: "font-url"`.++Google serves each file under `https://fonts.gstatic.com/s/` at a versioned path whose bytes never change, and it serves the same file to every page. Such a public font file belongs to no team:++- Its reference is `fonturl:v4:public:{hash}`, where `hash` is the first 22 characters of the base64url SHA-256 of the canonical URL. The hash has no key, because the URL holds no personal data. A person who holds a reference can test it against a Google Fonts URL, which tells them only which public font file a page used.+- The fetcher stores the file once, unsealed, at `fonts/url/public/{hash}`, and reads no team month key for it. All teams share one crawl history entry, so Google gets one request per file, and a later job from any team only refreshes the stored file.+- The fetcher stores the file only when the response came from the URL that the reference names, so a redirect target never becomes the shared file. It counts a refused one as `ai_research_v4_image_fetch_stored_images_total{asset="font",outcome="not_public"}`.+- A URL with a query, a port or a path outside `/s/` gets a team reference. Google serves a `text=` subset under `/l/`, and such a file holds the glyphs of the text that the page asked for.++An older fetcher sends a whole record to its dead letter topic when one job in it has a `fonturl` reference. Records never mix the two kinds of job, so while a recorder runs ahead of the fetcher, only font jobs are lost. The fetcher's role must also be allowed to write `fonts/url/` (`v4-staging.hcl` in posthog-cloud-infra). Until then, each font write fails and the fetcher tries the job again after a minute.diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex fa5af945a268..bbf2a05aae71 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -3550,6 +3550,28 @@ dependencies = ["hybrid-array",]+[[package]]+name = "cssparser"+version = "0.38.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "11119743ad110e8c1bdccd930d7f5c30c99e5fc76a7b63ec9807e84eef0c5f59"+dependencies = [+ "cssparser-macros",+ "dtoa-short",+ "itoa",+ "smallvec",+]++[[package]]+name = "cssparser-macros"+version = "0.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a"+dependencies = [+ "quote",+ "syn 3.0.3",+]+[[package]]name = "csv"version = "1.4.0"@@ -4317,6 +4339,21 @@ version = "0.11.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"+[[package]]+name = "dtoa"+version = "1.0.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"++[[package]]+name = "dtoa-short"+version = "0.3.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"+dependencies = [+ "dtoa",+]+[[package]]name = "dtor"version = "0.8.1"@@ -9795,7 +9832,7 @@ dependencies = [[[package]]name = "posthog-replay-anonymizer"-version = "0.13.0"+version = "0.14.0"dependencies = ["ahash 0.8.12","anyhow",@@ -11056,6 +11093,7 @@ dependencies = ["aws-sdk-kms","base64 0.22.1","chrono",+ "cssparser","flate2","futures","governor",diff --git a/rust/ai-research/replay-core-v4-node/src/index.ts b/rust/ai-research/replay-core-v4-node/src/index.tsindex 63640badd1f4..4e4a412bff44 100644--- a/rust/ai-research/replay-core-v4-node/src/index.ts+++ b/rust/ai-research/replay-core-v4-node/src/index.ts@@ -85,8 +85,14 @@ export function isUrlHistoryValue(key: string, value: string): boolean {return native.isUrlHistoryValue(key, value)}-/** Parses `imageurl:v4:{team}:{YYYY-MM}:{hash}`. `parseImageRef` refuses it, so no v2 or v3 reader takes one. */-export function parseV4UrlRef(reference: string): { teamId: number; keyMonth: string; hash: string } | null {+export type V4UrlAsset = 'image' | 'font'++export type V4UrlRef =+ | { scope: 'team'; asset: V4UrlAsset; teamId: number; keyMonth: string; hash: string }+ | { scope: 'public'; asset: 'font'; hash: string }++/** Parses `imageurl:v4:{team}:{YYYY-MM}:{hash}`, its `fonturl` twin and `fonturl:v4:public:{hash}`. `parseImageRef` refuses them all, so no v2 or v3 reader takes one. */+export function parseV4UrlRef(reference: string): V4UrlRef | null {return native.parseV4UrlRef(reference)}@@ -97,3 +103,12 @@ export function sealFetchedImage(teamMonthKey: Buffer, reference: string, image:export function imageUrlHash(teamMonthKey: Buffer, url: string): string {return native.imageUrlHash(teamMonthKey, url)}++/** The public font reference that a canonical fetch URL names, or `null` when its file belongs to a team. */+export function publicFontRefOf(fetchUrl: string): string | null {+ return native.publicFontRefOf(fetchUrl)+}++export function publicUrlHash(url: string): string {+ return native.publicUrlHash(url)+}diff --git a/rust/ai-research/replay-core-v4-node/src/lib.rs b/rust/ai-research/replay-core-v4-node/src/lib.rsindex 991fd24b67e5..c93f6629bb40 100644--- a/rust/ai-research/replay-core-v4-node/src/lib.rs+++ b/rust/ai-research/replay-core-v4-node/src/lib.rs@@ -6,7 +6,10 @@use neon::prelude::*;use neon::types::buffer::TypedArray;use replay_core_v4::fetch_history::UrlHistory;-use replay_core_v4::fetched_image::{image_url_hash, image_url_ref, seal_fetched_image};+use replay_core_v4::fetched_image::{+ fetched_url_ref, image_url_hash, public_font_ref_of, seal_fetched_image, FetchedRef,+};+use replay_core_v4::refs::{public_url_hash, UrlAsset};use replay_core_v4::url_policy::{is_public_host, politeness_key, try_canonicalize};/// The registrable domain of a host.@@ -73,18 +76,33 @@ fn is_url_history_value_ffi(mut cx: FunctionContext) -> JsResult<JsBoolean> {fn parse_v4_url_ref_ffi(mut cx: FunctionContext) -> JsResult<JsValue> {const MAX_SAFE_INTEGER: i64 = (1 << 53) - 1;let reference = cx.argument::<JsString>(0)?.value(&mut cx);- let Some(parsed) =- image_url_ref(&reference).filter(|parsed| parsed.team_id <= MAX_SAFE_INTEGER)- else {- return Ok(cx.null().upcast());- };let result = cx.empty_object();- let team_id = cx.number(parsed.team_id as f64);- result.set(&mut cx, "teamId", team_id)?;- let key_month = cx.string(parsed.key_month);- result.set(&mut cx, "keyMonth", key_month)?;- let hash = cx.string(parsed.hash);- result.set(&mut cx, "hash", hash)?;+ match fetched_url_ref(&reference) {+ Some(FetchedRef::Team { asset, parsed }) if parsed.team_id <= MAX_SAFE_INTEGER => {+ let scope = cx.string("team");+ result.set(&mut cx, "scope", scope)?;+ let asset = cx.string(match asset {+ UrlAsset::Image => "image",+ UrlAsset::Font => "font",+ });+ result.set(&mut cx, "asset", asset)?;+ let team_id = cx.number(parsed.team_id as f64);+ result.set(&mut cx, "teamId", team_id)?;+ let key_month = cx.string(parsed.key_month);+ result.set(&mut cx, "keyMonth", key_month)?;+ let hash = cx.string(parsed.hash);+ result.set(&mut cx, "hash", hash)?;+ }+ Some(FetchedRef::PublicFont { hash }) => {+ let scope = cx.string("public");+ result.set(&mut cx, "scope", scope)?;+ let asset = cx.string("font");+ result.set(&mut cx, "asset", asset)?;+ let hash = cx.string(hash);+ result.set(&mut cx, "hash", hash)?;+ }+ _ => return Ok(cx.null().upcast()),+ }Ok(result.upcast())}@@ -116,6 +134,19 @@ fn image_url_hash_ffi(mut cx: FunctionContext) -> JsResult<JsString> {Ok(cx.string(image_url_hash(&key, &url)))}+fn public_font_ref_of_ffi(mut cx: FunctionContext) -> JsResult<JsValue> {+ let url = cx.argument::<JsString>(0)?.value(&mut cx);+ Ok(match public_font_ref_of(&url) {+ Some(reference) => cx.string(reference).upcast(),+ None => cx.null().upcast(),+ })+}++fn public_url_hash_ffi(mut cx: FunctionContext) -> JsResult<JsString> {+ let url = cx.argument::<JsString>(0)?.value(&mut cx);+ Ok(cx.string(public_url_hash(&url)))+}+#[neon::main]fn main(mut cx: ModuleContext) -> NeonResult<()> {cx.export_function("politenessKey", politeness_key_ffi)?;@@ -126,5 +157,7 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> {cx.export_function("parseV4UrlRef", parse_v4_url_ref_ffi)?;cx.export_function("sealFetchedImage", seal_fetched_image_ffi)?;cx.export_function("imageUrlHash", image_url_hash_ffi)?;+ cx.export_function("publicFontRefOf", public_font_ref_of_ffi)?;+ cx.export_function("publicUrlHash", public_url_hash_ffi)?;Ok(())}diff --git a/rust/ai-research/replay-core-v4/Cargo.toml b/rust/ai-research/replay-core-v4/Cargo.tomlindex 0c219e86651e..9c1772771c2d 100644--- a/rust/ai-research/replay-core-v4/Cargo.toml+++ b/rust/ai-research/replay-core-v4/Cargo.toml@@ -21,6 +21,7 @@ aws-sdk-dynamodb = { workspace = true, optional = true }aws-sdk-kms = { workspace = true, optional = true }base64 = { workspace = true }chrono = { workspace = true, optional = true }+cssparser = { version = "0.38", default-features = false, features = ["fast_match_byte"] }futures = { workspace = true, optional = true }governor = { workspace = true, optional = true }hkdf = "0.12"diff --git a/rust/ai-research/replay-core-v4/src/bundled_fonts.txt b/rust/ai-research/replay-core-v4/src/bundled_fonts.txtnew file mode 100644index 000000000000..e6303e754416--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/bundled_fonts.txt@@ -0,0 +1,125 @@+# Font families that the v4 renderer ships, so that a block names them and the fetcher does not fetch them.+#+# One family name per line, as the font-family descriptor of an @font-face rule names it. Only ASCII letters, digits+# and single spaces. The recorder compares names without case and reads a run of whitespace as one space. A blank line+# and a line that starts with "#" are ignored.+#+# Each src URL of an @font-face rule whose family is on this list becomes fonturl:v4:bundled:{slug}, where the slug is+# the name in lowercase with "-" for each space. The renderer must hold each family here in every weight and style,+# and pick the face from the rule's font-weight and font-style descriptors.+#+# List only families whose licence lets the renderer image carry them: SIL OFL, Apache 2.0 or the Ubuntu Font Licence.+# An icon font is listed only when one publisher owns its name and never moves an icon to another code point or+# ligature, as Google does for Material Icons and Material Symbols. The renderer ships the full font, which holds every+# glyph of any subset a page asks for, such as an icon_names= subset. A site's own icon font is never listed: many+# sites share a generated name such as "icomoon", and icon font generators can number the glyphs again on each build.+#+# replay-core-v4 embeds this file, so a change ships with a recorder image build. A malformed or repeated line fails+# the crate's tests.++Albert Sans+Archivo+Arimo+Assistant+Barlow+Barlow Condensed+Be Vietnam Pro+Bebas Neue+Bitter+Cabin+Cairo+Catamaran+Comfortaa+Cormorant Garamond+Crimson Pro+Crimson Text+DM Sans+DM Serif Display+Dosis+EB Garamond+Exo 2+Figtree+Fira Code+Fira Sans+Geist+Geist Mono+Heebo+Hind+IBM Plex Mono+IBM Plex Sans+IBM Plex Serif+Inconsolata+Instrument Sans+Inter+Inter Tight+JetBrains Mono+Josefin Sans+Jost+Kanit+Karla+Lato+Lexend+Libre Baskerville+Libre Franklin+Lora+Manrope+Material Icons+Material Icons Outlined+Material Icons Round+Material Icons Sharp+Material Icons Two Tone+Material Symbols Outlined+Material Symbols Rounded+Material Symbols Sharp+Maven Pro+Merriweather+Montserrat+Mukta+Mulish+Noto Sans+Noto Sans JP+Noto Sans KR+Noto Sans SC+Noto Sans TC+Noto Serif+Nunito+Nunito Sans+Onest+Open Sans+Oswald+Outfit+Overpass+Oxygen+PT Sans+PT Serif+Playfair Display+Plus Jakarta Sans+Poppins+Prompt+Public Sans+Quicksand+Raleway+Red Hat Display+Red Hat Text+Roboto+Roboto Condensed+Roboto Flex+Roboto Mono+Roboto Slab+Rubik+Sarabun+Signika+Sora+Source Code Pro+Source Sans 3+Source Sans Pro+Source Serif 4+Space Grotesk+Space Mono+Tajawal+Titillium Web+Ubuntu+Urbanist+Varela Round+Work Sans+Zilla Slabdiff --git a/rust/ai-research/replay-core-v4/src/fetched_image.rs b/rust/ai-research/replay-core-v4/src/fetched_image.rsindex da337266b070..b9d1d2aaafe4 100644--- a/rust/ai-research/replay-core-v4/src/fetched_image.rs+++ b/rust/ai-research/replay-core-v4/src/fetched_image.rs@@ -1,33 +1,56 @@-//! The frame the image fetcher stores for each fetched image. The fetcher seals it through `replay-core-v4-node`, and a-//! reader rebuilds its context from the image's URL reference.+//! The frame the image fetcher stores for each fetched image or font file. The fetcher seals it through+//! `replay-core-v4-node`, and a reader rebuilds its context from the file's URL reference. A public font file belongs+//! to no team, so the fetcher stores it unsealed.use posthog_replay_anonymizer::v4_image_refs::{parse, Parsed, Ref, RefKind};use crate::envelope::{seal_frame, Codec, Context, EnvelopeError, Version};-use crate::refs::reference_hash;+use crate::refs::{public_font_url_ref, reference_hash, UrlAsset};use crate::seal::image_url_key;+use crate::url_policy::canonicalize;#[derive(Debug, thiserror::Error, PartialEq, Eq)]pub enum SealError {- #[error("not an image URL reference")]- NotAnImageUrlReference,+ #[error("not a team URL reference")]+ NotATeamUrlReference,#[error(transparent)]Envelope(#[from] EnvelopeError),}-pub fn image_url_ref(reference: &str) -> Option<Ref<'_>> {+/// A reference to a file that the image fetcher fetches and stores.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum FetchedRef<'a> {+ Team { asset: UrlAsset, parsed: Ref<'a> },+ PublicFont { hash: &'a str },+}++pub fn fetched_url_ref(reference: &str) -> Option<FetchedRef<'_>> {match parse(reference)? {- Parsed::Ref(parsed) if parsed.kind == RefKind::ImageUrl => Some(parsed),+ Parsed::Ref(parsed) if parsed.kind == RefKind::ImageUrl => Some(FetchedRef::Team {+ asset: UrlAsset::Image,+ parsed,+ }),+ Parsed::Ref(parsed) if parsed.kind == RefKind::FontUrl => Some(FetchedRef::Team {+ asset: UrlAsset::Font,+ parsed,+ }),+ Parsed::PublicFont(hash) => Some(FetchedRef::PublicFont { hash }),_ => None,}}pub fn fetched_image_context(reference: &str) -> Option<Context> {- let parsed = image_url_ref(reference)?;+ let Some(FetchedRef::Team { asset, parsed }) = fetched_url_ref(reference) else {+ return None;+ };+ let kind = match asset {+ UrlAsset::Image => "image-url",+ UrlAsset::Font => "font-url",+ };Some(Context {team_id: parsed.team_id,key_month: Some(parsed.key_month.to_owned()),- kind: "image-url".to_owned(),+ kind: kind.to_owned(),codec: Some(Codec::None),reference: Some(reference.to_owned()),..Context::default()@@ -39,7 +62,7 @@ pub fn seal_fetched_image(reference: &str,image: &[u8],) -> Result<Vec<u8>, SealError> {- let context = fetched_image_context(reference).ok_or(SealError::NotAnImageUrlReference)?;+ let context = fetched_image_context(reference).ok_or(SealError::NotATeamUrlReference)?;Ok(seal_frame(team_month_key, Version::V4, &context, image)?)}@@ -47,31 +70,43 @@ pub fn image_url_hash(team_month_key: &[u8; 32], url: &str) -> String {reference_hash(&image_url_key(team_month_key), url.as_bytes())}+/// The public font reference that a fetch URL names, which the fetcher compares with a job's reference so that a+/// redirect target never becomes the shared file.+pub fn public_font_ref_of(fetch_url: &str) -> Option<String> {+ public_font_url_ref(&canonicalize(fetch_url)?)+}+#[cfg(test)]mod tests {use super::*;use crate::envelope::open_frame;const REFERENCE: &str = "imageurl:v4:7:2026-10:TEX0ajC0UkaOC9js5c7oqr";+ const FONT_REFERENCE: &str = "fonturl:v4:7:2026-10:Bwu1146AUxxsfd2I6Co_JI";#[test]- fn seals_an_image_only_under_an_image_url_reference() {+ fn seals_a_file_only_under_a_team_url_reference() {let key = [3; 32];- let frame = seal_fetched_image(&key, REFERENCE, b"png").unwrap();- let context = fetched_image_context(REFERENCE).unwrap();- assert_eq!(- open_frame(&key, Version::V4, &context, &frame).unwrap(),- b"png"- );+ for (reference, kind) in [(REFERENCE, "image-url"), (FONT_REFERENCE, "font-url")] {+ let frame = seal_fetched_image(&key, reference, b"bytes").unwrap();+ let context = fetched_image_context(reference).unwrap();+ assert_eq!(context.kind, kind);+ assert_eq!(+ open_frame(&key, Version::V4, &context, &frame).unwrap(),+ b"bytes"+ );+ }for reference in ["image:v4:7:2026-10:TEX0ajC0UkaOC9js5c7oqr","imageurl:v4:07:2026-10:TEX0ajC0UkaOC9js5c7oqr","imageurl:v4:7:2026-13:TEX0ajC0UkaOC9js5c7oqr","imageurl:v4:declined:credential",+ "fonturl:v4:public:I6bjVBXgjPcaYLPITibNM4",+ "fonturl:v4:bundled:roboto",] {assert_eq!(- seal_fetched_image(&key, reference, b"png"),- Err(SealError::NotAnImageUrlReference),+ seal_fetched_image(&key, reference, b"bytes"),+ Err(SealError::NotATeamUrlReference),"{reference}");}diff --git a/rust/ai-research/replay-core-v4/src/fonts.rs b/rust/ai-research/replay-core-v4/src/fonts.rsnew file mode 100644index 000000000000..286fdd0c1412--- /dev/null+++ b/rust/ai-research/replay-core-v4/src/fonts.rs@@ -0,0 +1,136 @@+//! The `@font-face` families that the renderer ships, and the font files that the fetcher takes.++use std::collections::HashMap;+use std::sync::LazyLock;++const BUNDLED_FONTS: &str = include_str!("bundled_fonts.txt");++/// A longer descriptor is not read, because a page controls its length and the recorder reads it once per `src` URL.+const MAX_FAMILY_DESCRIPTOR_LEN: usize = 256;++static BUNDLED_FAMILIES: LazyLock<HashMap<String, String>> = LazyLock::new(|| {+ parse_bundled_fonts(BUNDLED_FONTS)+ .unwrap_or_else(|line| panic!("bundled_fonts.txt has a malformed entry: {line:?}"))+});++fn parse_bundled_fonts(text: &str) -> Result<HashMap<String, String>, String> {+ let mut families = HashMap::new();+ for line in text.lines().map(str::trim) {+ if line.is_empty() || line.starts_with('#') {+ continue;+ }+ let well_formed = line.len() < MAX_FAMILY_DESCRIPTOR_LEN+ && line.bytes().all(|b| b.is_ascii_alphanumeric() || b == b' ')+ && !line.contains(" ");+ let name = comparable_name(line);+ if !well_formed || families.contains_key(&name) {+ return Err(line.to_owned());+ }+ let slug = name.replace(' ', "-");+ families.insert(name, slug);+ }+ Ok(families)+}++fn comparable_name(name: &str) -> String {+ name.split_ascii_whitespace()+ .collect::<Vec<_>>()+ .join(" ")+ .to_ascii_lowercase()+}++/// `None` for an escape or a list of families, which an `@font-face` descriptor does not allow.+fn family_name(descriptor: &str) -> Option<&str> {+ if descriptor.len() > MAX_FAMILY_DESCRIPTOR_LEN {+ return None;+ }+ let value = descriptor.trim();+ if value.contains('\\') {+ return None;+ }+ match value.chars().next()? {+ quote @ ('"' | '\'') => value[1..]+ .strip_suffix(quote)+ .filter(|inner| !inner.contains(quote)),+ _ if value.contains([',', '"', '\'']) => None,+ _ => Some(value),+ }+}++pub fn bundled_font(family_descriptor: &str) -> Option<&'static str> {+ let name = comparable_name(family_name(family_descriptor)?);+ BUNDLED_FAMILIES.get(&name).map(String::as_str)+}++/// Chromium loads neither Embedded OpenType nor SVG fonts, so a fetch of one would only spend the origin's politeness budget.+pub fn is_fetched_font_url(url: &str) -> bool {+ let path = url.split(['?', '#']).next().unwrap_or_default();+ let file = path.rsplit(['/', '\\']).next().unwrap_or_default();+ let extension = file.rsplit_once('.').map_or("", |(_, extension)| extension);+ !["eot", "svg", "svgz"]+ .iter()+ .any(|refused| extension.eq_ignore_ascii_case(refused))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn the_shipped_list_parses() {+ assert!(BUNDLED_FAMILIES.len() > 50);+ }++ #[test]+ fn a_malformed_or_repeated_entry_fails() {+ for list in ["Inter\ninter", "Open Sans", "Inter;", "\"Inter\""] {+ assert!(parse_bundled_fonts(list).is_err(), "{list:?}");+ }+ }++ #[test]+ fn names_a_bundled_family_however_the_rule_writes_it() {+ for descriptor in [+ "Open Sans",+ " 'Open Sans' ",+ "\"open sans\"",+ "OPEN\tSANS",+ ] {+ assert_eq!(+ bundled_font(descriptor),+ Some("open-sans"),+ "{descriptor:?}"+ );+ }+ let padded = format!("{}Inter", " ".repeat(MAX_FAMILY_DESCRIPTOR_LEN));+ for descriptor in [+ "Open Sans, sans-serif",+ "\"Open Sans",+ "Open\\ Sans",+ "Brand Sans",+ "Font Awesome 6 Free",+ "",+ &padded,+ ] {+ assert_eq!(bundled_font(descriptor), None, "{descriptor:?}");+ }+ }++ #[test]+ fn skips_formats_a_renderer_never_loads() {+ for url in [+ "https://cdn.example.com/fa.eot?#iefix",+ "https://cdn.example.com/fa.svg#fontawesome",+ "https://cdn.example.com/FA.EOT",+ ] {+ assert!(!is_fetched_font_url(url), "{url}");+ }+ for url in [+ "https://cdn.example.com/fa.woff2?v=4.7.0",+ "https://cdn.example.com/font",+ "https://cdn.example.com/eot/font.woff",+ ] {+ assert!(is_fetched_font_url(url), "{url}");+ }+ }+}diff --git a/rust/ai-research/replay-core-v4/src/inspect/css.rs b/rust/ai-research/replay-core-v4/src/inspect/css.rsindex 7ebd48d81532..c52eea5f30b4 100644--- a/rust/ai-research/replay-core-v4/src/inspect/css.rs+++ b/rust/ai-research/replay-core-v4/src/inspect/css.rs@@ -1,9 +1,18 @@//! Image URLs in CSS: the `url()`s of an image property, or of a custom property that reaches one through `var()`.+//! Font URLs are the `url()`s in the `src` descriptor of an `@font-face` rule. `cssparser` tokenizes as CSS Syntax 3 does, so a name, an escape, a comment or a bad URL reads here as the renderer reads it.+use std::borrow::Cow;use std::sync::atomic::{AtomicUsize, Ordering};+use cssparser::{+ match_ignore_ascii_case, AtRuleParser, CowRcStr, DeclarationParser, Delimiter, ParseError,+ Parser, ParserState, QualifiedRuleParser, RuleBodyItemParser, RuleBodyParser, SourcePosition,+ Token,+};use url::Url;+use super::dom::{RemoteKind, RemoteUrl};+use crate::fonts::bundled_font;use crate::url_policy::{is_fetched_image_type, is_protocol_relative, is_remote, try_canonicalize, MAX_URL_LEN,};@@ -12,16 +21,27 @@ use crate::url_policy::{pub const MAX_RESOLVED_URLS: usize = 1024;/// Far more than a real `image-set()` lists. A longer set gets no rewrite, so the inspector never holds its candidates.const MAX_IMAGE_SET_CANDIDATES: usize = 32;+/// The depth at which `cssparser` enters no more blocks, to keep the stack bounded. Its default, set here so that the scan knows where it stops.+const NESTED_BLOCK_LIMIT: u8 = 75;#[derive(Debug, Clone, PartialEq, Eq)]pub enum CssUrl<'a> {Image(&'a str),Unshown(&'a str),- /// A resolved root-relative URL. The block holds only the relative text, which its rewrite replaces.+ /// A URL that the block holds only as other text, root-relative or written with CSS escapes, which its rewrite replaces.Resolved(&'a str),/// A root-relative `url()` that did not resolve. Its text keeps no rewrite, which another window would apply here.Unresolved(&'a str),Rewrite(CssRewrite),+ /// A `src` URL of an `@font-face` rule, with the raw value of that rule's `font-family` descriptor.+ Font {+ url: &'a str,+ family: Option<&'a str>,+ },+ /// A [`CssUrl::Resolved`] font URL of a family that the renderer does not ship.+ ResolvedFont(&'a str),+ /// Rules or functions nest to [`NESTED_BLOCK_LIMIT`], where `cssparser` enters no block, so a URL there goes unnamed.+ NestedTooDeep,}#[derive(Debug, Clone, PartialEq, Eq)]@@ -29,8 +49,8 @@ pub struct CssRewrite {pub text: String,pub replacement: String,pub dropped: Vec<String>,- /// The URL that `replacement` holds in place of a relative one, for the writer to replace with its reference.- pub resolved: Option<String>,+ /// The URL that `replacement` holds in place of a relative one, for the writer to replace with the reference of its kind.+ pub resolved: Option<RemoteUrl>,}/// The origin that a root-relative URL on `page_url` resolves against, as `https` because the fetcher loads nothing else.@@ -97,59 +117,104 @@ pub fn rewrite_functions(css: &str,replacement: &mut impl FnMut(&str) -> Option<String>,) -> Option<String> {- let bytes = css.as_bytes();let mut out: Option<String> = None;let mut copied_to = 0;- let mut position = 0;- while position < bytes.len() {- if starts_with(bytes, position, b"/*") {- position = skip_comment(bytes, position).unwrap_or(bytes.len());- continue;+ each_function(&mut Parser::new(css), 0, &mut |input, start, opened| {+ if !matches!(opened, Opened::UnquotedUrl(_)) {+ skip_block(input);}- if bytes[position] == b'\\' {- position = skip_escape(bytes, position);- continue;+ let start = start.byte_index();+ let end = input.position().byte_index();+ if let Some(replaced) = replacement(&css[start..end]) {+ let out = out.get_or_insert_with(|| String::with_capacity(css.len()));+ out.push_str(&css[copied_to..start]);+ out.push_str(&replaced);+ copied_to = end;}- if matches!(bytes[position], b'\'' | b'"') {- position = skip_quoted(bytes, position).unwrap_or(bytes.len());+ });+ let mut out = out?;+ out.push_str(&css[copied_to..]);+ Some(out)+}++pub fn rewrite_font_src_functions(+ css: &str,+ replacement: &mut impl FnMut(&str) -> Option<String>,+) -> Option<String> {+ let mut out: Option<String> = None;+ let mut copied_to = 0;+ for declaration in declarations(css).found {+ if !matches!(declaration.kind, DeclarationKind::FontSrc(_)) {continue;}- let image_set = if function_at(bytes, position, b"image-set") {- Some(b"image-set".len())- } else if function_at(bytes, position, b"-webkit-image-set") {- Some(b"-webkit-image-set".len())- } else {- None- };- let end = if let Some(name_len) = image_set {- // The inspector stops at an unterminated set, and each later opener would scan to the end again.- let Some(end) = matching_paren(bytes, position + name_len) else {- break;- };- end- } else if function_at(bytes, position, b"url") {- scan_url_function(css, position).0- } else {- position += 1;- continue;- };- if let Some(replaced) = replacement(&css[position..end]) {+ let value = &css[declaration.start..declaration.end];+ if let Some(rewritten) = rewrite_functions(value, replacement) {let out = out.get_or_insert_with(|| String::with_capacity(css.len()));- out.push_str(&css[copied_to..position]);- out.push_str(&replaced);- copied_to = end;+ out.push_str(&css[copied_to..declaration.start]);+ out.push_str(&rewritten);+ copied_to = declaration.end;}- position = end;}let mut out = out?;out.push_str(&css[copied_to..]);Some(out)}-/// Calls `found` with each image URL and rewrite in `css`: a `style` attribute, `<style>` text or `_cssText`.-pub fn image_urls(css: &str, resolver: &mut Resolver<'_>, found: &mut impl FnMut(CssUrl<'_>)) {- for (start, end) in image_value_ranges(css) {- value_urls(&css[start..end], resolver, found);+/// Calls `found` with each URL and rewrite in `css`: a `style` attribute, `<style>` text or `_cssText`.+pub fn urls(css: &str, resolver: &mut Resolver<'_>, found: &mut impl FnMut(CssUrl<'_>)) {+ let Declarations {+ found: declarations,+ nested_too_deep,+ ..+ } = declarations(css);+ if nested_too_deep {+ found(CssUrl::NestedTooDeep);+ }+ let mut families: Vec<Option<&str>> = Vec::new();+ for declaration in &declarations {+ if let DeclarationKind::FontFamily(face) = declaration.kind {+ if families.len() <= face {+ families.resize(face + 1, None);+ }+ families[face] = Some(&css[declaration.start..declaration.end]);+ }+ }+ for declaration in &declarations {+ let value = &css[declaration.start..declaration.end];+ match declaration.kind {+ DeclarationKind::Image => value_urls(value, resolver, EscapedSources::Skipped, found),+ DeclarationKind::FontSrc(face) => {+ let family = families.get(face).copied().flatten();+ let resolved_kind = match family.and_then(bundled_font) {+ Some(slug) => RemoteKind::BundledFont(slug),+ None => RemoteKind::Font,+ };+ value_urls(+ value,+ resolver,+ EscapedSources::Decoded,+ &mut |url| match url {+ CssUrl::Image(url) => found(CssUrl::Font { url, family }),+ CssUrl::Resolved(url) if resolved_kind == RemoteKind::Font => {+ found(CssUrl::ResolvedFont(url))+ }+ CssUrl::Rewrite(mut rewrite) => {+ if let Some(resolved) = &mut rewrite.resolved {+ resolved.kind = resolved_kind;+ }+ found(CssUrl::Rewrite(rewrite));+ }+ CssUrl::Unresolved(text) => found(CssUrl::Unresolved(text)),+ CssUrl::NestedTooDeep => found(CssUrl::NestedTooDeep),+ CssUrl::Resolved(_)+ | CssUrl::Unshown(_)+ | CssUrl::Font { .. }+ | CssUrl::ResolvedFont(_) => {}+ },+ );+ }+ DeclarationKind::FontFamily(_) => {}+ }}}@@ -160,7 +225,7 @@ pub fn declaration_image_urls(found: &mut impl FnMut(CssUrl<'_>),) {if can_hold_image(property.trim()) {- value_urls(value, resolver, found);+ value_urls(value, resolver, EscapedSources::Skipped, found);}}@@ -188,175 +253,324 @@ fn can_hold_image(property: &str) -> bool {property.starts_with("--") || is_image_property(property)}-fn image_value_ranges(css: &str) -> Vec<(usize, usize)> {- let bytes = css.as_bytes();- let mut ranges: Vec<(usize, usize)> = Vec::new();- // Where the text of the current declaration starts, after the whitespace that leads it.- let mut segment_start: Option<usize> = None;- let mut brace_depth = 0usize;- let mut paren_depth = 0usize;- let mut bracket_depth = 0usize;- let mut position = 0;- while position < bytes.len() {- if segment_start.is_none() && !bytes[position].is_ascii_whitespace() {- segment_start = Some(position);- }- if starts_with(bytes, position, b"/*") {- position = skip_comment(bytes, position).unwrap_or(bytes.len());- continue;- }- if bytes[position] == b'\\' {- position = skip_escape(bytes, position);- continue;- }- if matches!(bytes[position], b'\'' | b'"') {- position = skip_quoted(bytes, position).unwrap_or(bytes.len());- continue;- }- match bytes[position] {- b'(' => paren_depth += 1,- b')' => paren_depth = paren_depth.saturating_sub(1),- b'[' => bracket_depth += 1,- b']' => bracket_depth = bracket_depth.saturating_sub(1),- b'{' if paren_depth == 0 && bracket_depth == 0 => {- brace_depth += 1;- segment_start = None;- }- b'}' if paren_depth == 0 && bracket_depth == 0 => {- brace_depth = brace_depth.saturating_sub(1);- segment_start = None;- }- b';' if paren_depth == 0 && bracket_depth == 0 => segment_start = None,- // A colon in a custom property value, as in `--link: https://…`, is not a new declaration.- b':' if paren_depth == 0- && bracket_depth == 0- && ranges.last().is_none_or(|previous| position >= previous.1) =>- {- // Each colon trims only the whitespace before it, so a long selector list stays linear.- let property = css[segment_start.unwrap_or(position)..position].trim_end();- if can_hold_image(property) {- ranges.push((- position + 1,- declaration_end(bytes, position + 1, brace_depth),- ));- }- }- _ => {}- }- position += 1;+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum DeclarationKind {+ Image,+ FontSrc(usize),+ FontFamily(usize),+}++struct Declaration {+ start: usize,+ end: usize,+ kind: DeclarationKind,+}++#[derive(Clone, Copy)]+enum Block {+ Style,+ FontFace(usize),+}++/// The text can be a style attribute, a style sheet or one rule, so its top level reads as a rule body, which holds declarations and rules alike.+fn declarations(css: &str) -> Declarations {+ let mut declarations = Declarations {+ found: Vec::new(),+ font_faces: 0,+ block: Block::Style,+ depth: 0,+ nested_too_deep: false,+ };+ let mut input = Parser::new(css);+ input.set_nested_block_limit(NESTED_BLOCK_LIMIT);+ declarations.read_block(&mut input, Block::Style);+ declarations+}++struct Declarations {+ found: Vec<Declaration>,+ font_faces: usize,+ block: Block,+ depth: u8,+ nested_too_deep: bool,+}++impl Declarations {+ fn read_block(&mut self, input: &mut Parser<'_>, block: Block) {+ let outer = std::mem::replace(&mut self.block, block);+ self.nested_too_deep |= self.depth >= NESTED_BLOCK_LIMIT;+ let body: RuleBodyParser<'_, '_, Self, (), ()> = RuleBodyParser::new(input, self);+ for _ in body {}+ self.block = outer;+ }++ fn read_nested_block(&mut self, input: &mut Parser<'_>, block: Block) {+ self.depth += 1;+ self.read_block(input, block);+ self.depth -= 1;}- ranges}-fn declaration_end(bytes: &[u8], start: usize, brace_depth: usize) -> usize {- let mut position = start;- let mut paren_depth = 0usize;- let mut bracket_depth = 0usize;- while position < bytes.len() {- if starts_with(bytes, position, b"/*") {- position = skip_comment(bytes, position).unwrap_or(bytes.len());- continue;- }- if bytes[position] == b'\\' {- position = skip_escape(bytes, position);- continue;+impl<'i> DeclarationParser<'i> for Declarations {+ type Declaration = ();+ type Error = ();++ /// A value that holds a `{}` block is a nested rule, as `a:hover{…}` is, unless the property is a custom one.+ fn parse_value(+ &mut self,+ name: CowRcStr<'i>,+ input: &mut Parser<'i>,+ _: &ParserState,+ ) -> Result<(), ParseError<()>> {+ let start = input.position();+ let custom = name.starts_with("--");+ while let Ok(token) = input.next() {+ if !custom && matches!(token, Token::CurlyBracketBlock) {+ return Err(ParseError::unexpected_token());+ }}- if matches!(bytes[position], b'\'' | b'"') {- position = skip_quoted(bytes, position).unwrap_or(bytes.len());- continue;+ let kind = match self.block {+ Block::FontFace(face) => font_descriptor(&name, face),+ Block::Style => can_hold_image(&name).then_some(DeclarationKind::Image),+ };+ if let Some(kind) = kind {+ self.found.push(Declaration {+ start: start.byte_index(),+ end: input.position().byte_index(),+ kind,+ });}- match bytes[position] {- b'(' => paren_depth += 1,- b')' => paren_depth = paren_depth.saturating_sub(1),- b'[' => bracket_depth += 1,- b']' => bracket_depth = bracket_depth.saturating_sub(1),- b';' if paren_depth == 0 && bracket_depth == 0 => return position,- b'}' if paren_depth == 0 && bracket_depth == 0 && brace_depth > 0 => return position,- _ => {}+ Ok(())+ }+}++impl<'i> AtRuleParser<'i> for Declarations {+ type Prelude = Block;+ type AtRule = ();+ type Error = ();++ fn parse_prelude(+ &mut self,+ name: CowRcStr<'i>,+ input: &mut Parser<'i>,+ ) -> Result<Block, ParseError<()>> {+ let font_face = name.eq_ignore_ascii_case("font-face") && input.is_exhausted();+ while input.next().is_ok() {}+ if !font_face {+ return Ok(Block::Style);}- position += 1;+ self.font_faces += 1;+ Ok(Block::FontFace(self.font_faces - 1))+ }++ fn rule_without_block(&mut self, _: Block, _: &ParserState) -> Result<(), ()> {+ Ok(())+ }++ fn parse_block(+ &mut self,+ block: Block,+ _: &ParserState,+ input: &mut Parser<'i>,+ ) -> Result<(), ParseError<()>> {+ self.read_nested_block(input, block);+ Ok(())+ }+}++impl<'i> QualifiedRuleParser<'i> for Declarations {+ type Prelude = ();+ type QualifiedRule = ();+ type Error = ();++ fn parse_prelude(&mut self, input: &mut Parser<'i>) -> Result<(), ParseError<()>> {+ while input.next().is_ok() {}+ Ok(())+ }++ fn parse_block(+ &mut self,+ _: (),+ _: &ParserState,+ input: &mut Parser<'i>,+ ) -> Result<(), ParseError<()>> {+ self.read_nested_block(input, Block::Style);+ Ok(())}- bytes.len()}-/// The longest name [`is_image_property`] accepts, so that a longer text is not lowercased.-const MAX_IMAGE_PROPERTY_LEN: usize = "-webkit-mask-box-image-source".len();+impl<'i> RuleBodyItemParser<'i, (), ()> for Declarations {+ fn parse_declarations(&self) -> bool {+ true+ }++ fn parse_qualified(&self) -> bool {+ matches!(self.block, Block::Style)+ }+}++fn font_descriptor(descriptor: &str, face: usize) -> Option<DeclarationKind> {+ if descriptor.eq_ignore_ascii_case("src") {+ Some(DeclarationKind::FontSrc(face))+ } else if descriptor.eq_ignore_ascii_case("font-family") {+ Some(DeclarationKind::FontFamily(face))+ } else {+ None+ }+}fn is_image_property(property: &str) -> bool {- property.len() <= MAX_IMAGE_PROPERTY_LEN- && matches!(- property.to_ascii_lowercase().as_str(),- "background"- | "background-image"- | "border-image"- | "border-image-source"- | "content"- | "cursor"- | "list-style"- | "list-style-image"- | "mask"- | "mask-border"- | "mask-border-source"- | "mask-image"- | "shape-outside"- | "symbols"- | "-webkit-border-image"- | "-webkit-border-image-source"- | "-webkit-box-reflect"- | "-webkit-mask"- | "-webkit-mask-box-image"- | "-webkit-mask-box-image-source"- | "-webkit-mask-image"- )-}--fn value_urls(value: &str, resolver: &mut Resolver<'_>, found: &mut impl FnMut(CssUrl<'_>)) {- let bytes = value.as_bytes();- let mut position = 0;- while position < bytes.len() {- if starts_with(bytes, position, b"/*") {- position = skip_comment(bytes, position).unwrap_or(bytes.len());- continue;- }- if bytes[position] == b'\\' {- position = skip_escape(bytes, position);- continue;- }- if matches!(bytes[position], b'\'' | b'"') {- position = skip_quoted(bytes, position).unwrap_or(bytes.len());- continue;- }- let image_set = if function_at(bytes, position, b"image-set") {- Some(b"image-set".len())- } else if function_at(bytes, position, b"-webkit-image-set") {- Some(b"-webkit-image-set".len())- } else {- None+ match_ignore_ascii_case! { property,+ "background"+ | "background-image"+ | "border-image"+ | "border-image-source"+ | "content"+ | "cursor"+ | "list-style"+ | "list-style-image"+ | "mask"+ | "mask-border"+ | "mask-border-source"+ | "mask-image"+ | "shape-outside"+ | "symbols"+ | "-webkit-border-image"+ | "-webkit-border-image-source"+ | "-webkit-box-reflect"+ | "-webkit-mask"+ | "-webkit-mask-box-image"+ | "-webkit-mask-box-image-source"+ | "-webkit-mask-image" => true,+ _ => false,+ }+}++/// An image source that holds a CSS escape is skipped, as v3 skips it. A font has no v3 to match, and its URL must not stay.+#[derive(Clone, Copy, PartialEq, Eq)]+enum EscapedSources {+ Skipped,+ Decoded,+}++enum Opened<'i> {+ UnquotedUrl(CowRcStr<'i>),+ UrlFunction,+ ImageSet,+}++/// Calls `each` at each `url()` and `image-set()` in `input`, at any depth. Returns false when it skipped a block too deep to read whole, as `each` enters one more block for an `image-set()` candidate.+fn each_function<'i>(+ input: &mut Parser<'i>,+ depth: u8,+ each: &mut impl FnMut(&mut Parser<'i>, SourcePosition, Opened<'i>),+) -> bool {+ let mut whole = true;+ loop {+ let start = input.position();+ let token = match input.next_including_whitespace_and_comments() {+ Err(_) => return whole,+ Ok(token) => token,};- if let Some(name_len) = image_set {- let Some(end) = matching_paren(bytes, position + name_len) else {- return;- };- image_set_urls(&value[position..end], name_len, resolver, found);- position = end;+ let opens_block = matches!(+ token,+ Token::Function(_)+ | Token::ParenthesisBlock+ | Token::SquareBracketBlock+ | Token::CurlyBracketBlock+ );+ if opens_block && depth + 1 >= NESTED_BLOCK_LIMIT {+ whole = false;continue;}- if function_at(bytes, position, b"url") {- let (resume_at, source) = scan_url_function(value, position);- if let Some(source) = source.as_deref().and_then(collectable) {- url_function_urls(&value[position..resume_at], source, resolver, found);+ let opened = match token {+ Token::UnquotedUrl(url) => Opened::UnquotedUrl(url.clone()),+ Token::Function(name) if name.eq_ignore_ascii_case("url") => Opened::UrlFunction,+ Token::Function(name) if is_image_set(name) => Opened::ImageSet,+ _ if opens_block => {+ let _ = input.parse_nested_block(|input| {+ whole &= each_function(input, depth + 1, each);+ Ok::<_, ParseError<()>>(())+ });+ continue;}- position = resume_at;- continue;+ _ => continue,+ };+ each(input, start, opened);+ }+}++fn is_image_set(name: &str) -> bool {+ name.eq_ignore_ascii_case("image-set") || name.eq_ignore_ascii_case("-webkit-image-set")+}++fn skip_block(input: &mut Parser<'_>) {+ let _ = input.parse_nested_block(|_| Ok::<_, ParseError<()>>(()));+}++fn value_urls(+ value: &str,+ resolver: &mut Resolver<'_>,+ escaped: EscapedSources,+ found: &mut impl FnMut(CssUrl<'_>),+) {+ let mut input = Parser::new(value);+ input.set_nested_block_limit(NESTED_BLOCK_LIMIT);+ let whole = each_function(&mut input, 0, &mut |input, start, opened| {+ let source = match opened {+ Opened::ImageSet => return image_set_urls(input, start, resolver, found),+ Opened::UnquotedUrl(url) => Some(unquoted_url_source(url, input.slice_from(start))),+ Opened::UrlFunction => quoted_url_source(input),+ };+ let Some(source) = source else { return };+ let function = input.slice_from(start);+ match collectable(&source) {+ Some(url) => url_function_urls(function, url, resolver, found),+ None if source.escaped && escaped == EscapedSources::Decoded => {+ escaped_url_function_urls(function, &source.url, resolver, found)+ }+ None => {}}- position += 1;+ });+ if !whole {+ found(CssUrl::NestedTooDeep);}}+struct UrlSource<'i> {+ url: CowRcStr<'i>,+ escaped: bool,+}++/// The name before the `(` can be escaped, as in `\75 rl(`, and still read as `url`, so only the text after it counts.+fn unquoted_url_source<'i>(url: CowRcStr<'i>, token: &str) -> UrlSource<'i> {+ let contents = token+ .split_once('(')+ .map_or(token, |(_, contents)| contents);+ UrlSource {+ escaped: contents.contains('\\'),+ url,+ }+}++fn quoted_url_source<'i>(input: &mut Parser<'i>) -> Option<UrlSource<'i>> {+ input+ .parse_nested_block(|input| {+ input.skip_whitespace();+ let start = input.position();+ let url = input.expect_string_cloned()?;+ Ok::<_, ParseError<()>>(UrlSource {+ escaped: input.slice_from(start).contains('\\'),+ url,+ })+ })+ .ok()+}+/// v3 collects neither a fragment nor a source with an escape in it.-fn collectable(source: &str) -> Option<&str> {- let source = source.trim();- (!source.starts_with('#') && !source.contains('\\')).then_some(source)+fn collectable<'s>(source: &'s UrlSource<'_>) -> Option<&'s str> {+ let url = source.url.trim();+ (!source.escaped && !url.is_empty() && !url.starts_with('#')).then_some(url)}fn url_function_urls(@@ -372,7 +586,10 @@ fn url_function_urls(text: function.to_owned(),replacement: url_function(&absolute),dropped: Vec::new(),- resolved: Some(absolute),+ resolved: Some(RemoteUrl {+ url: absolute,+ kind: RemoteKind::Image,+ }),}));}None if is_root_relative(source) => found(CssUrl::Unresolved(function)),@@ -380,6 +597,42 @@ fn url_function_urls(}}+/// The block does not hold the URL a browser reads from an escaped source, so the rewrite replaces the whole `url()`.+fn escaped_url_function_urls(+ function: &str,+ decoded: &str,+ resolver: &mut Resolver<'_>,+ found: &mut impl FnMut(CssUrl<'_>),+) {+ let decoded = decoded.trim();+ if is_root_relative(decoded) {+ return url_function_urls(function, decoded, resolver, found);+ }+ let decoded = if is_protocol_relative(decoded) {+ Cow::Owned(format!("https:{decoded}"))+ } else {+ Cow::Borrowed(decoded)+ };+ // A serialized URL percent-encodes `"` and holds no backslash, so the quotes of the rewrite hold it.+ let Some(absolute) = Url::parse(&decoded)+ .ok()+ .filter(|url| matches!(url.scheme(), "http" | "https"))+ .map(String::from)+ else {+ return;+ };+ found(CssUrl::Resolved(&absolute));+ found(CssUrl::Rewrite(CssRewrite {+ text: function.to_owned(),+ replacement: url_function(&absolute),+ dropped: Vec::new(),+ resolved: Some(RemoteUrl {+ url: absolute,+ kind: RemoteKind::Image,+ }),+ }));+}+fn is_root_relative(source: &str) -> bool {source.starts_with('/') && !is_protocol_relative(source)}@@ -392,37 +645,23 @@ fn url_function(url: &str) -> String {/// A renderer picks the candidate for its pixel ratio and fails offline on any the fetcher did not take, so the set keeps/// one: the highest resolution of a stored type. Fetching one candidate per set limits the load on the image's host.fn image_set_urls(- function: &str,- name_len: usize,+ input: &mut Parser<'_>,+ start: SourcePosition,resolver: &mut Resolver<'_>,found: &mut impl FnMut(CssUrl<'_>),) {- let contents = &function[name_len + 1..function.len() - 1];- let mut candidates = Vec::new();- let mut count = 0;- let mut readable = true;- let split = split_image_set(contents, &mut |text| {- count += 1;- if count <= MAX_IMAGE_SET_CANDIDATES {- match image_set_candidate(text) {- Some(candidate) => candidates.push(candidate),- None => readable = false,- }- }- });- if split.is_none() || !readable {- return;- }- if count > MAX_IMAGE_SET_CANDIDATES {- split_image_set(contents, &mut |text| {- if let Some(source) = image_set_candidate(text).and_then(|candidate| candidate.source) {- if is_remote(&source) {- found(CssUrl::Unshown(&source));- }- }- });+ let opening = input.slice_from(start);+ let name = &opening[..opening.len() - 1];+ let candidates = input+ .parse_nested_block(|input| {+ Ok::<_, ParseError<()>>(image_set_candidates(input, &mut *found))+ })+ .ok()+ .flatten();+ let function = input.slice_from(start);+ let Some(candidates) = candidates else {return;- }+ };let loads = |source: &str| {is_data_url(source)|| (is_remote(source) && try_canonicalize(source).is_ok())@@ -473,9 +712,12 @@ fn image_set_urls(if !holds_data_url && (candidates.len() > 1 || resolved.is_some()) {found(CssUrl::Rewrite(CssRewrite {text: function.to_owned(),- replacement: format!("{}({candidate})", &function[..name_len]),+ replacement: format!("{name}({candidate})"),dropped,- resolved,+ resolved: resolved.map(|url| RemoteUrl {+ url,+ kind: RemoteKind::Image,+ }),}));}}@@ -493,226 +735,122 @@ struct ImageSetCandidate<'a> {resolution_dppx: Option<f64>,}-fn split_image_set<'a>(contents: &'a str, each: &mut impl FnMut(&'a str)) -> Option<()> {- let bytes = contents.as_bytes();- let mut start = 0;- let mut position = 0;- let mut depth = 0usize;- while position <= bytes.len() {- if position == bytes.len() || (bytes[position] == b',' && depth == 0) {- each(&contents[start..position]);- start = position + 1;- position += 1;- continue;- }- if bytes[position] == b'\\' {- position = skip_escape(bytes, position);- continue;- }- if starts_with(bytes, position, b"/*") {- position = skip_comment(bytes, position)?;- continue;- }- if matches!(bytes[position], b'\'' | b'"') {- position = skip_quoted(bytes, position)?;- continue;- }- match bytes[position] {- b'(' => depth += 1,- b')' => depth = depth.saturating_sub(1),- _ => {}- }- position += 1;- }- Some(())-}--fn image_set_candidate(candidate: &str) -> Option<ImageSetCandidate<'_>> {- let candidate = candidate.trim();- if candidate.is_empty() {- return None;- }- let bytes = candidate.as_bytes();- let (source, descriptors) = if function_at(bytes, 0, b"url") {- let (resume_at, source) = scan_url_function(candidate, 0);- (source, &candidate[resume_at..])- } else if matches!(bytes[0], b'\'' | b'"') {- let end = skip_quoted(bytes, 0)?;- (Some(candidate[1..end - 1].to_owned()), &candidate[end..])- } else {- (None, "")- };- Some(ImageSetCandidate {- text: candidate,- source: source.as_deref().and_then(collectable).map(str::to_owned),- descriptors,- resolution_dppx: parse_descriptors(descriptors),- })-}--fn parse_descriptors(descriptors: &str) -> Option<f64> {- let bytes = descriptors.as_bytes();- let mut resolution = None;- let mut position = 0;+/// `None` when a candidate is empty, or when the set lists more than [`MAX_IMAGE_SET_CANDIDATES`]. Then each remote candidate is reported as unshown, as the set is read, so that a long set is never held.+fn image_set_candidates<'i>(+ input: &mut Parser<'i>,+ found: &mut impl FnMut(CssUrl<'_>),+) -> Option<Vec<ImageSetCandidate<'i>>> {+ let mut candidates = Vec::new();+ let mut count = 0;+ let mut readable = true;loop {- skip_whitespace(bytes, &mut position);- if position == bytes.len() {- break;- }- if function_at(bytes, position, b"type") {- let end = matching_paren(bytes, position + 4)?;- let media_type = descriptors[position + 5..end - 1]- .trim()- .trim_matches(['"', '\'']);- if !is_fetched_image_type(media_type) {- return None;+ let candidate = input+ .parse_until_before(Delimiter::Comma, |input| {+ Ok::<_, ParseError<()>>(image_set_candidate(input))+ })+ .ok()+ .flatten();+ count += 1;+ if count <= MAX_IMAGE_SET_CANDIDATES {+ match candidate {+ Some(candidate) => candidates.push(candidate),+ None => readable = false,+ }+ } else if readable {+ let held = std::mem::take(&mut candidates);+ for source in held.into_iter().chain(candidate).filter_map(|c| c.source) {+ if is_remote(&source) {+ found(CssUrl::Unshown(&source));+ }}- position = end;- continue;}- let token_end = bytes[position..]- .iter()- .position(u8::is_ascii_whitespace)- .map_or(bytes.len(), |length| position + length);- if resolution- .replace(parse_resolution(&descriptors[position..token_end])?)- .is_some()- {- return None;+ if input.next().is_err() {+ break;}- position = token_end;}- let value = resolution.unwrap_or(1.0);- (value.is_finite() && value > 0.0).then_some(value)+ (readable && count <= MAX_IMAGE_SET_CANDIDATES).then_some(candidates)}-fn parse_resolution(token: &str) -> Option<f64> {- if let Some(value) = token.strip_suffix("dppx") {- value.parse::<f64>().ok()- } else if let Some(value) = token.strip_suffix("dpi") {- value.parse::<f64>().ok().map(|value| value / 96.0)- } else if let Some(value) = token.strip_suffix("dpcm") {- value.parse::<f64>().ok().map(|value| value * 2.54 / 96.0)- } else {- token.strip_suffix('x')?.parse::<f64>().ok()- }+enum CandidateStart<'i> {+ Url(CowRcStr<'i>),+ String(CowRcStr<'i>),+ UrlFunction,+ Other,}-fn scan_url_function(value: &str, start: usize) -> (usize, Option<String>) {- let bytes = value.as_bytes();- let Some(resume_at) = matching_paren(bytes, start + 3) else {- return (bytes.len(), None);- };- let contents_end = resume_at - 1;- let mut position = start + 4;- skip_whitespace(bytes, &mut position);- let source = if matches!(bytes.get(position), Some(b'\'' | b'"')) {- let content_start = position + 1;- let Some(end) = skip_quoted(bytes, position) else {- return (resume_at, None);- };- position = end;- skip_whitespace(bytes, &mut position);- if position != contents_end {- return (resume_at, None);+fn image_set_candidate<'i>(input: &mut Parser<'i>) -> Option<ImageSetCandidate<'i>> {+ input.skip_whitespace();+ let start = input.position();+ let first = match input.next() {+ Err(_) => return None,+ Ok(Token::UnquotedUrl(url)) => CandidateStart::Url(url.clone()),+ Ok(Token::QuotedString(url)) => CandidateStart::String(url.clone()),+ Ok(Token::Function(name)) if name.eq_ignore_ascii_case("url") => {+ CandidateStart::UrlFunction}- &value[content_start..end - 1]- } else {- &value[position..contents_end]+ Ok(_) => CandidateStart::Other,+ };+ let source = match first {+ CandidateStart::Url(url) => Some(unquoted_url_source(url, input.slice_from(start))),+ CandidateStart::String(url) => Some(UrlSource {+ escaped: input.slice_from(start).contains('\\'),+ url,+ }),+ CandidateStart::UrlFunction => quoted_url_source(input),+ CandidateStart::Other => None,};- let source = source.trim();- (- resume_at,- (!source.is_empty() && !source.contains('\\')).then(|| source.to_owned()),- )+ let source_end = input.position();+ let resolution_dppx = parse_descriptors(input);+ Some(ImageSetCandidate {+ text: input.slice_from(start).trim_end(),+ source: source.as_ref().and_then(collectable).map(str::to_owned),+ descriptors: input.slice(source_end..input.position()),+ resolution_dppx,+ })}-fn function_at(bytes: &[u8], start: usize, name: &[u8]) -> bool {- bytes- .get(start..start + name.len())- .is_some_and(|candidate| candidate.eq_ignore_ascii_case(name))- && bytes.get(start + name.len()) == Some(&b'(')- && (start == 0- || !matches!(bytes[start - 1], b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-'))+enum Descriptor {+ Resolution(f64),+ Type,}-fn matching_paren(bytes: &[u8], open: usize) -> Option<usize> {- if bytes.get(open) != Some(&b'(') {- return None;- }- let mut position = open + 1;- let mut depth = 1usize;- while position < bytes.len() {- if starts_with(bytes, position, b"/*") {- position = skip_comment(bytes, position)?;- continue;- }- if bytes[position] == b'\\' {- position = skip_escape(bytes, position);- continue;- }- if matches!(bytes[position], b'\'' | b'"') {- position = skip_quoted(bytes, position)?;- continue;- }- match bytes[position] {- b'(' => depth += 1,- b')' => {- depth -= 1;- if depth == 0 {- return Some(position + 1);- }+fn parse_descriptors(input: &mut Parser<'_>) -> Option<f64> {+ let mut resolution = None;+ let mut readable = true;+ loop {+ let descriptor = match input.next() {+ Err(_) => break,+ Ok(Token::Dimension { value, unit, .. }) => {+ resolution_dppx(f64::from(*value), unit).map(Descriptor::Resolution)}- _ => {}- }- position += 1;- }- None-}--fn skip_quoted(bytes: &[u8], start: usize) -> Option<usize> {- let quote = *bytes.get(start)?;- let mut position = start + 1;- while position < bytes.len() {- match bytes[position] {- b'\\' => position += 2,- value if value == quote => return Some(position + 1),- _ => position += 1,- }+ Ok(Token::Function(name)) if name.eq_ignore_ascii_case("type") => {+ Some(Descriptor::Type)+ }+ Ok(_) => None,+ };+ readable &= match descriptor {+ Some(Descriptor::Resolution(dppx)) => resolution.replace(dppx).is_none(),+ Some(Descriptor::Type) => input+ .parse_nested_block(|input| {+ Ok::<_, ParseError<()>>(is_fetched_image_type(input.expect_string()?.trim()))+ })+ .unwrap_or(false),+ None => false,+ };}- None-}--/// The position after the escape at `start` and the character it escapes, so that an escaped quote, bracket or-/// parenthesis does not count as one.-fn skip_escape(bytes: &[u8], start: usize) -> usize {- let width = match bytes.get(start + 1) {- Some(0xf0..) => 4,- Some(0xe0..) => 3,- Some(0xc0..) => 2,- _ => 1,- };- (start + 1 + width).min(bytes.len())-}--fn skip_comment(bytes: &[u8], start: usize) -> Option<usize> {- let relative = bytes- .get(start + 2..)?- .windows(2)- .position(|pair| pair == b"*/")?;- Some(start + 2 + relative + 2)+ let value = resolution.unwrap_or(1.0);+ (readable && value.is_finite() && value > 0.0).then_some(value)}-fn skip_whitespace(bytes: &[u8], position: &mut usize) {- while bytes.get(*position).is_some_and(u8::is_ascii_whitespace) {- *position += 1;+fn resolution_dppx(value: f64, unit: &str) -> Option<f64> {+ match_ignore_ascii_case! { unit,+ "x" | "dppx" => Some(value),+ "dpi" => Some(value / 96.0),+ "dpcm" => Some(value * 2.54 / 96.0),+ _ => None,}}-fn starts_with(bytes: &[u8], start: usize, needle: &[u8]) -> bool {- bytes.get(start..start + needle.len()) == Some(needle)-}-#[cfg(test)]mod tests {use super::*;@@ -720,7 +858,7 @@ mod tests {fn found(css: &str, page_url: Option<&str>) -> Vec<String> {let base = page_url.and_then(page_base);let mut found = Vec::new();- image_urls(css, &mut Resolver::new(base.as_ref(), None), &mut |url| {+ urls(css, &mut Resolver::new(base.as_ref(), None), &mut |url| {found.push(match url {CssUrl::Image(url) => format!("image {url}"),CssUrl::Unshown(url) => format!("unshown {url}"),@@ -730,20 +868,70 @@ mod tests {"rewrite {} => {} dropping {:?}",rewrite.text, rewrite.replacement, rewrite.dropped),+ CssUrl::Font { url, family } => format!("font {url} {family:?}"),+ CssUrl::ResolvedFont(url) => format!("resolved font {url}"),+ CssUrl::NestedTooDeep => "nested too deep".to_owned(),})});found}+ #[test]+ fn reports_each_font_face_src_url_with_its_rules_family() {+ let css = concat!(+ "/* fonts */@FONT-FACE{font-family:'Brand Sans';src:url(https://cdn.example.com/brand.woff2) format('woff2'),local(Arial),url(\"https://cdn.example.com/brand.woff\")}",+ "@media print{@font-face{src:url(https://cdn.example.com/print.woff2);font-family:Print}}",+ "@font-face{src:url(https://cdn.example.com/nameless.woff2)}",+ "@font-face{font-family:Self;src:url(/fonts/self.woff2)}",+ ".a{font-family:Inter;src:url(https://cdn.example.com/not-a-face.woff2);background:url(https://cdn.example.com/a.png)}",+ ".\\@font-face{src:url(https://cdn.example.com/escaped.woff2);background:url(https://cdn.example.com/escaped.png)}",+ "@font-face/* trailing */{font-family:Trail;src:url(https://cdn.example.com/trail.woff2)}",+ "@\\66 ont-face{font-family:Keyword;src:url(https://cdn.example.com/keyword.woff2)}",+ "@font-face{font-family:Descriptor;\\73 rc:url(https://cdn.example.com/descriptor.woff2)}",+ "@font-face{font-family:Function;src:\\75 rl(https://cdn.example.com/function.woff2)}",+ "@font-face{font-family:Source;src:url(https://cdn.example.com/source\\2e woff2)}",+ "@font-face{font-family:Relative;src:url(//cdn.example.com/relative\\2e woff2)}",+ "@font-face{font-family:Continued;src:url(\"https://cdn.example.com/continued.\\\nwoff2\")}",+ "@font-face{font-family:Broken;src:url(https://cdn.example.com/broken.\\\nwoff2)}",+ "@font-face{font-family:Commented;/* lead */src/**/:url(\"https://cdn.example.com/commented.woff2\"/* tail */)}",+ );+ assert_eq!(+ found(css, Some("https://app.example.com/")),+ [+ "font https://cdn.example.com/brand.woff2 Some(\"'Brand Sans'\")",+ "font https://cdn.example.com/brand.woff Some(\"'Brand Sans'\")",+ "font https://cdn.example.com/print.woff2 Some(\"Print\")",+ "font https://cdn.example.com/nameless.woff2 None",+ "resolved font https://app.example.com/fonts/self.woff2",+ "rewrite url(/fonts/self.woff2) => url(\"https://app.example.com/fonts/self.woff2\") dropping []",+ "image https://cdn.example.com/a.png",+ "image https://cdn.example.com/escaped.png",+ "font https://cdn.example.com/trail.woff2 Some(\"Trail\")",+ "font https://cdn.example.com/keyword.woff2 Some(\"Keyword\")",+ "font https://cdn.example.com/descriptor.woff2 Some(\"Descriptor\")",+ "font https://cdn.example.com/function.woff2 Some(\"Function\")",+ "resolved font https://cdn.example.com/source.woff2",+ "rewrite url(https://cdn.example.com/source\\2e woff2) => url(\"https://cdn.example.com/source.woff2\") dropping []",+ "resolved font https://cdn.example.com/relative.woff2",+ "rewrite url(//cdn.example.com/relative\\2e woff2) => url(\"https://cdn.example.com/relative.woff2\") dropping []",+ "resolved font https://cdn.example.com/continued.woff2",+ "rewrite url(\"https://cdn.example.com/continued.\\\nwoff2\") => url(\"https://cdn.example.com/continued.woff2\") dropping []",+ "font https://cdn.example.com/commented.woff2 Some(\"Commented\")",+ ]+ );+ }+#[test]fn reads_past_escapes_outside_strings() {// Tailwind escapes brackets, parentheses and quotes in its arbitrary value class names.- let css = r".bg-\[url\(\'\/hero\.png\'\)\]{background-image:url('/hero.png')}.a{background:url(https://cdn.example.com/a.png)}";+ let css = r".bg-\[url\(\'\/hero\.png\'\)\]{background-image:url('/hero.png')}.a{background:url(https://cdn.example.com/a.png)}.p{background-\69mage:url(https://cdn.example.com/p.png)}.f{background:\75 rl(https://cdn.example.com/f.png)}";assert_eq!(found(css, None),["unresolved url('/hero.png')",- "image https://cdn.example.com/a.png"+ "image https://cdn.example.com/a.png",+ "image https://cdn.example.com/p.png",+ "image https://cdn.example.com/f.png",]);}@@ -776,6 +964,10 @@ mod tests {".a{background:url(../img/a.png)}",vec!["image ../img/a.png"],),+ (+ ".a{/* lead */background/**/:url(\"https://cdn.example.com/c.png\"/* tail */)}",+ vec!["image https://cdn.example.com/c.png"],+ ),(r#".a{background-image:image-set("https://cdn.example.com/1x.png" 1x, url(https://cdn.example.com/2x.png) 2x)}"#,vec![@@ -795,6 +987,14 @@ mod tests {".a{background:image-set(url(https://cdn.example.com/a.png))}",vec!["image https://cdn.example.com/a.png"],),+ (+ ".a{background:url(https://cdn.example.com/unclosed.png",+ vec!["image https://cdn.example.com/unclosed.png"],+ ),+ (+ ".a{background:url(https://cdn.example.com/a(1).png)}",+ vec![],+ ),] {assert_eq!(found(css, page), expected, "{css}");}@@ -802,7 +1002,7 @@ mod tests {let base = page_base("https://app.example.com/shop/item");let budget = AtomicUsize::new("https://app.example.com/img/a.png".len());let mut named = Vec::new();- image_urls(+ urls(".a{background:url(/img/a.png)} .b{background:url(/img/b.png)}",&mut Resolver::new(base.as_ref(), Some(&budget)),&mut |url| {diff --git a/rust/ai-research/replay-core-v4/src/inspect/dom.rs b/rust/ai-research/replay-core-v4/src/inspect/dom.rsindex fd93af338d1c..967877c46625 100644--- a/rust/ai-research/replay-core-v4/src/inspect/dom.rs+++ b/rust/ai-research/replay-core-v4/src/inspect/dom.rs@@ -9,6 +9,7 @@ use std::sync::Arc;use url::Url;use crate::bytescan::{self as scan, Span};+use crate::fonts::{bundled_font, is_fetched_font_url};use crate::url_policy::{is_fetched_image_type, is_remote, MAX_URL_LEN};use super::css::{self, CssRewrite, CssUrl, Resolver};@@ -27,6 +28,7 @@ use crate::scan::tokens::{Token, Tokens};pub const MAX_TRACKED_NODES: usize = 200_000;/// Remote image URLs kept per message, as the v3 collector caps URLs per message.pub const MAX_IMAGE_URLS: usize = 512;+pub const MAX_FONT_URLS: usize = 512;/// Attribute changes read per mutation, as many as the nodes tracked. A mutation with more drops its block, because/// the inspector does not name the URLs in the rest.pub const MAX_ATTRIBUTE_ITEMS: usize = MAX_TRACKED_NODES;@@ -34,6 +36,9 @@ pub const MAX_ATTRIBUTE_ITEMS: usize = MAX_TRACKED_NODES;/// the build takes memory in proportion to their bytes.pub const MAX_BLOCK_REMOTE_URLS: usize = 10_000;pub const MAX_BLOCK_REMOTE_URL_BYTES: usize = 2 * 1024 * 1024;+/// Font URLs have a budget of their own, so that many font faces never stop the naming of image URLs. Past it the writer scrubs the block's unnamed remote URLs, as past an image limit.+pub const MAX_BLOCK_FONT_URLS: usize = 4096;+pub const MAX_BLOCK_FONT_URL_BYTES: usize = 256 * 1024;/// Windows tracked per session. A new window past this evicts the window used least recently.pub const MAX_WINDOWS: usize = 16;/// Evicted windows remembered, so that one that comes back is known to have lost its DOM.@@ -191,13 +196,17 @@ struct WindowDom {page_base: Option<Arc<Url>>,}-/// Why the inspector stopped naming a block's remote image URLs. The writer replaces only the URLs it is given, so it-/// scrubs every other remote URL token in the block.+/// Why the inspector stopped naming a block's remote image or font URLs. The writer replaces only the URLs it is given, so it scrubs every other remote URL token in the block.#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum UrlLimit {UrlCount,UrlBytes,AttributeItems,+ FontUrlCount,+ FontUrlBytes,+ /// A font rewrite can hold an escaped remote URL, which no other rewrite replaces.+ FontRewrites,+ NestedBlocks,}impl UrlLimit {@@ -207,17 +216,49 @@ impl UrlLimit {UrlLimit::UrlCount => "too_many_urls",UrlLimit::UrlBytes => "too_many_url_bytes",UrlLimit::AttributeItems => "too_many_attribute_items",+ UrlLimit::FontUrlCount => "too_many_font_urls",+ UrlLimit::FontUrlBytes => "too_many_font_url_bytes",+ UrlLimit::FontRewrites => "too_many_font_rewrites",+ UrlLimit::NestedBlocks => "too_many_nested_blocks",}}++ /// The scrub replaces only the URL tokens it finds in the text, so it cannot vouch for CSS that the inspector could not read.+ pub fn drops_block(self) -> bool {+ matches!(self, UrlLimit::NestedBlocks)+ }++ pub fn is_font_budget(self) -> bool {+ matches!(+ self,+ UrlLimit::FontUrlCount | UrlLimit::FontUrlBytes | UrlLimit::FontRewrites+ )+ }+}++/// One URL gets one reference, so a URL used as several kinds takes the greatest. A fetched font beats a bundled one because it is the page's own file.+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]+pub enum RemoteKind {+ BundledFont(&'static str),+ Font,+ Image,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct RemoteUrl {+ pub url: String,+ pub kind: RemoteKind,}-/// Distinct remote image URLs within [`MAX_BLOCK_REMOTE_URLS`] and-/// [`MAX_BLOCK_REMOTE_URL_BYTES`]. Past a limit it keeps no more, and `limit` says why.+/// Distinct remote URLs, within a count and a byte budget per kind. Past a budget it keeps no more of that kind, and [`RemoteUrls::url_limit`] says why.#[derive(Debug, Default)]pub struct RemoteUrls {- urls: HashSet<String>,+ urls: HashMap<String, RemoteKind>,bytes: usize,+ font_urls: u16,+ font_bytes: u32,pub limit: Option<UrlLimit>,+ font_limit: Option<UrlLimit>,/// Each image-slot srcset value, and the slot that holds it. The writer rebuilds each from the candidates it keeps,/// and it builds one matcher over them as over `urls`, so they share the byte budget of `urls`. Past it this keeps no/// more, which is safe: the writer still replaces each URL in `urls` on its own.@@ -250,19 +291,36 @@ pub struct Srcset {}impl RemoteUrls {- fn insert(&mut self, url: &str) {- if self.limit.is_some() || self.urls.contains(url) {+ fn insert(&mut self, url: &str, kind: RemoteKind) {+ if let Some(known) = self.urls.get_mut(url) {+ *known = (*known).max(kind);+ return;+ }+ if kind != RemoteKind::Image {+ let font_bytes = self.font_bytes as usize + url.len();+ if usize::from(self.font_urls) >= MAX_BLOCK_FONT_URLS {+ self.font_limit = self.font_limit.or(Some(UrlLimit::FontUrlCount));+ } else if font_bytes > MAX_BLOCK_FONT_URL_BYTES {+ self.font_limit = self.font_limit.or(Some(UrlLimit::FontUrlBytes));+ } else {+ self.font_urls += 1;+ self.font_bytes = font_bytes as u32;+ self.urls.insert(url.to_owned(), kind);+ }+ return;+ }+ if self.limit.is_some() {return;}// A URL of any length is named: the policy declines one too long to fetch, and its bytes count toward the// block's budget.- self.limit = if self.urls.len() >= MAX_BLOCK_REMOTE_URLS {+ self.limit = if self.urls.len() - usize::from(self.font_urls) >= MAX_BLOCK_REMOTE_URLS {Some(UrlLimit::UrlCount)} else if self.bytes + url.len() > MAX_BLOCK_REMOTE_URL_BYTES {Some(UrlLimit::UrlBytes)} else {self.bytes += url.len();- self.urls.insert(url.to_owned());+ self.urls.insert(url.to_owned(), kind);None};}@@ -295,6 +353,11 @@ impl RemoteUrls {!= rewrite.as_ref().map(|rewrite| &rewrite.replacement){*known = None;+ } else if let (Some(known), Some(resolved)) = (+ known.as_mut().and_then(|known| known.resolved.as_mut()),+ rewrite.and_then(|rewrite| rewrite.resolved),+ ) {+ known.kind = known.kind.max(resolved.kind);}return;}@@ -302,22 +365,31 @@ impl RemoteUrls {+ rewrite.as_ref().map_or(0, |rewrite| {rewrite.replacement.len()+ rewrite.dropped.iter().map(String::len).sum::<usize>()- + rewrite.resolved.as_ref().map_or(0, String::len)+ + rewrite+ .resolved+ .as_ref()+ .map_or(0, |resolved| resolved.url.len())});if rewrites.values.len() < MAX_BLOCK_REMOTE_URLS&& rewrites.bytes + bytes <= MAX_BLOCK_REMOTE_URL_BYTES{rewrites.bytes += bytes;rewrites.values.insert(text, rewrite);+ } else if rewrite+ .and_then(|rewrite| rewrite.resolved)+ .is_some_and(|resolved| resolved.kind != RemoteKind::Image)+ {+ self.font_limit = self.font_limit.or(Some(UrlLimit::FontRewrites));}}pub fn extend(&mut self, other: RemoteUrls) {// The URLs go in before the other's limit, so that a part that hit a limit still names what it holds.- for url in &other.urls {- self.insert(url);+ for (url, &kind) in &other.urls {+ self.insert(url, kind);}self.limit = self.limit.or(other.limit);+ self.font_limit = self.font_limit.or(other.font_limit);for (raw, (text, slots)) in other.srcsets.iter().flat_map(|s| &s.values) {for &slot in slots {self.insert_srcset(raw, text, slot);@@ -328,6 +400,11 @@ impl RemoteUrls {}}+ /// Why the writer scrubs the block. Past a font limit, unlike past `limit`, the inspector still names image URLs.+ pub fn url_limit(&self) -> Option<UrlLimit> {+ self.limit.or(self.font_limit)+ }+pub fn take_css_rewrites(&mut self) -> Vec<CssRewrite> {self.css_rewrites.take()@@ -354,8 +431,11 @@ impl RemoteUrls {.collect()}- pub fn into_urls(self) -> Vec<String> {- self.urls.into_iter().collect()+ pub fn into_urls(self) -> Vec<RemoteUrl> {+ self.urls+ .into_iter()+ .map(|(url, kind)| RemoteUrl { url, kind })+ .collect()}}@@ -364,11 +444,19 @@ impl RemoteUrls {struct Urls {urls: Vec<String>,in_message: HashSet<String>,+ /// Boxed, because the session state counts this struct's size and most messages queue no font file.+ fonts: Option<Box<FontQueue>>,/// Every remote URL in an image slot, past the fetch cap and in a hidden img too. The writer replaces each one in/// the stored block, and the fetch cap limits only the fetch jobs.remote: RemoteUrls,}+#[derive(Default)]+struct FontQueue {+ urls: Vec<String>,+ in_message: HashSet<String>,+}+/// The URL as the fetcher takes it: remote, and no longer than v3 fetches. An inline image is moved out of the block/// by the image scan instead.fn fetchable(url: &str) -> Option<&str> {@@ -394,7 +482,32 @@ impl Urls {fn push_remote(&mut self, url: &str) {let url = url.trim();if is_remote(url) {- self.remote.insert(url);+ self.remote.insert(url, RemoteKind::Image);+ }+ }++ fn push_font(&mut self, url: &str, family: Option<&str>) {+ let url = url.trim();+ if !is_remote(url) {+ return;+ }+ match family.and_then(bundled_font) {+ Some(slug) => self.remote.insert(url, RemoteKind::BundledFont(slug)),+ None => {+ self.remote.insert(url, RemoteKind::Font);+ self.queue_font(url);+ }+ }+ }++ fn queue_font(&mut self, url: &str) {+ let Some(url) = fetchable(url).filter(|url| is_fetched_font_url(url)) else {+ return;+ };+ let fonts = self.fonts.get_or_insert_with(Box::default);+ if fonts.in_message.len() < MAX_FONT_URLS && !fonts.in_message.contains(url) {+ fonts.in_message.insert(url.to_owned());+ fonts.urls.push(url.to_owned());}}@@ -407,6 +520,11 @@ impl Urls {CssUrl::Rewrite(rewrite) => self.remote.insert_css_rewrite(rewrite.text.clone(), Some(rewrite)),+ CssUrl::Font { url, family } => self.push_font(url, family),+ CssUrl::ResolvedFont(url) => self.queue_font(url),+ CssUrl::NestedTooDeep => {+ self.remote.limit = self.remote.limit.or(Some(UrlLimit::NestedBlocks))+ }}}@@ -415,6 +533,9 @@ impl Urls {for url in &walk.urls {self.push_fetch(url);}+ for url in walk.fonts.iter().flat_map(|fonts| &fonts.urls) {+ self.queue_font(url);+ }self.remote.extend(walk.remote);}}@@ -535,7 +656,7 @@ impl Walk<'_> {fn push_css(&mut self, bytes: &[u8], value: Option<Span>) {let Some(value) = value else { return };if let Some(text) = text(bytes, Some(value)) {- css::image_urls(&text, &mut self.resolver, &mut |found| {+ css::urls(&text, &mut self.resolver, &mut |found| {self.image_urls.push_css(found)});} else if scan::is_object(bytes, value) {@@ -1288,12 +1409,23 @@ impl DomInspector {/// Starts a message: the image URL cap counts per message.pub fn start_message(&mut self) {self.urls.in_message.clear();+ if let Some(fonts) = &mut self.urls.fonts {+ fonts.in_message.clear();+ }}/// Ends a message. The URLs it saw go, so that they do not stay in memory that [`Self::retained_bytes`] does not/// count.pub fn finish_message(&mut self) {self.urls.in_message = HashSet::new();+ if self+ .urls+ .fonts+ .as_ref()+ .is_some_and(|fonts| fonts.urls.is_empty())+ {+ self.urls.fonts = None;+ }}/// Updates the window's DOM with one event and adds what it found to the block.@@ -1411,6 +1543,14 @@ impl DomInspector {std::mem::take(&mut self.urls.urls)}+ pub fn take_font_urls(&mut self) -> Vec<String> {+ self.urls+ .fonts+ .as_mut()+ .map(|fonts| std::mem::take(&mut fonts.urls))+ .unwrap_or_default()+ }+/// Every remote image URL the events since the last call hold, for the writer to replace.pub fn take_remote_urls(&mut self) -> RemoteUrls {std::mem::take(&mut self.urls.remote)@@ -1725,13 +1865,13 @@ fn style_sheet_urls(if let Some(adds) = adds.and_then(|adds| decoded_list(inner, adds, expansion)) {for item in items(adds) {if let Some(rule) = text(adds.bytes, field(adds.bytes, item, b"rule")) {- css::image_urls(&rule, resolver, &mut push);+ css::urls(&rule, resolver, &mut push);}}}for sheet in [replace, replace_sync] {if let Some(sheet) = text(inner, sheet) {- css::image_urls(&sheet, resolver, &mut push);+ css::urls(&sheet, resolver, &mut push);}}}@@ -1759,7 +1899,7 @@ fn style_sheet_urls(span: rules,}) {if let Some(rule) = text(inner, field(inner, rule, b"rule")) {- css::image_urls(&rule, resolver, &mut push);+ css::urls(&rule, resolver, &mut push);}}}@@ -2106,7 +2246,11 @@ mod tests {.into_iter().map(|rewrite| rewrite.text).collect();- let remote = remote.into_urls();+ let remote: Vec<String> = remote+ .into_urls()+ .into_iter()+ .map(|remote| remote.url)+ .collect();for name in ["hidden","display-none",@@ -2179,6 +2323,89 @@ mod tests {assert_eq!(found, expected);}+ #[test]+ fn names_font_face_urls_and_fetches_only_unbundled_loadable_ones() {+ let font = |name: &str| format!("https://fonts.example.com/{name}");+ let face =+ |family: &str, src: &str| format!("@font-face{{font-family:{family};src:{src}}}");+ let style = [+ face(+ "'Brand Sans'",+ &format!(+ "url({}) format('woff2'),url({}) format('embedded-opentype')",+ font("brand.woff2"),+ font("brand.eot?#iefix")+ ),+ ),+ face(+ "\"Open Sans\"",+ &format!("url({})", font("open-sans.woff2")),+ ),+ face("Inter", &format!("url({})", font("shared.woff2"))),+ face("Brand Inter", &format!("url({})", font("shared.woff2"))),+ format!(".logo{{background:url({})}}", font("logo.png")),+ face("Logo", &format!("url({})", font("logo.png"))),+ face("Brand Serif", "url(/fonts/serif.woff2)"),+ face("Roboto", "url(/fonts/roboto.woff2)"),+ ]+ .concat();+ let snapshot = json!({"node": {"type": 0, "childNodes": [element(2, "html", json!({}), json!([+ element(3, "style", json!({}), json!([{"type": 3, "textContent": style, "id": 4}])),+ ]))], "id": 1}});+ let message = events(json!([+ {"type": 2, "timestamp": 1, "data": snapshot},+ {"type": 3, "timestamp": 2, "data": {"source": 8, "id": 3, "adds": [{"rule": face("Icons", &format!("url({})", font("icons.woff"))), "index": 0}]}},+ ]));+ let (mut dom, _) = observe_all(&message, "w");+ let mut fetched = dom.take_font_urls();+ fetched.sort();+ let mut expected = ["brand.woff2", "icons.woff", "logo.png", "shared.woff2"]+ .map(font)+ .to_vec();+ expected.insert(0, "https://app.example.com/fonts/serif.woff2".to_owned());+ assert_eq!(fetched, expected);+ let mut named = dom.take_remote_urls();+ let mut resolved: Vec<RemoteUrl> = named+ .take_css_rewrites()+ .into_iter()+ .filter_map(|rewrite| rewrite.resolved)+ .collect();+ resolved.sort_by(|a, b| a.url.cmp(&b.url));+ assert_eq!(+ resolved,+ [+ RemoteUrl {+ url: "https://app.example.com/fonts/roboto.woff2".to_owned(),+ kind: RemoteKind::BundledFont("roboto"),+ },+ RemoteUrl {+ url: "https://app.example.com/fonts/serif.woff2".to_owned(),+ kind: RemoteKind::Font,+ },+ ]+ );+ let mut remote: Vec<(String, RemoteKind)> = named+ .into_urls()+ .into_iter()+ .map(|remote| (remote.url, remote.kind))+ .collect();+ remote.sort();+ assert_eq!(+ remote,+ [+ (font("brand.eot?#iefix"), RemoteKind::Font),+ (font("brand.woff2"), RemoteKind::Font),+ (font("icons.woff"), RemoteKind::Font),+ (font("logo.png"), RemoteKind::Image),+ (+ font("open-sans.woff2"),+ RemoteKind::BundledFont("open-sans")+ ),+ (font("shared.woff2"), RemoteKind::Font),+ ]+ );+ }+#[test]fn keeps_one_srcset_candidate_of_a_hidden_img() {let srcset = |count: usize| {@@ -2207,7 +2434,7 @@ mod tests {};let limit = |urls: &mut dyn Iterator<Item = String>| {let mut remote = RemoteUrls::default();- urls.for_each(|u| remote.insert(&u));+ urls.for_each(|u| remote.insert(&u, RemoteKind::Image));remote.limit};let longest = 8 * 1024;@@ -2232,6 +2459,16 @@ mod tests {] {assert_eq!(limit(&mut urls.into_iter()), expected);}+ let mut remote = RemoteUrls::default();+ (0..=MAX_BLOCK_FONT_URLS).for_each(|i| remote.insert(&url(i, 40), RemoteKind::Font));+ remote.insert(&url(MAX_BLOCK_FONT_URLS + 1, 40), RemoteKind::Image);+ assert_eq!(remote.limit, None);+ assert_eq!(remote.url_limit(), Some(UrlLimit::FontUrlCount));+ assert_eq!(remote.into_urls().len(), MAX_BLOCK_FONT_URLS + 1);+ let mut remote = RemoteUrls::default();+ remote.insert(&url(0, MAX_BLOCK_FONT_URL_BYTES + 1), RemoteKind::Font);+ assert_eq!(remote.url_limit(), Some(UrlLimit::FontUrlBytes));+// Srcset values share the byte budget, past which the writer falls back to replacing each URL.let mut remote = RemoteUrls::default();// A value counts its raw and its unescaped form.diff --git a/rust/ai-research/replay-core-v4/src/inspect/session.rs b/rust/ai-research/replay-core-v4/src/inspect/session.rsindex 8f2ee7d6dcb5..2670708c2f63 100644--- a/rust/ai-research/replay-core-v4/src/inspect/session.rs+++ b/rust/ai-research/replay-core-v4/src/inspect/session.rs@@ -5,7 +5,7 @@ use std::sync::Arc;use super::clicks::{ClickKind, ClickTracker};use super::css::CssRewrite;-use super::dom::{DomInspector, RemoteUrls, Srcset, UrlLimit};+use super::dom::{DomInspector, RemoteUrl, RemoteUrls, Srcset, UrlLimit};use super::events::{href_from, number, Activity, DataFields, EventCounter};use super::features::{BlockFeatures, FeatureRecorder, SurfacingFeatures};use super::rrweb::{@@ -52,6 +52,7 @@ pub struct BlockInspection {features: FeatureRecorder,moments: Vec<Moment>,image_urls: Vec<String>,+ font_urls: Vec<String>,remote_urls: RemoteUrls,events: u32,resolve_budget: Option<Arc<AtomicUsize>>,@@ -65,8 +66,9 @@ pub struct Inspected {pub moments: Vec<Moment>,/// Remote image URLs the block's DOM referenced, for the image fetcher.pub image_urls: Vec<String>,- /// Every remote URL in an image slot of the block, which the writer replaces with its reference.- pub remote_urls: Vec<String>,+ pub font_urls: Vec<String>,+ /// Every remote URL in an image slot or `@font-face` rule of the block, which the writer replaces with its reference.+ pub remote_urls: Vec<RemoteUrl>,/// Set when `remote_urls` stops short, so that the block must not be stored.pub url_limit: Option<UrlLimit>,/// The image-slot srcset values of the block, which the writer rebuilds from the candidates it keeps.@@ -92,7 +94,8 @@ impl BlockInspection {features,moments: self.moments,image_urls: self.image_urls,- url_limit: self.remote_urls.limit,+ font_urls: self.font_urls,+ url_limit: self.remote_urls.url_limit(),srcsets: self.remote_urls.take_srcsets(),css_rewrites: self.remote_urls.take_css_rewrites(),remote_urls: self.remote_urls.into_urls(),@@ -187,6 +190,7 @@ impl SessionInspector {&mut block.counter.stats,);block.image_urls.extend(self.dom.take_image_urls());+ block.font_urls.extend(self.dom.take_font_urls());block.remote_urls.extend(self.dom.take_remote_urls());if observed.full_snapshot {self.cold = false;diff --git a/rust/ai-research/replay-core-v4/src/lib.rs b/rust/ai-research/replay-core-v4/src/lib.rsindex 07ba84681498..a4dee6991b87 100644--- a/rust/ai-research/replay-core-v4/src/lib.rs+++ b/rust/ai-research/replay-core-v4/src/lib.rs@@ -5,6 +5,7 @@ pub mod compression;pub mod envelope;pub mod fetch_history;pub mod fetched_image;+pub mod fonts;pub mod gate;pub mod inspect;#[cfg(feature = "store")]diff --git a/rust/ai-research/replay-core-v4/src/refs.rs b/rust/ai-research/replay-core-v4/src/refs.rsindex 8e815172bdb0..8f53469944b1 100644--- a/rust/ai-research/replay-core-v4/src/refs.rs+++ b/rust/ai-research/replay-core-v4/src/refs.rs@@ -1,11 +1,16 @@-//! The keyed hash of the image and URL references in `posthog_replay_anonymizer::v4_image_refs`, and the URL-//! fingerprints of the block index.+//! The keyed hash of the image and URL references in `posthog_replay_anonymizer::v4_image_refs`, the asset of a URL reference, and the URL fingerprints of the block index.use base64::engine::general_purpose::URL_SAFE_NO_PAD;use base64::Engine;use hmac::{Hmac, Mac};-use posthog_replay_anonymizer::v4_image_refs::HASH_CHARS;-use sha2::Sha256;+use posthog_replay_anonymizer::v4_image_refs::{+ font_url_declined_marker, font_url_ref, image_url_ref, public_font_ref, url_declined_marker,+ HASH_CHARS,+};+use posthog_replay_anonymizer::Decline;+use sha2::{Digest, Sha256};++use crate::url_policy::CanonicalUrl;/// 64 bits of the HMAC: enough to tell the pages of one session apart.const FINGERPRINT_BYTES: usize = 8;@@ -20,6 +25,48 @@ pub fn reference_hash(key: &[u8; 32], bytes: &[u8]) -> String {encoded}+/// Google serves each file under `/s/` at a versioned path whose bytes never change. A `/l/` file is a subset that a page's `text=` parameter picked, so its glyphs can hold page text, and it stays with its team.+const PUBLIC_FONT_HOST: &str = "fonts.gstatic.com";+const PUBLIC_FONT_PATH_PREFIX: &str = "/s/";++/// The hash has no key because the URL holds no personal data, so anyone can test a reference against a Google Fonts URL.+pub fn public_font_url_ref(url: &CanonicalUrl) -> Option<String> {+ let parsed = url::Url::parse(&url.fetch).ok()?;+ let shared = parsed.scheme() == "https"+ && parsed.host_str() == Some(PUBLIC_FONT_HOST)+ && parsed.port().is_none()+ && parsed.path().starts_with(PUBLIC_FONT_PATH_PREFIX)+ && parsed.query().is_none();+ shared.then(|| public_font_ref(&public_url_hash(&url.fetch)))+}++/// The unkeyed hash of a public font reference, which the fetcher also binds the font's cached validators with.+pub fn public_url_hash(url: &str) -> String {+ let mut encoded = URL_SAFE_NO_PAD.encode(Sha256::digest(url.as_bytes()));+ encoded.truncate(HASH_CHARS);+ encoded+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]+pub enum UrlAsset {+ Image,+ Font,+}++pub fn url_ref(asset: UrlAsset, team_id: i64, key_month: &str, hash: &str) -> String {+ match asset {+ UrlAsset::Image => image_url_ref(team_id, key_month, hash),+ UrlAsset::Font => font_url_ref(team_id, key_month, hash),+ }+}++pub fn declined_url_ref(asset: UrlAsset, reason: Decline) -> String {+ match asset {+ UrlAsset::Image => url_declined_marker(reason),+ UrlAsset::Font => font_url_declined_marker(reason),+ }+}+/// A page URL as the block index shows it in plaintext: lowercase hex of the HMAC under/// [`crate::seal::url_fingerprint_key`].pub fn url_fingerprint(key: &[u8; 32], url: &str) -> String {diff --git a/rust/ai-research/replay-core-v4/src/url_policy.rs b/rust/ai-research/replay-core-v4/src/url_policy.rsindex ba50b1f9500e..69fad8961f2d 100644--- a/rust/ai-research/replay-core-v4/src/url_policy.rs+++ b/rust/ai-research/replay-core-v4/src/url_policy.rs@@ -368,7 +368,7 @@ pub fn is_remote(url: &str) -> bool {}}-/// The media types the image fetcher stores, the `ALLOWED_CONTENT_TYPES` of `image-fetch-v4/image-fetcher.ts`.+/// The media types the image fetcher stores, the `SUPPORTED_IMAGE_MEDIA_TYPES` of `image-fetch-v4/image-transport.ts`.pub const FETCHED_IMAGE_TYPES: [&str; 5] = ["image/png","image/jpeg",diff --git a/rust/ai-research/replay-core-v4/tests/v4_vector.rs b/rust/ai-research/replay-core-v4/tests/v4_vector.rsindex 704569479a7a..e438fb138e13 100644--- a/rust/ai-research/replay-core-v4/tests/v4_vector.rs+++ b/rust/ai-research/replay-core-v4/tests/v4_vector.rs@@ -4,8 +4,8 @@ use posthog_replay_anonymizer::v4_image_refs::{image_ref, image_url_ref};use replay_core_v4::envelope::{open_frame, seal_frame_with_nonce, seal_json_envelope_with_nonce, Codec, Context, Version,};-use replay_core_v4::fetched_image::{fetched_image_context, image_url_hash};-use replay_core_v4::refs::{reference_hash, url_fingerprint};+use replay_core_v4::fetched_image::{fetched_image_context, image_url_hash, public_font_ref_of};+use replay_core_v4::refs::{reference_hash, url_fingerprint, url_ref, UrlAsset};use replay_core_v4::seal::{image_content_key, image_url_key, open_session_key, seal_session_key_with_nonce,url_fingerprint_key, SealedKey, WrappingContext,@@ -171,6 +171,34 @@ fn reproduces_every_pinned_v4_value() {.unwrap();assert_eq!(fetched_frame, bytes(&fetched["frame"]));+ let font = &vector["fontUrl"];+ let font_hash = image_url_hash(&team_month_key, font["url"].as_str().unwrap());+ assert_eq!(+ url_ref(UrlAsset::Font, team_id, key_month, &font_hash),+ font["ref"].as_str().unwrap()+ );+ let font_context = fetched_image_context(font["ref"].as_str().unwrap()).unwrap();+ assert_eq!(font_context, context(&font["context"], Some(Codec::None)));+ let font_frame = seal_frame_with_nonce(+ &team_month_key,+ Version::V4,+ &font_context,+ &bytes(&font["bytes"]),+ nonce(&font["nonce"]),+ )+ .unwrap();+ assert_eq!(font_frame, bytes(&font["frame"]));++ let public_font = &vector["publicFontUrl"];+ let public_ref = |url: &Value| public_font_ref_of(url.as_str().unwrap());+ assert_eq!(+ public_ref(&public_font["url"]),+ str_field(&public_font["ref"])+ );+ for url in public_font["teamUrls"].as_array().unwrap() {+ assert_eq!(public_ref(url), None, "{url}");+ }+let index = &vector["index"];let envelope = seal_json_envelope_with_nonce(&session_key,diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsindex cbae593f95a4..02816142c108 100644--- a/rust/ai-research/replay-recorder-v4/src/batch.rs+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -9,7 +9,7 @@ use futures::stream::{self, StreamExt};use metrics::counter;use replay_core_v4::compression::{GZIP_MAGIC, MAX_DECOMPRESSED_BYTES};use replay_core_v4::inspect::css::CssRewrite;-use replay_core_v4::inspect::dom::{Srcset, UrlLimit};+use replay_core_v4::inspect::dom::{RemoteUrl, Srcset, UrlLimit};use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};use replay_core_v4::inspect::stats::BlockStats;use replay_core_v4::scan::cv::{Expansion, MESSAGE_DECODE_BUDGET};@@ -97,8 +97,9 @@ pub struct Block {pub moments: Vec<Moment>,/// Remote image URLs the block's DOM referenced.pub image_urls: Vec<String>,- /// Every remote URL in an image slot of the block, which the writer replaces with its reference.- pub remote_urls: Vec<String>,+ pub font_urls: Vec<String>,+ /// Every remote URL in an image slot or `@font-face` rule of the block, which the writer replaces with its reference.+ pub remote_urls: Vec<RemoteUrl>,/// Set when the inspector stopped short of naming every remote URL, so that the block must not be stored.pub url_limit: Option<UrlLimit>,/// The image-slot srcset values of the block, which the writer rebuilds from the candidates it keeps.@@ -527,6 +528,7 @@ fn inspect_session(stats: inspected.stats,moments: inspected.moments,image_urls: inspected.image_urls,+ font_urls: inspected.font_urls,remote_urls: inspected.remote_urls,url_limit: inspected.url_limit,srcsets: inspected.srcsets,diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 406220ae3557..3578cedd4ceb 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -1009,6 +1009,7 @@ mod tests {stats: BlockStats::default(),moments: Vec::new(),image_urls: Vec::new(),+ font_urls: Vec::new(),remote_urls: Vec::new(),url_limit: None,srcsets: Vec::new(),diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rsindex 00b88b3e61b6..c7614a2313ae 100644--- a/rust/ai-research/replay-recorder-v4/src/flush.rs+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs@@ -4,7 +4,7 @@//! The index rows go to the index topic, and the Parquet sink writes them to S3. The flush does not wait for their//! delivery: the data is stored by then, and a lost row is counted.-use std::collections::{BTreeMap, BTreeSet, HashMap};+use std::collections::{BTreeSet, HashMap};use std::sync::{Arc, Mutex, OnceLock, PoisonError};use std::time::Duration;@@ -26,7 +26,7 @@ use crate::batch::Block;use crate::history::{mark_fresh, FetchHistory};use crate::index::{Index, IndexMessage};use crate::store::BlobStore;-use crate::urls::{fetch_jobs, records, FlushImages};+use crate::urls::{fetch_jobs, records, FetchJobs, FlushUrls};use crate::write::{block_index_rows, compress_block, new_images, object_key, seal_blocking, seal_compressed,session_keyed_frames, write_blocks, write_images, CompressedBlock, FlushAttempt, WriteError,@@ -175,7 +175,7 @@ pub async fn write_flush(.sent_urls.contains_key(&(day.clone(), reference.to_owned()))};- let images = FlushImages::new(+ let urls = FlushUrls::new(blocks.iter().map(|block| &**block),&keys,month,@@ -212,7 +212,7 @@ pub async fn write_flush(stream::iter(seals).buffer_unordered(concurrency.max(1)).collect::<Vec<_>>(),- mark_fresh_images(writer, &images, &day),+ mark_fresh_urls(writer, &urls, &day),);for result in results {match result {@@ -255,7 +255,7 @@ pub async fn write_flush(kept[index] = true;}let jobs = fetch_jobs(- &images,+ &urls,blocks.iter().zip(stats.iter_mut())@@ -265,7 +265,7 @@ pub async fn write_flush(now.timestamp_millis(),&already_sent,);- drop(images);+ drop(urls);let session_frames = session_keyed_frames(&sealed);let image_rows = write_images(@@ -312,7 +312,7 @@ pub async fn write_flush(Ok(())}-async fn mark_fresh_images(writer: &Writer, images: &FlushImages, day: &str) {+async fn mark_fresh_urls(writer: &Writer, urls: &FlushUrls, day: &str) {let Some(history) = &writer.fetch_history else {return;};@@ -320,7 +320,7 @@ async fn mark_fresh_images(writer: &Writer, images: &FlushImages, day: &str) {mark_fresh(history.as_ref(),&writer.sent_urls,- images.unsent_references(),+ urls.unsent_references(),day,deadline,)@@ -330,12 +330,7 @@ async fn mark_fresh_images(writer: &Writer, images: &FlushImages, day: &str) {/// Sends every fetch job of the flush, then waits for all deliveries before the flush commits, as the v3 mirror does./// A send that fails does not fail the flush, because its blocks are stored by then. The references it did not send/// stay unmarked, so a later flush that sees them sends them again.-async fn produce_jobs(- writer: &Writer,- jobs: BTreeMap<String, Vec<serde_json::Value>>,- day: &str,- now_ms: i64,-) {+async fn produce_jobs(writer: &Writer, jobs: FetchJobs, day: &str, now_ms: i64) {let captured_at = now_ms.to_string();let deliveries: Vec<_> = records(jobs).into_iter()diff --git a/rust/ai-research/replay-recorder-v4/src/rewrite.rs b/rust/ai-research/replay-recorder-v4/src/rewrite.rsindex fba47bb0814f..5110ecaeb8a7 100644--- a/rust/ai-research/replay-recorder-v4/src/rewrite.rs+++ b/rust/ai-research/replay-recorder-v4/src/rewrite.rs@@ -16,12 +16,15 @@ use std::collections::{BTreeMap, HashMap};use aho_corasick::{AhoCorasick, AhoCorasickKind, MatchKind};use metrics::counter;use posthog_replay_anonymizer::v4_image_refs::{- url_declined_marker, ImageDecline, IMAGE_DECLINED_PREFIX,+ bundled_font_ref, ImageDecline, FONT_URL_UNCHECKED_MARKER, IMAGE_DECLINED_PREFIX,};use replay_core_v4::bytescan::{unescape, write_json_string, Span};-use replay_core_v4::inspect::css::{escape_in_identifier, rewrite_functions, CssRewrite};-use replay_core_v4::inspect::dom::{for_each_srcset_candidate, Srcset};+use replay_core_v4::inspect::css::{+ escape_in_identifier, rewrite_font_src_functions, rewrite_functions, CssRewrite,+};+use replay_core_v4::inspect::dom::{for_each_srcset_candidate, RemoteKind, RemoteUrl, Srcset};use replay_core_v4::inspect::srcset::shown_candidate;+use replay_core_v4::refs::{declined_url_ref, UrlAsset};use replay_core_v4::scan::tokens::{Token, Tokens};use replay_core_v4::seal::image_url_key;use replay_core_v4::url_policy::is_remote;@@ -29,6 +32,32 @@ use replay_core_v4::url_policy::is_remote;use crate::scrub::{Scrub, Scrubber};use crate::urls::url_reference;+/// The text that replaces `url` in the block, and the policy's reason when it declines the URL.+fn named_as(+ url: &str,+ kind: RemoteKind,+ team_id: i64,+ key_month: &str,+ url_key: &[u8; 32],+) -> (String, Option<&'static str>) {+ let asset = match kind {+ RemoteKind::Image => UrlAsset::Image,+ RemoteKind::Font | RemoteKind::BundledFont(_) => UrlAsset::Font,+ };+ match (kind, url_reference(url, asset, team_id, key_month, url_key)) {+ // The marker names no URL, but a declined URL keeps its reason, so that a copy inside a longer token is replaced too.+ (RemoteKind::BundledFont(slug), named) => (+ bundled_font_ref(slug),+ named.err().map(|decline| decline.label()),+ ),+ (_, Ok((_, reference))) => (reference, None),+ (_, Err(decline)) => (+ declined_url_ref(asset, decline.into()),+ Some(decline.label()),+ ),+ }+}+/// A byte that a URL can go on with. A match next to one is part of a longer URL, so it stays as it is. A comma, a/// parenthesis or a quote can be in a URL too, but in a srcset or a CSS `url()` it ends one.fn continues_url(byte: u8) -> bool {@@ -197,6 +226,11 @@ impl Iterator for CssStrings<'_> {}}+/// CSS escapes can spell `font-face` in many ways, but never its `@`.+fn holds_at_rule(bytes: &[u8]) -> bool {+ bytes.contains(&b'@')+}+/// The metric label of a rewrite: the inspector rewrites a whole `url()` only to resolve a relative URL.fn css_rewrite_kind(text: &str) -> &'static str {if text@@ -211,6 +245,7 @@ fn css_rewrite_kind(text: &str) -> &'static str {struct Target {url: String,+ kind_label: &'static str,replacement: Vec<u8>,/// The policy's reason, for a declined URL.declined: Option<&'static str>,@@ -241,6 +276,7 @@ pub struct UrlRewrite {url_key: [u8; 32],fallback: Fallback,scrubber: Scrubber,+ font_source_scrubber: Scrubber,scrubbed: u64,/// The JSON key whose value held the first declined URL that stayed, for the log. It names the shape without any URL.kept_in: Option<String>,@@ -253,7 +289,7 @@ pub struct UrlRewrite {pub struct PassCounters {srcset_rewrites: BTreeMap<&'static str, u64>,css_rewrites: BTreeMap<(&'static str, &'static str), u64>,- url_rewrites: BTreeMap<(&'static str, &'static str), u64>,+ url_rewrites: BTreeMap<(&'static str, &'static str, &'static str), u64>,pub removed_images: [u64; ImageDecline::ALL.len()],}@@ -271,9 +307,10 @@ impl PassCounters {).increment(count);}- for (&(outcome, reason), &count) in &self.url_rewrites {+ for (&(kind, outcome, reason), &count) in &self.url_rewrites {counter!("ai_research_v4_recorder_image_url_rewrites_total",+ "kind" => kind,"outcome" => outcome,"reason" => reason,)@@ -292,6 +329,8 @@ pub struct Fallback {/// Replace a declined URL wherever its bytes are, also inside a longer URL that would otherwise keep it.pub overwrite_declined: bool,pub scrub: Scrub,+ /// Replace every remote URL in a `url()` of an `@font-face` `src`, also where `scrub` replaces nothing.+ pub scrub_font_sources: bool,/// The metric label of why the fallback runs.pub reason: Option<&'static str>,}@@ -314,7 +353,7 @@ pub struct RewriteReport {impl UrlRewrite {/// `team_key` is the team's month key for `key_month`, the write month, as for the fetch jobs.pub fn new(- urls: &[String],+ urls: &[RemoteUrl],srcsets: &[Srcset],css_rewrites: &[CssRewrite],team_id: i64,@@ -324,15 +363,18 @@ impl UrlRewrite {let url_key = image_url_key(team_key);let mut patterns = Vec::with_capacity(urls.len() + srcsets.len());let mut targets = Vec::with_capacity(urls.len() + srcsets.len());- for url in urls {+ for RemoteUrl { url, kind } in urls {// The block holds the URL as a JSON string holds it.patterns.push(json_string_contents(url));- let (replacement, declined) = match url_reference(url, team_id, key_month, &url_key) {- Ok((_, reference)) => (reference, None),- Err(decline) => (url_declined_marker(decline.into()), Some(decline.label())),+ let kind_label = match kind {+ RemoteKind::Image => "image",+ RemoteKind::Font => "font",+ RemoteKind::BundledFont(_) => "bundled_font",};+ let (replacement, declined) = named_as(url, *kind, team_id, key_month, &url_key);targets.push(Target {url: url.clone(),+ kind_label,replacement: replacement.into_bytes(),declined,found: Cell::new(false),@@ -341,7 +383,7 @@ impl UrlRewrite {let url_targets: HashMap<String, usize> = urls.iter().enumerate()- .map(|(index, url)| (url.clone(), index))+ .map(|(index, remote)| (remote.url.clone(), index)).collect();let css_matcher = (!css_rewrites.is_empty()).then(|| {@@ -381,12 +423,14 @@ impl UrlRewrite {.map(|rewrite| {let mut covers = Vec::new();let resolved = match &rewrite.resolved {- Some(absolute) => {- let reference = match url_reference(absolute, team_id, key_month, &url_key)- {- Ok((_, reference)) => reference,- Err(decline) => url_declined_marker(decline.into()),- };+ Some(RemoteUrl {+ url: absolute,+ kind,+ }) => {+ let kind = url_targets+ .get(absolute)+ .map_or(*kind, |&index| urls[index].kind.max(*kind));+ let (reference, _) = named_as(absolute, kind, team_id, key_month, &url_key);rewrite.replacement.replacen(&format!("\"{absolute}\""),&format!("\"{reference}\""),@@ -463,6 +507,7 @@ impl UrlRewrite {url_key,fallback: Fallback::default(),scrubber: Scrubber::new(&Scrub::Nothing),+ font_source_scrubber: Scrubber::new(&Scrub::Nothing),scrubbed: 0,kept_in: None,counters: PassCounters::default(),@@ -471,6 +516,11 @@ impl UrlRewrite {pub fn with_fallback(mut self, fallback: Fallback) -> Self {self.scrubber = Scrubber::new(&fallback.scrub);+ self.font_source_scrubber = Scrubber::new(if fallback.scrub_font_sources {+ &Scrub::AllRemote+ } else {+ &fallback.scrub+ });self.fallback = fallback;self}@@ -586,6 +636,7 @@ impl UrlRewrite {} else {replaced};+ let replaced = then_stage!(replaced, self.mark_unchecked_fonts);let (scrubbed, count) = self.scrubber.apply(&replaced);if count == 0 {return replaced;@@ -594,6 +645,44 @@ impl UrlRewrite {Cow::Owned(scrubbed.into_owned())}+ /// Runs before the scrub, which would leave an image marker where the renderer loads a font.+ fn mark_unchecked_fonts<'b>(&mut self, bytes: &'b [u8]) -> Cow<'b, [u8]> {+ if !self.font_source_scrubber.replaces_tokens() || !holds_at_rule(bytes) {+ return Cow::Borrowed(bytes);+ }+ let mut out: Option<Vec<u8>> = None;+ let mut copied_to = 0;+ for string in CssStrings::new(bytes) {+ if !holds_at_rule(&bytes[string.0..string.1]) {+ continue;+ }+ let Ok(css) = unescape(bytes, string) else {+ continue;+ };+ let rewritten = rewrite_font_src_functions(&css, &mut |function| {+ let mut json = Vec::with_capacity(function.len() + 2);+ write_json_string(function, &mut json);+ (self.font_source_scrubber.apply(&json).1 > 0).then(|| {+ self.scrubbed += 1;+ format!("url(\"{FONT_URL_UNCHECKED_MARKER}\")")+ })+ });+ if let Some(rewritten) = rewritten {+ let out = out.get_or_insert_with(|| Vec::with_capacity(bytes.len()));+ out.extend_from_slice(&bytes[copied_to..string.0]);+ write_json_string(&rewritten, out);+ copied_to = string.1;+ }+ }+ match out {+ Some(mut out) => {+ out.extend_from_slice(&bytes[copied_to..]);+ Cow::Owned(out)+ }+ None => Cow::Borrowed(bytes),+ }+ }+fn replace_escaped_urls<'b>(&mut self, bytes: &'b [u8]) -> Cow<'b, [u8]> {let escaped = self.escaped@@ -688,7 +777,11 @@ impl UrlRewrite {};*counters.url_rewrites- .entry((outcome, target.declined.unwrap_or("none")))+ .entry((+ target.kind_label,+ outcome,+ target.declined.unwrap_or("none"),+ )).or_default() += 1;}RewriteReport {@@ -879,7 +972,7 @@ fn rebuilt_srcset(match position {CandidatePosition::Remote(_) => {covers.extend(url_targets.get(url));- match url_reference(url, team_id, key_month, url_key) {+ match url_reference(url, UrlAsset::Image, team_id, key_month, url_key) {Ok((_, reference)) if is_kept => keep(&reference, descriptors),_ => removed = true,}@@ -912,16 +1005,30 @@ mod tests {const TEAM_KEY: [u8; 32] = [6; 32];+ fn images(urls: &[&str]) -> Vec<RemoteUrl> {+ urls.iter()+ .map(|url| RemoteUrl {+ url: (*url).to_owned(),+ kind: RemoteKind::Image,+ })+ .collect()+ }+fn rewrite(urls: &[&str], bytes: &str) -> String {- let urls: Vec<String> = urls.iter().map(|u| (*u).to_owned()).collect();- let mut rewrite = UrlRewrite::new(&urls, &[], &[], 7, "2026-10", &TEAM_KEY);+ let mut rewrite = UrlRewrite::new(&images(urls), &[], &[], 7, "2026-10", &TEAM_KEY);String::from_utf8(rewrite.apply(bytes.as_bytes(), true).into_owned()).unwrap()}fn reference(url: &str) -> String {- url_reference(url, 7, "2026-10", &image_url_key(&TEAM_KEY))- .unwrap()- .1+ url_reference(+ url,+ UrlAsset::Image,+ 7,+ "2026-10",+ &image_url_key(&TEAM_KEY),+ )+ .unwrap()+ .1}#[test]@@ -947,7 +1054,7 @@ mod tests {let declined = "https://10.0.0.7/c.png";let event = format!(r#"{{"srcset":"{a} 1x,\n{b} 2x,\t{declined} 3x"}}"#);let mut rewrite = UrlRewrite::new(- &[a.to_owned(), b.to_owned(), declined.to_owned()],+ &images(&[a, b, declined]),&[],&[],7,@@ -1048,7 +1155,7 @@ mod tests {let url = "https://cdn.example.com/a.png?token=secret";let hidden = r#"{"src":"https:\/\/cdn.example.com\/a.png?token=secr\u0065t","alt":"a\nb"}"#;let canonical = canonical_strings(hidden.as_bytes());- let mut rewrite = UrlRewrite::new(&[url.to_owned()], &[], &[], 7, "2026-10", &TEAM_KEY);+ let mut rewrite = UrlRewrite::new(&images(&[url]), &[], &[], 7, "2026-10", &TEAM_KEY);assert_eq!(String::from_utf8(rewrite.apply(&canonical, true).into_owned()).unwrap(),r#"{"src":"imageurl:v4:declined:credential","alt":"a\nb"}"#@@ -1061,7 +1168,7 @@ mod tests {Cow::Borrowed(_)));// A declined URL the block still holds fails the block.- let rewrite = UrlRewrite::new(&[url.to_owned()], &[], &[], 7, "2026-10", &TEAM_KEY);+ let rewrite = UrlRewrite::new(&images(&[url]), &[], &[], 7, "2026-10", &TEAM_KEY);assert!(!rewrite.finish().safe);}@@ -1088,7 +1195,7 @@ mod tests {}// A second copy inside a longer URL stays as it is, so the block fails.let url = "https://bucket.s3.amazonaws.com/a.png?X-Amz-Signature=abc123";- let mut rewrite = UrlRewrite::new(&[url.to_owned()], &[], &[], 7, "2026-10", &TEAM_KEY);+ let mut rewrite = UrlRewrite::new(&images(&[url]), &[], &[], 7, "2026-10", &TEAM_KEY);rewrite.apply(format!(r#"{{"src":"{url}","alt":"x{url}"}}"#).as_bytes(),true,diff --git a/rust/ai-research/replay-recorder-v4/src/scrub.rs b/rust/ai-research/replay-recorder-v4/src/scrub.rsindex 73077012b356..e7dc4a08a593 100644--- a/rust/ai-research/replay-recorder-v4/src/scrub.rs+++ b/rust/ai-research/replay-recorder-v4/src/scrub.rs@@ -1,6 +1,4 @@-//! The fallback for a block whose remote URLs the writer could not all name or replace. Each remote URL token left in-//! the stored bytes becomes [`URL_UNCHECKED_MARKER`], so no URL the policy would refuse is stored, and the block is-//! kept instead of dropped.+//! The fallback for a block whose remote URLs the writer could not all name or replace. Each remote URL token left in the stored bytes becomes [`URL_UNCHECKED_MARKER`], or `fonturl:v4:declined:unchecked` in the `src` of an `@font-face` rule, so no URL the policy would refuse is stored, and the block is kept instead of dropped.//!//! The scrub reads the bytes one character at a time through their JSON escapes and CSS escapes, so it never splits an//! escape and the stored JSON stays valid. A URL that starts after `(`, `'` or `"` ends at `)`, `'` or `"`, because CSS@@ -38,6 +36,8 @@ enum Kind {StringEnd,/// A character a CSS escape names. It is part of the URL whatever it is.CssEscaped,+ /// An escaped line break, which a CSS string drops, so it is no character of the URL.+ Continuation,}/// One character of the stored bytes and the bytes that spell it.@@ -104,6 +104,17 @@ fn json_unit(bytes: &[u8], at: usize) -> (u32, usize) {}}+/// The position past the `\n` of a `\r\n` whose `\r` ends at `after`, because CSS reads the pair as one line break.+fn past_crlf(bytes: &[u8], after: usize, ch: u32) -> usize {+ if is_char(ch, b'\r') && after < bytes.len() {+ let (next, next_len) = json_unit(bytes, after);+ if is_char(next, b'\n') {+ return after + next_len;+ }+ }+ after+}+/// The character at `at` after its JSON escape and a CSS escape inside it. A CSS hex escape takes the one whitespace/// after it, as CSS reads it.fn unit(bytes: &[u8], at: usize) -> Unit {@@ -138,7 +149,7 @@ fn unit(bytes: &[u8], at: usize) -> Unit {if end < bytes.len() {let (next, next_len) = json_unit(bytes, end);if is_space(next) {- end += next_len;+ end = past_crlf(bytes, end + next_len, next);}}return Unit {@@ -151,6 +162,14 @@ fn unit(bytes: &[u8], at: usize) -> Unit {return plain;}let (next, next_len) = json_unit(bytes, end);+ if is_char(next, b'\n') || is_char(next, b'\r') || is_char(next, 0x0c) {+ end = past_crlf(bytes, end + next_len, next);+ return Unit {+ ch: next,+ len: end - at,+ kind: Kind::Continuation,+ };+ }if is_space(next) || (next_len == 1 && is_char(next, b'"')) {return plain;}@@ -174,8 +193,11 @@ fn remote_start(bytes: &[u8], at: usize) -> Option<Start> {if next.kind == Kind::StringEnd {break;}- chars[count] = lower(next.ch);pos += next.len;+ if next.kind == Kind::Continuation {+ continue;+ }+ chars[count] = lower(next.ch);count += 1;}let slash = |ch: u32| is_char(ch, b'/') || is_char(ch, b'\\');@@ -223,7 +245,7 @@ fn closer_after(prev: Option<Unit>) -> Closer {fn ends_url(next: Unit, closer: Closer) -> bool {match next.kind {Kind::StringEnd => true,- Kind::CssEscaped => false,+ Kind::CssEscaped | Kind::Continuation => false,Kind::Plain => match closer {Closer::Whitespace => is_space(next.ch),Closer::Nothing => false,@@ -259,6 +281,10 @@ fn scan(bytes: &[u8]) -> Scan {None => string_start = Some(at + 1),}}+ if current.kind == Kind::Continuation {+ at += current.len;+ continue;+ }let first = lower(current.ch);let starts = string_start.is_some()&& current.kind != Kind::StringEnd@@ -298,7 +324,9 @@ fn decoded(bytes: &[u8], (start, end): (usize, usize)) -> String {let mut at = start;while at < end {let next = unit(bytes, at);- text.extend(char::from_u32(lower(next.ch)));+ if next.kind != Kind::Continuation {+ text.extend(char::from_u32(lower(next.ch)));+ }at += next.len;}text@@ -365,6 +393,10 @@ impl Scrubber {}}+ pub fn replaces_tokens(&self) -> bool {+ self.all_remote || self.hosts.is_some()+ }+/// The bytes with each URL token replaced, and how many were.pub fn apply<'b>(&self, bytes: &'b [u8]) -> (Cow<'b, [u8]>, u64) {let spans = if self.all_remote {@@ -487,6 +519,19 @@ mod tests {r#"{"style":"url(\\2f\\2f bucket.example.com/a.png?sig=x)"}"#.to_owned(),format!(r#"{{"style":"url({m})"}}"#),),+ // A CSS string drops an escaped line break, so a browser reads a URL that one splits whole.+ (+ r#"{"style":"url(\"htt\\\nps://cdn.example.com/a.png?token=x\")"}"#.to_owned(),+ format!(r#"{{"style":"url(\"{m}\")"}}"#),+ ),+ (+ r#"{"style":"url('ht\\\r\ntps://cdn.example.com/a.png?token=x')"}"#.to_owned(),+ format!(r#"{{"style":"url('{m}')"}}"#),+ ),+ (+ r#"{"style":"url(\"\\68\r\nttps://cdn.example.com/a.png?token=x\")"}"#.to_owned(),+ format!(r#"{{"style":"url(\"{m}\")"}}"#),+ ),// A browser reads a backslash as a slash, and the inspector reads any host after two of them as remote.(r#"{"src":"\\\\10.0.0.7/a.png"}"#.to_owned(),diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rsindex 94e01032560b..f5f1cb549c8b 100644--- a/rust/ai-research/replay-recorder-v4/src/urls.rs+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs@@ -1,12 +1,11 @@-//! Fetch jobs for the remote images a flush's blocks referenced, in the v3 fetch topic's job shape.+//! Fetch jobs for the remote images and fonts a flush's blocks referenced, in the v3 fetch topic's job shape.use std::collections::{BTreeMap, HashMap};use std::time::Duration;use metrics::counter;-use posthog_replay_anonymizer::v4_image_refs::image_url_ref;use replay_core_v4::keys::batch::{FlushKeys, SessionId};-use replay_core_v4::refs::reference_hash;+use replay_core_v4::refs::{public_font_url_ref, reference_hash, url_ref, UrlAsset};use replay_core_v4::seal::image_url_key;use replay_core_v4::url_policy::{is_public_host, is_tracking_beacon, try_canonicalize, CanonicalUrl, Decline,@@ -23,20 +22,23 @@ const MAX_RECORD_BYTES: usize = 512 * 1024;/// How many redirects a job may follow, as in v3.const REMAINING_HOPS: u32 = 10;-/// The remote image URLs of a flush's blocks that the URL policy accepts. The flush holds these until it builds its-/// jobs, so a reference is held once, and only while this process had not sent it today when the flush started.-pub struct FlushImages {- unsent: Vec<(CanonicalUrl, String)>,- blocks: Vec<BlockImages>,+/// Grouped by asset as well as by domain, the topic's key, because a fetcher that cannot parse one reference dead-letters the whole record.+pub type FetchJobs = BTreeMap<(String, UrlAsset), Vec<Value>>;++/// The remote image and font URLs of a flush's blocks that the URL policy accepts. The flush holds these until it builds+/// its jobs, so a reference is held once, and only while this process had not sent it today when the flush started.+pub struct FlushUrls {+ unsent: Vec<(CanonicalUrl, UrlAsset, String)>,+ blocks: Vec<BlockUrls>,}#[derive(Default)]-struct BlockImages {- accepted: u32,+struct BlockUrls {+ accepted_images: u32,unsent: Vec<usize>,}-impl FlushImages {+impl FlushUrls {pub fn new<'b>(blocks: impl IntoIterator<Item = &'b Block>,keys: &FlushKeys,@@ -48,31 +50,37 @@ impl FlushImages {let blocks = blocks.into_iter().map(|block| {- let mut images = BlockImages::default();- for (url, reference) in remote_images(block, keys, key_month) {- images.accepted += 1;+ let mut urls = BlockUrls::default();+ for (url, asset, reference) in remote_urls(block, keys, key_month) {+ if asset == UrlAsset::Image {+ urls.accepted_images += 1;+ }if already_sent(&reference) {continue;}let position = *positions.entry(reference.clone()).or_insert_with(|| {- unsent.push((url, reference));+ unsent.push((url, asset, reference));unsent.len() - 1});- images.unsent.push(position);+ urls.unsent.push(position);}- images+ urls}).collect();Self { unsent, blocks }}pub fn unsent_references(&self) -> impl Iterator<Item = &String> {- self.unsent.iter().map(|(_, reference)| reference)+ self.unsent.iter().map(|(_, _, reference)| reference)}}/// Empty when the flush has no keys for the block's session.-fn remote_images(block: &Block, keys: &FlushKeys, key_month: &str) -> Vec<(CanonicalUrl, String)> {+fn remote_urls(+ block: &Block,+ keys: &FlushKeys,+ key_month: &str,+) -> Vec<(CanonicalUrl, UrlAsset, String)> {let id = SessionId {team_id: block.session.team_id,session_id: block.session.session_id.clone(),@@ -84,44 +92,51 @@ fn remote_images(block: &Block, keys: &FlushKeys, key_month: &str) -> Vec<(Canonreturn Vec::new();};let url_key = image_url_key(team_key);- block- .image_urls- .iter()- .filter_map(|raw| url_reference(raw, block.session.team_id, key_month, &url_key).ok())+ let images = block.image_urls.iter().map(|url| (url, UrlAsset::Image));+ let fonts = block.font_urls.iter().map(|url| (url, UrlAsset::Font));+ images+ .chain(fonts)+ .filter_map(|(raw, asset)| {+ let (url, reference) =+ url_reference(raw, asset, block.session.team_id, key_month, &url_key).ok()?;+ Some((url, asset, reference))+ }).collect()}-/// Builds one job per new remote image URL, grouped by registrable domain, the topic's key. Counts the URLs of each-/// block in the stats paired with it. `index` is the block's position in the blocks that `images` was built from.-/// `already_sent` names references this process sent or found fresh today.+/// Builds one job per new remote image or font URL. Counts the image URLs of each block in the stats paired with it.+/// `index` is the block's position in the blocks that `urls` was built from. `already_sent` names references this+/// process sent or found fresh today.pub fn fetch_jobs<'b>(- images: &FlushImages,+ urls: &FlushUrls,blocks: impl IntoIterator<Item = (usize, &'b Block, &'b mut BlockStats)>,now_ms: i64,already_sent: &dyn Fn(&str) -> bool,-) -> BTreeMap<String, Vec<Value>> {- let mut jobs: BTreeMap<String, Vec<Value>> = BTreeMap::new();- let mut seen = vec![false; images.unsent.len()];+) -> FetchJobs {+ let mut jobs = FetchJobs::new();+ let mut seen = vec![false; urls.unsent.len()];for (index, block, stats) in blocks {- let block_images = &images.blocks[index];- stats.image_urls += block_images.accepted;- for &position in &block_images.unsent {- let (url, reference) = &images.unsent[position];+ let block_urls = &urls.blocks[index];+ stats.image_urls += block_urls.accepted_images;+ for &position in &block_urls.unsent {+ let (url, asset, reference) = &urls.unsent[position];if seen[position] || already_sent(reference) {continue;}seen[position] = true;- jobs.entry(url.domain.clone()).or_default().push(json!({- "sessionId": block.session.session_id,- "originalRef": reference,- "currentUrl": url.fetch,- "remainingHops": REMAINING_HOPS,- "notBeforeMs": 0,- "firstSeenAtMs": now_ms,- "fetchCount": 0,- "republishCount": 0,- "lastRepublishReason": null,- }));+ jobs.entry((url.domain.clone(), *asset))+ .or_default()+ .push(json!({+ "sessionId": block.session.session_id,+ "originalRef": reference,+ "currentUrl": url.fetch,+ "remainingHops": REMAINING_HOPS,+ "notBeforeMs": 0,+ "firstSeenAtMs": now_ms,+ "fetchCount": 0,+ "republishCount": 0,+ "lastRepublishReason": null,+ }));}}jobs@@ -131,6 +146,7 @@ pub fn fetch_jobs<'b>(/// the URL. `url_key` is the team's URL key for `key_month`.pub fn url_reference(raw: &str,+ asset: UrlAsset,team_id: i64,key_month: &str,url_key: &[u8; 32],@@ -145,21 +161,28 @@ pub fn url_reference(if is_tracking_beacon(&url.host, &path) {return Err(Decline::TrackingBeacon);}- let reference = image_url_ref(- team_id,- key_month,- &reference_hash(url_key, url.dedup.as_bytes()),- );+ let public = match asset {+ UrlAsset::Font => public_font_url_ref(&url),+ UrlAsset::Image => None,+ };+ let reference = public.unwrap_or_else(|| {+ url_ref(+ asset,+ team_id,+ key_month,+ &reference_hash(url_key, url.dedup.as_bytes()),+ )+ });Ok((url, reference))}-/// Splits each domain's jobs into records the fetcher accepts. Returns (key, payload, references) per record.+/// Splits each group of jobs into records the fetcher accepts. Returns (domain, payload, references) per record./// The fetcher rejects a whole record whose value exceeds its byte limit, so a job too large for any record is dropped.-pub fn records(jobs: BTreeMap<String, Vec<Value>>) -> Vec<(String, Vec<u8>, Vec<String>)> {+pub fn records(jobs: FetchJobs) -> Vec<(String, Vec<u8>, Vec<String>)> {let empty_bytes = payload(Vec::new()).len();let mut records = Vec::new();let mut oversized = 0u64;- for (domain, domain_jobs) in jobs {+ for ((domain, _), domain_jobs) in jobs {let mut batch: Vec<Value> = Vec::new();let mut bytes = empty_bytes;for job in domain_jobs {@@ -252,6 +275,7 @@ mod tests {stats: BlockStats::default(),moments: Vec::new(),image_urls: urls.iter().map(|u| (*u).to_owned()).collect(),+ font_urls: Vec::new(),remote_urls: Vec::new(),url_limit: None,srcsets: Vec::new(),@@ -300,14 +324,14 @@ mod tests {.dedup.as_bytes(),);- let sent = image_url_ref(7, "2026-10", &sent_hash);+ let sent = url_ref(UrlAsset::Image, 7, "2026-10", &sent_hash);let already_sent = |reference: &str| reference == sent;- let images = FlushImages::new([&block], &keys(&["s"]), "2026-10", &already_sent);- let jobs = fetch_jobs(&images, [(0, &block, &mut stats)], 1_000, &already_sent);- let domains: Vec<_> = jobs.keys().cloned().collect();- assert_eq!(domains, ["example.com"]);- let fetched: Vec<_> = jobs["example.com"]+ let urls = FlushUrls::new([&block], &keys(&["s"]), "2026-10", &already_sent);+ let jobs = fetch_jobs(&urls, [(0, &block, &mut stats)], 1_000, &already_sent);+ let images = ("example.com".to_owned(), UrlAsset::Image);+ assert_eq!(jobs.keys().collect::<Vec<_>>(), [&images]);+ let fetched: Vec<_> = jobs[&images].iter().map(|j| j["currentUrl"].as_str().unwrap().to_owned()).collect();@@ -318,7 +342,7 @@ mod tests {"https://img2.cdn.example.com/b.png"]);- let job = &jobs["example.com"][0];+ let job = &jobs[&images][0];assert!(job["originalRef"].as_str().unwrap()@@ -331,6 +355,10 @@ mod tests {assert_eq!(stats.image_urls, 4);}+ fn jobs_of(asset: UrlAsset, jobs: Vec<Value>) -> FetchJobs {+ BTreeMap::from([(("example.com".to_owned(), asset), jobs)])+ }+#[test]fn holds_each_unsent_reference_once_and_sends_it_for_the_first_kept_block() {let shared = "https://cdn.example.com/shared.png";@@ -342,23 +370,23 @@ mod tests {),block("second", &[shared]),];- let images = FlushImages::new(+ let urls = FlushUrls::new(&blocks,&keys(&["dropped", "first", "second"]),"2026-10",&|_| false,);- assert_eq!(images.unsent_references().count(), 2);+ assert_eq!(urls.unsent_references().count(), 2);let mut stats = [BlockStats::default(), BlockStats::default()];let [first_stats, second_stats] = &mut stats;let jobs = fetch_jobs(- &images,+ &urls,[(1, &blocks[1], first_stats), (2, &blocks[2], second_stats)],1_000,&|_| false,);- let sessions: Vec<_> = jobs["example.com"]+ let sessions: Vec<_> = jobs[&("example.com".to_owned(), UrlAsset::Image)].iter().map(|job| job["sessionId"].as_str().unwrap()).collect();@@ -367,18 +395,25 @@ mod tests {}#[test]- fn splits_a_domains_jobs_at_the_fetchers_record_limit() {- let jobs = BTreeMap::from([(- "example.com".to_owned(),+ fn splits_a_domains_jobs_at_the_fetchers_record_limit_and_by_asset() {+ let mut jobs = jobs_of(+ UrlAsset::Image,(0..2500).map(|i| json!({"originalRef": format!("r{i}")})).collect(),- )]);- let sizes: Vec<_> = records(jobs)+ );+ jobs.extend(jobs_of(+ UrlAsset::Font,+ vec![json!({"originalRef": "font"})],+ ));+ let records = records(jobs);+ let sizes: Vec<_> = records.iter().map(|(_, _, references)| references.len()).collect();- assert_eq!(sizes, [1000, 1000, 500]);+ assert_eq!(sizes, [1000, 1000, 500, 1]);+ assert_eq!(records[3].2, ["font"]);+ assert!(records.iter().all(|(domain, _, _)| domain == "example.com"));}fn padded(reference: &str, pad: usize) -> Value {@@ -392,10 +427,10 @@ mod tests {// Two jobs and a comma that fill the record value to the exact limit.let first = MAX_RECORD_BYTES - empty - 2 * bare - 1 - 10;for (extra, expected) in [(0, vec![2]), (1, vec![1, 1])] {- let jobs = BTreeMap::from([(- "example.com".to_owned(),+ let jobs = jobs_of(+ UrlAsset::Image,vec![padded("a", first), padded("b", 10 + extra)],- )]);+ );let records = records(jobs);let sizes: Vec<_> = records.iter().map(|(_, _, r)| r.len()).collect();assert_eq!(sizes, expected, "extra {extra}");@@ -409,14 +444,14 @@ mod tests {#[test]fn drops_only_a_job_too_large_for_any_record() {let room = MAX_RECORD_BYTES - payload(Vec::new()).len() - padded("a", 0).to_string().len();- let jobs = BTreeMap::from([(- "example.com".to_owned(),+ let jobs = jobs_of(+ UrlAsset::Image,vec![padded("fits", 0),padded("a", room + 1),padded("full", room - 3),],- )]);+ );let records = records(jobs);let references: Vec<_> = records.iter().map(|(_, _, r)| r.clone()).collect();assert_eq!(diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rsindex 1f2288912ce2..0c0c33ff0633 100644--- a/rust/ai-research/replay-recorder-v4/src/write.rs+++ b/rust/ai-research/replay-recorder-v4/src/write.rs@@ -192,9 +192,17 @@ fn prepare_block(/// block is kept without a URL nobody checked.fn first_fallback(block: &Block) -> Fallback {match block.url_limit {+ // Past the font budget the inspector still named every URL it reads except font sources, so links and text keep theirs.+ Some(limit) if limit.is_font_budget() => Fallback {+ overwrite_declined: true,+ scrub: Scrub::Nothing,+ scrub_font_sources: true,+ reason: Some(limit.label()),+ },Some(limit) => Fallback {overwrite_declined: true,scrub: Scrub::AllRemote,+ scrub_font_sources: false,reason: Some(limit.label()),},None => Fallback::default(),@@ -217,19 +225,27 @@ fn second_fallback(first: &Fallback, report: &RewriteReport) -> Fallback {Fallback {overwrite_declined: true,scrub,+ scrub_font_sources: first.scrub_font_sources,reason: first.reason.or(Some(reason)),}}/// Writes a block's lines to an output from `open`. A block that the first pass leaves unsafe is written again to a-/// fresh output with the fallback that makes it safe, so a block is dropped only when even that fails. Moves the-/// images out and returns them with the output.+/// fresh output with the fallback that makes it safe, so a block is dropped only when even that fails, or when its+/// limit leaves nothing a fallback can make safe. Moves the images out and returns them with the output.pub fn write_block<W: Write>(block: &Block,team_key: &[u8; 32],key_month: &str,mut open: impl FnMut() -> io::Result<W>,) -> io::Result<(W, Vec<Image>, WriterCounts)> {+ if let Some(limit) = block.url_limit.filter(|limit| limit.drops_block()) {+ let counts = WriterCounts {+ dropped: Some(limit.label()),+ ..WriterCounts::default()+ };+ return Ok((open()?, Vec::new(), counts));+ }let first = first_fallback(block);let mut out = open()?;let (images, mut counts, report) = prepare_block(block, team_key, key_month, &first, &mut out)?;@@ -716,7 +732,10 @@ mod tests {use super::*;use crate::batch::{Origin, Tp};- use replay_core_v4::inspect::dom::MAX_BLOCK_REMOTE_URLS;+ use replay_core_v4::inspect::dom::{+ RemoteKind, RemoteUrl, MAX_BLOCK_FONT_URLS, MAX_BLOCK_REMOTE_URLS,+ };+ use replay_core_v4::refs::UrlAsset;#[derive(Default)]struct MemoryStore {@@ -784,6 +803,7 @@ mod tests {stats: BlockStats::default(),moments: Vec::new(),image_urls: Vec::new(),+ font_urls: Vec::new(),remote_urls: Vec::new(),url_limit: None,srcsets: Vec::new(),@@ -807,7 +827,15 @@ mod tests {stored(block_of("kept", events), extra)}- fn stored(mut inspected: Block, extra: &[&str]) -> String {+ fn stored(inspected: Block, extra: &[&str]) -> String {+ let (lines, counts) = written(inspected, extra);+ if counts.dropped.is_some() {+ return String::new();+ }+ lines+ }++ fn written(mut inspected: Block, extra: &[&str]) -> (String, WriterCounts) {let mut inspector = replay_core_v4::inspect::session::SessionInspector::default();let mut inspection = replay_core_v4::inspect::session::BlockInspection::default();for message in &inspected.messages {@@ -820,13 +848,13 @@ mod tests {inspected.css_rewrites = found.css_rewrites;inspected.remote_urls- .extend(extra.iter().map(|url| (*url).to_owned()));+ .extend(extra.iter().map(|url| RemoteUrl {+ url: (*url).to_owned(),+ kind: RemoteKind::Image,+ }));let (lines, _, counts) =write_block(&inspected, &[6; 32], "2026-10", || Ok(Vec::new())).unwrap();- if counts.dropped.is_some() {- return String::new();- }- String::from_utf8(lines).unwrap()+ (String::from_utf8(lines).unwrap(), counts)}#[test]@@ -842,6 +870,7 @@ mod tests {let lines = stored_lines(&events, &[]);let reference = crate::urls::url_reference(public,+ replay_core_v4::refs::UrlAsset::Image,7,"2026-10",&replay_core_v4::seal::image_url_key(&[6; 32]),@@ -914,36 +943,119 @@ mod tests {"{lines}");assert!(!stored_lines(&event, &[public]).is_empty());- // A declined URL that a longer URL also holds is overwritten there too, and the block is kept.- let events = json!([{"type": 3, "timestamp": 1, "data": {"source": 0, "adds": [], "removes": [], "texts": [],- "attributes": [{"id": 99, "attributes": {"src": signed, "alt": format!("{signed}&size=2")}}]}}])- .to_string();- let lines = stored_lines(&events, &[]);- assert!(- !lines.is_empty() && !lines.contains("X-Amz-Signature"),- "{lines}"- );+ // A declined URL that a longer URL also holds is overwritten there too, and the block is kept, also for a bundled font, which names no URL.+ let bundled_font = json!({"parentId": 1, "node": {"type": 2, "tagName": "style",+ "attributes": {"_cssText": format!("@font-face{{font-family:Roboto;src:url({signed})}}")}, "childNodes": [], "id": 5}});+ for (adds, attributes) in [+ (+ json!([]),+ json!({"src": signed, "alt": format!("{signed}&size=2")}),+ ),+ (+ json!([bundled_font]),+ json!({"alt": format!("{signed}&size=2")}),+ ),+ ] {+ let events = json!([{"type": 3, "timestamp": 1, "data": {"source": 0, "adds": adds, "removes": [], "texts": [],+ "attributes": [{"id": 99, "attributes": attributes}]}}])+ .to_string();+ let lines = stored_lines(&events, &[]);+ assert!(+ !lines.is_empty() && !lines.contains("X-Amz-Signature"),+ "{lines}"+ );+ }}#[test]fn keeps_a_block_past_the_url_limit_without_a_url_it_did_not_check() {- let urls = MAX_BLOCK_REMOTE_URLS + 1;- let nodes: Vec<_> = (0..urls)+ let images: Vec<_> = (0..=MAX_BLOCK_REMOTE_URLS).map(|i| json!({"type": 2, "tagName": "img", "attributes": {"src": format!("https://cdn.example.com/{i}.png")}, "childNodes": [], "id": i + 2})).collect();- let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"type": 0, "id": 1, "childNodes": nodes}}}]).to_string();- let lines = stored_lines(&events, &[]);- // The inspector named the first URLs, which keep their references; the fallback scrubbed the rest.- assert!(- lines.contains("imageurl:v4:7:2026-10:"),- "named URLs keep their references"+ let faces: String = (0..=MAX_BLOCK_FONT_URLS)+ .map(|i| format!("@font-face{{font-family:Brand{i};src:url(https://fonts.example.com/{i}.woff2)}}"))+ .collect();+ let link = "https://app.example.com/pricing";+ let fonts = vec![+ json!({"type": 2, "tagName": "style", "attributes": {"_cssText": faces}, "childNodes": [], "id": 2}),+ json!({"type": 2, "tagName": "a", "attributes": {"href": link}, "childNodes": [], "id": 3}),+ json!({"type": 2, "tagName": "style", "attributes": {"_cssText": "@\\66 ont-face{font-family:Escaped;src:url(https://fonts.example.com/escaped.woff2)}"}, "childNodes": [], "id": 4}),+ ];+ // Each escaped URL takes a rewrite, so past the rewrite budget its URL is not named, even where a line continuation splits its scheme.+ let escaped_faces: String = (0..=MAX_BLOCK_REMOTE_URLS)+ .map(|i| format!("@font-face{{font-family:Escaped{i};src:url(https://fonts.example.com/{i}\\2e woff2)}}"))+ .collect();+ let split_face = "@font-face{font-family:Split;src:url(\"htt\\\nps://fonts.example.com/split.woff2?token=secret\")}";+ let escaped_fonts = vec![+ json!({"type": 2, "tagName": "style", "attributes": {"_cssText": escaped_faces}, "childNodes": [], "id": 2}),+ json!({"type": 2, "tagName": "a", "attributes": {"href": link}, "childNodes": [], "id": 3}),+ json!({"type": 2, "tagName": "style", "attributes": {"_cssText": split_face}, "childNodes": [], "id": 4}),+ ];+ use posthog_replay_anonymizer::v4_image_refs::{+ FONT_URL_UNCHECKED_MARKER, URL_UNCHECKED_MARKER,+ };+ for (nodes, named, marker, other_marker) in [+ (+ images,+ "imageurl:v4:7:2026-10:",+ URL_UNCHECKED_MARKER,+ FONT_URL_UNCHECKED_MARKER,+ ),+ (+ fonts,+ "fonturl:v4:7:2026-10:",+ FONT_URL_UNCHECKED_MARKER,+ URL_UNCHECKED_MARKER,+ ),+ (+ escaped_fonts,+ "fonturl:v4:7:2026-10:",+ FONT_URL_UNCHECKED_MARKER,+ URL_UNCHECKED_MARKER,+ ),+ ] {+ let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"type": 0, "id": 1, "childNodes": nodes}}}]).to_string();+ let lines = stored_lines(&events, &[]);+ assert!(lines.contains(named), "named URLs keep their references");+ assert!(+ lines.contains(marker) && !lines.contains(other_marker),+ "{marker}"+ );+ for raw in ["https://cdn.", "https://fonts.", "token=secret"] {+ assert!(!lines.contains(raw), "{raw}");+ }+ assert_eq!(+ lines.contains(link),+ marker == FONT_URL_UNCHECKED_MARKER,+ "only the font budget keeps links"+ );+ }+ }++ #[test]+ fn drops_a_block_whose_css_nests_too_deep_to_read() {+ let deep_face = format!(+ "{}@font-face{{font-family:Deep;src:url(\"https://fonts.example.com/deep.woff2?token=secret\")}}{}",+ "@media all{".repeat(80),+ "}".repeat(80)+ );+ let deep_value = format!(+ "background-image:{}url('https://cdn.example.com/deep.png?token=secret'){}",+ "var(--a,".repeat(80),+ ")".repeat(80));- assert!(lines.contains("imageurl:v4:declined:unchecked"));- assert!(!lines.contains("https://"));+ for node in [+ json!({"type": 2, "tagName": "style", "attributes": {"_cssText": deep_face}, "childNodes": [], "id": 2}),+ json!({"type": 2, "tagName": "div", "attributes": {"style": deep_value}, "childNodes": [], "id": 2}),+ ] {+ let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"type": 0, "id": 1, "childNodes": [node]}}}]).to_string();+ let (_, counts) = written(block_of("kept", &events), &[]);+ assert_eq!(counts.dropped, Some("too_many_nested_blocks"));+ }}#[test]- fn leaves_css_only_images_an_offline_render_can_load() {+ fn leaves_css_only_images_and_fonts_an_offline_render_can_load() {let compressed: String = replay_core_v4::compression::gzip(json!([{"rule": ".a{background:url(/img/a.png)} .q{content:\"url(/img/a.png)\"} .b{background-image:image-set(url(https://cdn.example.com/1x.png) 1x, url(https://cdn.example.com/2x.png) 2x)} .c{background:url(/c.png?X-Amz-Signature=abc123)} .bg-\\[url\\(https\\:\\/\\/cdn\\.example\\.com\\/t\\.png\\)\\]{background-image:url(https://cdn.example.com/t.png)} .s-\\[url\\(https\\:\\/\\/cdn\\.example\\.com\\/s\\.png\\?X-Amz-Signature\\=abc123\\)\\]{background-image:url(https://cdn.example.com/s.png?X-Amz-Signature=abc123)}", "index": 0}]).to_string()@@ -958,11 +1070,13 @@ mod tests {{"type": 3, "timestamp": 2, "cv": "2024-10", "data": {"source": 8, "id": 3, "adds": compressed}},{"type": 3, "timestamp": 3, "data": {"source": 0, "adds": [], "removes": [], "texts": [], "attributes": [{"id": 9, "attributes": {"class": "bg-[url(/img/a.png)]"}}, {"id": 10, "attributes": {"class": "bg-[url(https://cdn.example.com/t.png)]"}}]}},{"type": 3, "timestamp": 4, "data": {"source": 8, "id": 3, "replaceSync": ".\\/\\/cdn\\.example\\.com\\/p\\.png\\?X-Amz-Signature\\=abc123{background:url(//cdn.example.com/p.png?X-Amz-Signature=abc123)}"}},+ {"type": 3, "timestamp": 5, "data": {"source": 8, "id": 4, "replaceSync": "@font-face{font-family:Brand;src:url(/fonts/brand.woff2)}@font-face{font-family:Roboto;src:url(/fonts/roboto.woff2)}@font-face{font-family:Escaped;src:url(https://fonts.example.com/escaped\\2e woff2)}"}},]);let lines = stored_lines(&events.to_string(), &[]);let reference = |url: &str| {crate::urls::url_reference(url,+ UrlAsset::Image,7,"2026-10",&replay_core_v4::seal::image_url_key(&[6; 32]),@@ -980,6 +1094,31 @@ mod tests {reference("https://cdn.example.com/2x.png")),r#"url(\"imageurl:v4:declined:credential\")"#.to_owned(),+ format!(+ r#"src:url(\"{}\")"#,+ crate::urls::url_reference(+ "https://app.example.com/fonts/brand.woff2",+ UrlAsset::Font,+ 7,+ "2026-10",+ &replay_core_v4::seal::image_url_key(&[6; 32]),+ )+ .unwrap()+ .1+ ),+ r#"src:url(\"fonturl:v4:bundled:roboto\")"#.to_owned(),+ format!(+ r#"src:url(\"{}\")"#,+ crate::urls::url_reference(+ "https://fonts.example.com/escaped.woff2",+ UrlAsset::Font,+ 7,+ "2026-10",+ &replay_core_v4::seal::image_url_key(&[6; 32]),+ )+ .unwrap()+ .1+ ),] {assert!(lines.contains(&expected), "{expected} in {lines}");}@@ -996,7 +1135,14 @@ mod tests {] {assert!(lines.contains(&expected), "{expected} in {lines}");}- for gone in [":url(/img/a.png)", "1x.png", "1x,", "Signature"] {+ for gone in [+ ":url(/img/a.png)",+ "1x.png",+ "1x,",+ "Signature",+ "/fonts/",+ "fonts.example.com",+ ] {assert!(!lines.contains(gone), "{gone} in {lines}");}}@@ -1115,6 +1261,7 @@ mod tests {let reference = |url: &str| {crate::urls::url_reference(url,+ replay_core_v4::refs::UrlAsset::Image,7,"2026-10",&replay_core_v4::seal::image_url_key(&[6; 32]),@@ -1196,6 +1343,7 @@ mod tests {let lines = stored_lines(&both, &[]);let reference = crate::urls::url_reference("https://cdn.example.com/a.png",+ replay_core_v4::refs::UrlAsset::Image,7,"2026-10",&replay_core_v4::seal::image_url_key(&[6; 32]),@@ -1249,27 +1397,64 @@ mod tests {"https://cdn.example.com/a.png","https://cdn.example.com/a.png?token=secret",);- let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": fetched, "srcset": format!("{declined} 2x")}}}}]);+ let (font, bundled, public) = (+ "https://fonts.example.com/brand.woff2",+ "https://fonts.example.com/inter.woff2",+ "https://fonts.gstatic.com/s/roboto/v30/KFOmCnqEu92Fr1Mu4mxK.woff2",+ );+ let css = format!("@font-face{{font-family:Brand;src:url({font})}}@font-face{{font-family:Inter;src:url({bundled})}}@font-face{{font-family:Brand Display;src:url({public})}}");+ let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": fetched, "srcset": format!("{declined} 2x"), "_cssText": css}}}}]);let mut block = block("kept", events);block.image_urls = vec![fetched.to_owned()];- block.remote_urls = vec![fetched.to_owned(), declined.to_owned()];+ block.font_urls = vec![font.to_owned(), public.to_owned()];+ let remote = |url: &str, kind| RemoteUrl {+ url: url.to_owned(),+ kind,+ };+ block.remote_urls = vec![+ remote(fetched, RemoteKind::Image),+ remote(declined, RemoteKind::Image),+ remote(font, RemoteKind::Font),+ remote(bundled, RemoteKind::BundledFont("inter")),+ remote(public, RemoteKind::Font),+ ];let mut keys = FlushKeys::default();let (id, key) = session_key("kept");keys.sessions.insert(id, key);keys.team_months.insert(7, [6; 32]);let mut stats = BlockStats::default();- let images = crate::urls::FlushImages::new([&block], &keys, "2026-10", &|_| false);- let jobs = crate::urls::fetch_jobs(&images, [(0, &block, &mut stats)], 1, &|_| false);+ let urls = crate::urls::FlushUrls::new([&block], &keys, "2026-10", &|_| false);+ let jobs = crate::urls::fetch_jobs(&urls, [(0, &block, &mut stats)], 1, &|_| false);let (lines, _, _) = write_block(&block, &[6; 32], "2026-10", || Ok(Vec::new())).unwrap();-let lines = String::from_utf8(lines).unwrap();- let reference = jobs["example.com"][0]["originalRef"].as_str().unwrap();- assert!(- lines.contains(&format!(r#""src":"{reference}""#)),- "{lines}"+ let references: Vec<_> = jobs+ .iter()+ .flat_map(|((_, asset), jobs)| {+ jobs.iter()+ .map(move |job| (*asset, job["originalRef"].as_str().unwrap()))+ })+ .collect();+ let [(UrlAsset::Image, image_reference), (UrlAsset::Font, font_reference), (UrlAsset::Font, public_reference)] =+ references.as_slice()+ else {+ panic!("a job each for the image and the fetched fonts, none for the bundled font: {references:?}");+ };+ assert!(font_reference.starts_with("fonturl:v4:7:2026-10:"));+ assert_eq!(+ *public_reference,+ "fonturl:v4:public:I6bjVBXgjPcaYLPITibNM4");- assert!(lines.contains(r#""srcset":"imageurl:v4:declined:credential 2x""#));- assert!(!lines.contains("cdn.example.com"));+ for expected in [+ format!(r#""src":"{image_reference}""#),+ r#""srcset":"imageurl:v4:declined:credential 2x""#.to_owned(),+ format!("src:url({font_reference})"),+ "src:url(fonturl:v4:bundled:inter)".to_owned(),+ format!("src:url({public_reference})"),+ ] {+ assert!(lines.contains(&expected), "{expected} in {lines}");+ }+ assert!(!lines.contains("example.com/"), "{lines}");+ assert!(!lines.contains("gstatic.com/"), "{lines}");}fn session_key(session: &str) -> (SessionId, Key) {diff --git a/rust/replay-anonymizer/Cargo.toml b/rust/replay-anonymizer/Cargo.tomlindex c3fea1242595..187f1833d0c7 100644--- a/rust/replay-anonymizer/Cargo.toml+++ b/rust/replay-anonymizer/Cargo.toml@@ -1,6 +1,6 @@[package]name = "posthog-replay-anonymizer"-version = "0.13.0"+version = "0.14.0"edition = "2021"description = "PII scrubber for rrweb session-replay events: text/URL redaction, image collection for an out-of-band scrub, and canvas/cv neutralization"license = "MIT"diff --git a/rust/replay-anonymizer/README.md b/rust/replay-anonymizer/README.mdindex 7b1809aee0b6..c38f9fe0083a 100644--- a/rust/replay-anonymizer/README.md+++ b/rust/replay-anonymizer/README.md@@ -6,7 +6,7 @@ Consumers:- `replay-anonymizer-node` (this workspace) wraps it as a Neon addon for the Node ingestion workers.- The MLHog training pipelines consume it from crates.io.-- The v4 recorder crates (`replay-core-v4`, `replay-recorder-v4`) write their image references and decline markers with the `v4_image_refs` module, by path and with `default-features = false`. A reader of v4 blocks parses those strings with `v4_image_refs::parse`, so the writer and the reader share one definition.+- The v4 recorder crates (`replay-core-v4`, `replay-recorder-v4`) write their image and font references and decline markers with the `v4_image_refs` module, by path and with `default-features = false`. A reader of v4 blocks parses those strings with `v4_image_refs::parse`, so the writer and the reader share one definition.Behavior is pinned by the JSON fixtures under `tests/fixtures/`, shared with the Node addon's Jest suite. Scrubbing operates on untrusted input; the public entry points contain panics and convert them to errors, so callers fail closed (drop the message) without their own `catch_unwind`. Under `panic = "abort"` that backstop cannot run — builds that must fail closed need `panic = "unwind"`.diff --git a/rust/replay-anonymizer/src/v4_image_refs.rs b/rust/replay-anonymizer/src/v4_image_refs.rsindex 3f6dad05c43f..a3d291dabc62 100644--- a/rust/replay-anonymizer/src/v4_image_refs.rs+++ b/rust/replay-anonymizer/src/v4_image_refs.rs@@ -1,6 +1,4 @@-//! The references and decline markers that stand in for an image in a v4 replay block. A captured-//! page can hold text that looks like a reference, so [`parse`] accepts only the exact strings the-//! v4 recorder writes.+//! The references and decline markers that stand in for an image or a web font in a v4 replay block. A captured page can hold text that looks like a reference, so [`parse`] accepts only the exact strings the v4 recorder writes.use std::fmt;@@ -10,13 +8,19 @@ use crate::url_policy::Decline;pub const IMAGE_PREFIX: &str = "image:v4:";/// Starts a reference whose hash names the image URL. The image fetcher stores the bytes.pub const IMAGE_URL_PREFIX: &str = "imageurl:v4:";+/// Starts a reference whose hash names the URL of a font file in an `@font-face` `src`. The image fetcher stores the bytes.+pub const FONT_URL_PREFIX: &str = "fonturl:v4:";/// The length v3 truncates a reference hash to: 132 bits of the HMAC, in URL-safe base64.pub const HASH_CHARS: usize = 22;/// What a remote URL becomes when the recorder replaces it without a check by the URL policy.pub const URL_UNCHECKED_MARKER: &str = "imageurl:v4:declined:unchecked";+/// What a remote URL in an `@font-face` `src` becomes when the recorder replaces it without a check by the URL policy.+pub const FONT_URL_UNCHECKED_MARKER: &str = "fonturl:v4:declined:unchecked";pub const IMAGE_DECLINED_PREFIX: &str = "image:v4:declined:";const DECLINED: &str = "declined:";+const BUNDLED: &str = "bundled:";+const PUBLIC: &str = "public:";#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum ImageDecline {@@ -60,13 +64,17 @@ impl ImageDecline {pub enum RefKind {Image,ImageUrl,+ FontUrl,}impl RefKind {+ const ALL: [RefKind; 3] = [RefKind::Image, RefKind::ImageUrl, RefKind::FontUrl];+pub const fn prefix(self) -> &'static str {match self {RefKind::Image => IMAGE_PREFIX,RefKind::ImageUrl => IMAGE_URL_PREFIX,+ RefKind::FontUrl => FONT_URL_PREFIX,}}}@@ -81,18 +89,24 @@ pub struct Ref<'a> {pub hash: &'a str,}-/// A marker for an image that the dataset does not keep. No bytes stand behind it.+/// A marker for an image or a font that the dataset does not keep. No bytes stand behind it.#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum Declined {Image(ImageDecline),Url(Decline),UrlUnchecked,+ FontUrl(Decline),+ FontUrlUnchecked,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub enum Parsed<'a> {Ref(Ref<'a>),Declined(Declined),+ /// A font family that the renderer ships, by its slug. No bytes stand behind it.+ BundledFont(&'a str),+ /// A font file that one public URL serves to every team, by an unkeyed hash of that URL. The image fetcher stores it once.+ PublicFont(&'a str),}impl fmt::Display for Ref<'_> {@@ -114,6 +128,8 @@ impl fmt::Display for Declined {Declined::Image(reason) => f.write_str(reason.marker()),Declined::Url(reason) => write!(f, "{IMAGE_URL_PREFIX}{DECLINED}{}", reason.label()),Declined::UrlUnchecked => f.write_str(URL_UNCHECKED_MARKER),+ Declined::FontUrl(reason) => write!(f, "{FONT_URL_PREFIX}{DECLINED}{}", reason.label()),+ Declined::FontUrlUnchecked => f.write_str(FONT_URL_UNCHECKED_MARKER),}}}@@ -123,6 +139,8 @@ impl fmt::Display for Parsed<'_> {match self {Parsed::Ref(reference) => reference.fmt(f),Parsed::Declined(declined) => declined.fmt(f),+ Parsed::BundledFont(slug) => write!(f, "{FONT_URL_PREFIX}{BUNDLED}{slug}"),+ Parsed::PublicFont(hash) => write!(f, "{FONT_URL_PREFIX}{PUBLIC}{hash}"),}}}@@ -147,20 +165,49 @@ pub fn image_url_ref(team_id: i64, key_month: &str, hash: &str) -> String {.to_string()}+pub fn font_url_ref(team_id: i64, key_month: &str, hash: &str) -> String {+ Ref {+ kind: RefKind::FontUrl,+ team_id,+ key_month,+ hash,+ }+ .to_string()+}+pub fn url_declined_marker(reason: Decline) -> String {Declined::Url(reason).to_string()}+pub fn font_url_declined_marker(reason: Decline) -> String {+ Declined::FontUrl(reason).to_string()+}++pub fn bundled_font_ref(slug: &str) -> String {+ Parsed::BundledFont(slug).to_string()+}++pub fn public_font_ref(hash: &str) -> String {+ Parsed::PublicFont(hash).to_string()+}+/// Reads a reference or a marker. Any other string is `None`, also one that differs from a string/// the recorder writes only in letter case, a leading zero, whitespace or a trailing byte.pub fn parse(s: &str) -> Option<Parsed<'_>> {if let Some(declined) = parse_declined(s) {return Some(Parsed::Declined(declined));}- let (kind, rest) = match s.strip_prefix(IMAGE_PREFIX) {- Some(rest) => (RefKind::Image, rest),- None => (RefKind::ImageUrl, s.strip_prefix(IMAGE_URL_PREFIX)?),- };+ if let Some(rest) = s.strip_prefix(FONT_URL_PREFIX) {+ if let Some(slug) = rest.strip_prefix(BUNDLED) {+ return is_slug(slug).then_some(Parsed::BundledFont(slug));+ }+ if let Some(hash) = rest.strip_prefix(PUBLIC) {+ return is_hash(hash).then_some(Parsed::PublicFont(hash));+ }+ }+ let (kind, rest) = RefKind::ALL+ .into_iter()+ .find_map(|kind| Some((kind, s.strip_prefix(kind.prefix())?)))?;let mut fields = rest.split(':');let (team_id, key_month, hash) = (fields.next()?, fields.next()?, fields.next()?);if fields.next().is_some() || !is_key_month(key_month) || !is_hash(hash) {@@ -178,17 +225,34 @@ fn parse_declined(s: &str) -> Option<Declined> {if s == URL_UNCHECKED_MARKER {return Some(Declined::UrlUnchecked);}+ if s == FONT_URL_UNCHECKED_MARKER {+ return Some(Declined::FontUrlUnchecked);+ }if let Some(reason) = ImageDecline::ALL.into_iter().find(|reason| reason.marker() == s){return Some(Declined::Image(reason));}- let label = s.strip_prefix(IMAGE_URL_PREFIX)?.strip_prefix(DECLINED)?;+ let (declined, label): (fn(Decline) -> Declined, _) = match s.strip_prefix(IMAGE_URL_PREFIX) {+ Some(rest) => (Declined::Url, rest),+ None => (Declined::FontUrl, s.strip_prefix(FONT_URL_PREFIX)?),+ };+ let label = label.strip_prefix(DECLINED)?;Decline::ALL.into_iter().find(|reason| reason.label() == label)- .map(Declined::Url)+ .map(declined)+}++/// Lowercase ASCII words joined by single `-`, as the recorder makes a slug from a family name.+fn is_slug(slug: &str) -> bool {+ slug.split('-').all(|word| {+ !word.is_empty()+ && word+ .bytes()+ .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit())+ })}/// Digits alone, as `i64` writes a positive number: no sign and no leading zero.@@ -234,6 +298,7 @@ mod tests {};let image = |reason| Parsed::Declined(Declined::Image(reason));let url = |reason| Parsed::Declined(Declined::Url(reason));+ let font_url = |reason| Parsed::Declined(Declined::FontUrl(reason));let vectors = [("image:v4:7:2026-10:AbCdEfGhIjKlMnOpQr-_09",@@ -281,6 +346,26 @@ mod tests {"imageurl:v4:declined:tracking_beacon",url(Decline::TrackingBeacon),),+ (+ "fonturl:v4:7:2026-10:AbCdEfGhIjKlMnOpQr-_09",+ reference(RefKind::FontUrl),+ ),+ (+ "fonturl:v4:declined:unchecked",+ Parsed::Declined(Declined::FontUrlUnchecked),+ ),+ (+ "fonturl:v4:declined:credential",+ font_url(Decline::Credential),+ ),+ (+ "fonturl:v4:bundled:source-sans-3",+ Parsed::BundledFont("source-sans-3"),+ ),+ (+ "fonturl:v4:public:AbCdEfGhIjKlMnOpQr-_09",+ Parsed::PublicFont(HASH),+ ),];for (written, parsed) in vectors {assert_eq!(parse(written), Some(parsed), "{written}");@@ -288,17 +373,35 @@ mod tests {}assert_eq!(image_ref(7, "2026-10", HASH), vectors[0].0);assert_eq!(image_url_ref(7, "2026-10", HASH), vectors[1].0);+ assert_eq!(+ font_url_ref(7, "2026-10", HASH),+ format!("fonturl:v4:7:2026-10:{HASH}")+ );+ assert_eq!(+ bundled_font_ref("source-sans-3"),+ "fonturl:v4:bundled:source-sans-3"+ );+ assert_eq!(public_font_ref(HASH), format!("fonturl:v4:public:{HASH}"));+ for reason in Decline::ALL {+ assert_eq!(+ parse(&font_url_declined_marker(reason)),+ Some(font_url(reason))+ );+ }let every_marker: HashSet<String> = ImageDecline::ALL.map(Declined::Image).into_iter().chain(Decline::ALL.map(Declined::Url))- .chain([Declined::UrlUnchecked])+ .chain([Declined::UrlUnchecked, Declined::FontUrlUnchecked]).map(|declined| declined.to_string()).collect();let vector_markers: HashSet<String> = vectors.iter()- .filter(|(_, parsed)| matches!(parsed, Parsed::Declined(_)))+ // The loop above covers every font URL decline.+ .filter(|(_, parsed)| {+ matches!(parsed, Parsed::Declined(declined) if !matches!(declined, Declined::FontUrl(_)))+ }).map(|(written, _)| written.to_string()).collect();assert_eq!(every_marker, vector_markers);@@ -352,6 +455,23 @@ mod tests {"imageurl:v4:declined:credential:x".to_owned(),"imageurl:v4:declined:session_token".to_owned(),"imageurl:v3:declined:credential".to_owned(),+ format!("Fonturl:v4:7:2026-10:{HASH}"),+ format!("fonturl:v4:07:2026-10:{HASH}"),+ "fonturl:v4:declined:avif".to_owned(),+ "fonturl:v4:declined:Unchecked".to_owned(),+ "fonturl:v4:bundled:".to_owned(),+ "fonturl:v4:bundled:Roboto".to_owned(),+ "fonturl:v4:bundled:open sans".to_owned(),+ "fonturl:v4:bundled:open--sans".to_owned(),+ "fonturl:v4:bundled:-roboto".to_owned(),+ "fonturl:v4:bundled:roboto-".to_owned(),+ "fonturl:v4:bundled:roboto:x".to_owned(),+ "imageurl:v4:bundled:roboto".to_owned(),+ "fonturl:v4:public:".to_owned(),+ format!("fonturl:v4:public:{}", &HASH[1..]),+ format!("fonturl:v4:public:{HASH}:x"),+ format!("fonturl:v4:Public:{HASH}"),+ format!("imageurl:v4:public:{HASH}"),];for near_miss in near_misses {assert_eq!(parse(&near_miss), None, "{near_miss:?}");