:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai-research-v4): record the age of republished image urls by reason (#125343)
Nicholas Waltz30c4a3529cf0
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.tsindex 9964ed2e1933..a78bb573236a 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts@@ -128,7 +128,7 @@ describe('FrontierPublisher', () => {expect(await batch.republish(candidate(), candidate(), 'retry', waitMs)).toBe('queued')await batch.flush()- expect(republished).toHaveBeenCalledWith('retry', 'delay')+ expect(republished).toHaveBeenCalledWith('retry', 'delay', 0)expect(sent[0].topic).toBe(topic)expect(parseCollectedUrlsRecord(sent[0].value, 'example.com')).toMatchObject({ok: true,@@ -191,22 +191,26 @@ describe('FrontierPublisher', () => {expect(sent.map((message) => message.topic).sort()).toEqual(['long', 'short'])})- it('records messages, registrable domains, and delivery time for each destination topic', async () => {+ it('records URL age, messages, registrable domains, and delivery time for each destination topic', async () => {const { batch } = build()const observeRepublishBatch = jest.spyOn(ImageFetchRequestMetrics, 'observeRepublishBatch').mockImplementation()const observeRepublishFlush = jest.spyOn(ImageFetchRequestMetrics, 'observeRepublishFlush').mockImplementation()+ const incRepublished = jest.spyOn(ImageFetchRequestMetrics, 'incRepublished').mockImplementation()const other = candidate({originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,currentUrl: 'https://img.other.net/b.png',host: 'img.other.net',origin: 'https://img.other.net',registrableDomain: 'other.net',+ firstSeenAtMs: 1_700_000_000_000 - 7_200_000,})await batch.republish(candidate(), candidate(), 'pass_deadline')await batch.republish(other, other, 'retry', 30_000)await batch.flush()+ expect(incRepublished).toHaveBeenCalledWith('pass_deadline', 'frontier', 0)+ expect(incRepublished).toHaveBeenCalledWith('retry', 'delay', 7_200)expect(observeRepublishBatch).toHaveBeenCalledWith('frontier', 1, 1, expect.any(Number))expect(observeRepublishBatch).toHaveBeenCalledWith('retry_1m', 1, 1, expect.any(Number))expect(observeRepublishFlush).toHaveBeenCalledWith(expect.any(Number))diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.tsindex 511876b551c1..8bd4866ae4a8 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts@@ -311,8 +311,13 @@ class BufferedRepublishBatch implements RepublishBatch {this.recordFailedPlan(plan)continue}- for (const reason of plan.reasons) {- ImageFetchRequestMetrics.incRepublished(reason, metricTopic === 'frontier' ? 'frontier' : 'delay')+ const nowMs = Date.now()+ for (let job = 0; job < plan.candidates.length; job++) {+ ImageFetchRequestMetrics.incRepublished(+ plan.reasons[job],+ metricTopic === 'frontier' ? 'frontier' : 'delay',+ Math.max(0, nowMs - plan.candidates[job].firstSeenAtMs) / 1000+ )}}if (attemptedMessages > 0) {diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.tsindex f73ffe51e9e1..ca473c7dfb89 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts@@ -542,6 +542,17 @@ export class ImageFetchRequestMetrics {help: 'URLs published back to Kafka by bounded reason and destination class. "redirect" left the origin. "retry" hit a transient failure and waits in a delay topic. "not_ready" arrived before its wait ended',labelNames: ['reason', 'topic'],})+ /**+ * The capture watermark shows only the oldest URL the lane holds. This shows how many URLs are that old, and the+ * reason they went around. Old `not_ready` URLs wait on a domain that refuses them. Old `pass_deadline` URLs mean+ * the lane is behind.+ */+ private static readonly republishedUrlAge = new Histogram({+ name: 'ai_research_v4_image_fetch_republished_url_age_seconds',+ help: 'Time from first collection until a URL is published back to Kafka, by bounded republish reason',+ labelNames: ['reason'],+ buckets: [60, 600, 3600, 6 * 3600, 12 * 3600, 24 * 3600, 2 * 24 * 3600, 7 * 24 * 3600],+ })private static readonly republishFailed = new Counter({name: 'ai_research_v4_image_fetch_republish_failed_total',help: 'URLs in a Kafka message whose republish delivery failed. The input batch throws and leaves its offsets uncommitted',@@ -583,8 +594,9 @@ export class ImageFetchRequestMetrics {buckets: [0, 1, 2, 3, 5, 10],})- public static incRepublished(reason: RepublishReason, destination: RepublishDestination): void {+ public static incRepublished(reason: RepublishReason, destination: RepublishDestination, ageSeconds: number): void {this.republished.labels(reason, destination).inc()+ this.republishedUrlAge.labels(reason).observe(ageSeconds)}public static incRepublishFailed(reason: RepublishReason): void {this.republishFailed.labels(reason).inc()