Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai-research-v4): record the age of republished image urls by reason (#125343)

Nicholas Waltz30c4a3529cf0
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts
index 9964ed2e1933..a78bb573236a 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.test.ts
@@ -128,7 +128,7 @@ describe('FrontierPublisher', () => {
expect(await batch.republish(candidate(), candidate(), 'retry', waitMs)).toBe('queued')
await batch.flush()
- expect(republished).toHaveBeenCalledWith('retry', 'delay')
+ expect(republished).toHaveBeenCalledWith('retry', 'delay', 0)
expect(sent[0].topic).toBe(topic)
expect(parseCollectedUrlsRecord(sent[0].value, 'example.com')).toMatchObject({
ok: true,
@@ -191,22 +191,26 @@ describe('FrontierPublisher', () => {
expect(sent.map((message) => message.topic).sort()).toEqual(['long', 'short'])
})
- it('records messages, registrable domains, and delivery time for each destination topic', async () => {
+ it('records URL age, messages, registrable domains, and delivery time for each destination topic', async () => {
const { batch } = build()
const observeRepublishBatch = jest.spyOn(ImageFetchRequestMetrics, 'observeRepublishBatch').mockImplementation()
const observeRepublishFlush = jest.spyOn(ImageFetchRequestMetrics, 'observeRepublishFlush').mockImplementation()
+ const incRepublished = jest.spyOn(ImageFetchRequestMetrics, 'incRepublished').mockImplementation()
const other = candidate({
originalRef: `imageurl:v4:7:2026-10:${'b'.repeat(22)}`,
currentUrl: 'https://img.other.net/b.png',
host: 'img.other.net',
origin: 'https://img.other.net',
registrableDomain: 'other.net',
+ firstSeenAtMs: 1_700_000_000_000 - 7_200_000,
})
await batch.republish(candidate(), candidate(), 'pass_deadline')
await batch.republish(other, other, 'retry', 30_000)
await batch.flush()
+ expect(incRepublished).toHaveBeenCalledWith('pass_deadline', 'frontier', 0)
+ expect(incRepublished).toHaveBeenCalledWith('retry', 'delay', 7_200)
expect(observeRepublishBatch).toHaveBeenCalledWith('frontier', 1, 1, expect.any(Number))
expect(observeRepublishBatch).toHaveBeenCalledWith('retry_1m', 1, 1, expect.any(Number))
expect(observeRepublishFlush).toHaveBeenCalledWith(expect.any(Number))
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts
index 511876b551c1..8bd4866ae4a8 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/frontier-publisher.ts
@@ -311,8 +311,13 @@ class BufferedRepublishBatch implements RepublishBatch {
this.recordFailedPlan(plan)
continue
}
- for (const reason of plan.reasons) {
- ImageFetchRequestMetrics.incRepublished(reason, metricTopic === 'frontier' ? 'frontier' : 'delay')
+ const nowMs = Date.now()
+ for (let job = 0; job < plan.candidates.length; job++) {
+ ImageFetchRequestMetrics.incRepublished(
+ plan.reasons[job],
+ metricTopic === 'frontier' ? 'frontier' : 'delay',
+ Math.max(0, nowMs - plan.candidates[job].firstSeenAtMs) / 1000
+ )
}
}
if (attemptedMessages > 0) {
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts
index f73ffe51e9e1..ca473c7dfb89 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/ai-research/image-fetch-v4/metrics.ts
@@ -542,6 +542,17 @@ export class ImageFetchRequestMetrics {
help: 'URLs published back to Kafka by bounded reason and destination class. "redirect" left the origin. "retry" hit a transient failure and waits in a delay topic. "not_ready" arrived before its wait ended',
labelNames: ['reason', 'topic'],
})
+ /**
+ * The capture watermark shows only the oldest URL the lane holds. This shows how many URLs are that old, and the
+ * reason they went around. Old `not_ready` URLs wait on a domain that refuses them. Old `pass_deadline` URLs mean
+ * the lane is behind.
+ */
+ private static readonly republishedUrlAge = new Histogram({
+ name: 'ai_research_v4_image_fetch_republished_url_age_seconds',
+ help: 'Time from first collection until a URL is published back to Kafka, by bounded republish reason',
+ labelNames: ['reason'],
+ buckets: [60, 600, 3600, 6 * 3600, 12 * 3600, 24 * 3600, 2 * 24 * 3600, 7 * 24 * 3600],
+ })
private static readonly republishFailed = new Counter({
name: 'ai_research_v4_image_fetch_republish_failed_total',
help: 'URLs in a Kafka message whose republish delivery failed. The input batch throws and leaves its offsets uncommitted',
@@ -583,8 +594,9 @@ export class ImageFetchRequestMetrics {
buckets: [0, 1, 2, 3, 5, 10],
})
- public static incRepublished(reason: RepublishReason, destination: RepublishDestination): void {
+ public static incRepublished(reason: RepublishReason, destination: RepublishDestination, ageSeconds: number): void {
this.republished.labels(reason, destination).inc()
+ this.republishedUrlAge.labels(reason).observe(ageSeconds)
}
public static incRepublishFailed(reason: RepublishReason): void {
this.republishFailed.labels(reason).inc()