Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): trace FirePDF cache lookups with their outcome (#5116)
Abimael Martellf8105593130f
diff --git a/apps/api/src/lib/otel-tracer.test.ts b/apps/api/src/lib/otel-tracer.test.tsindex 134b6e0a1..a9332f038 100644--- a/apps/api/src/lib/otel-tracer.test.ts+++ b/apps/api/src/lib/otel-tracer.test.ts@@ -10,6 +10,7 @@ import {} from "@opentelemetry/sdk-trace-node";import {createTracerProvider,+ recordSpanException,serializeTraceContext,setSpanAttributes,SpanKind,@@ -104,6 +105,20 @@ describe("otel-tracer", () => {expect(span.events.map(event => event.name)).toContain("exception");});+ it("keeps the error status of a failure the span handled and returned from", async () => {+ const result = await withSpan("test.handled", async span => {+ recordSpanException(span, new Error("upstream down"));+ return "fallback";+ });++ expect(result).toBe("fallback");+ const [span] = await exportedSpans();+ expect(span.status).toEqual({+ code: SpanStatusCode.ERROR,+ message: "upstream down",+ });+ });+it("parents nested and library spans under the active span", async () => {await withSpan("test.parent", async () => {await withSpan("test.child", async () => {});diff --git a/apps/api/src/lib/otel-tracer.ts b/apps/api/src/lib/otel-tracer.tsindex 7efdfb9fb..f32d0b0e3 100644--- a/apps/api/src/lib/otel-tracer.ts+++ b/apps/api/src/lib/otel-tracer.ts@@ -282,6 +282,9 @@ function cleanAttributes(return cleaned;}+// Spans marked failed by `recordSpanException`; `withSpan` keeps their error status.+const erroredSpans = new WeakSet<Span>();+interface SpanOptions {attributes?: SpanAttributeInput;kind?: SpanKind;@@ -306,7 +309,9 @@ export async function withSpan<T>(async span => {try {const result = await fn(span);- span.setStatus({ code: SpanStatusCode.OK });+ if (!erroredSpans.has(span)) {+ span.setStatus({ code: SpanStatusCode.OK });+ }return result;} catch (error) {recordSpanException(span, error);@@ -324,6 +329,7 @@ export async function withSpan<T>(* failures that are handled (returned) rather than thrown out of `withSpan`.*/export function recordSpanException(span: Span, error: unknown): void {+ erroredSpans.add(span);span.recordException(error instanceof Error ? error : String(error));span.setStatus({code: SpanStatusCode.ERROR,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.tsindex def6a0072..50ea24001 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts@@ -20,6 +20,17 @@ import {firePdfCacheRefusedWritesTotal,} from "../fire-pdf/metrics";import crypto from "crypto";+import {+ context,+ propagation,+ SpanStatusCode,+ trace,+} from "@opentelemetry/api";+import {+ InMemorySpanExporter,+ type NodeTracerProvider,+} from "@opentelemetry/sdk-trace-node";+import { createTracerProvider } from "../../../../../lib/otel-tracer";import { config } from "../../../../../config";import { robustFetch } from "../../../lib/fetch";@@ -1687,4 +1698,147 @@ describe("FirePDF cache through the lookup service", () => {});expect(saveCached).not.toHaveBeenCalled();});++ describe("lookup span", () => {+ const exporter = new InMemorySpanExporter();+ let provider: NodeTracerProvider;++ beforeAll(() => {+ provider = createTracerProvider({+ exporter,+ serviceName: "test-service",+ serviceInstanceId: "pod-1",+ });+ provider.register();+ });++ afterAll(async () => {+ await provider.shutdown();+ trace.disable();+ context.disable();+ propagation.disable();+ });++ beforeEach(() => exporter.reset());++ async function lookupSpanAttributes() {+ await tryGetCached(+ serviceMeta(),+ base64,+ undefined,+ undefined,+ undefined,+ false,+ false,+ ).catch(() => undefined);+ await provider.forceFlush();+ const spans = exporter+ .getFinishedSpans()+ .filter(span => span.name === "pdf.cache_lookup");+ expect(spans).toHaveLength(1);+ return spans[0].attributes;+ }++ it("records a hit with the entry's variant and generation", async () => {+ answerWith({+ outcome: "hit",+ key: rawKey,+ variant: "base",+ generation: "2026-09-16.1",+ result: { markdown: "# doc" },+ });+ expect(await lookupSpanAttributes()).toEqual({+ "pdf.cache_outcome": "hit",+ "pdf.cache_reason": "served",+ "pdf.cache_variant": "base",+ "pdf.cache_generation": "2026-09-16.1",+ });+ });++ it("records a stale serve with its campaign, and whether the re-parse budget was spent", async () => {+ answerWith({+ outcome: "stale",+ key: inlineKey,+ variant: "base",+ generation: "unknown",+ campaign: "c1",+ result: { markdown: "# old" },+ });+ expect(await lookupSpanAttributes()).toEqual({+ "pdf.cache_outcome": "stale",+ "pdf.cache_reason": "campaign",+ "pdf.cache_variant": "base",+ "pdf.cache_generation": "unknown",+ "pdf.cache_campaign": "c1",+ });+ exporter.reset();+ answerWith({+ outcome: "stale",+ key: inlineKey,+ variant: "base",+ generation: "unknown",+ campaign: "c1",+ capped: true,+ result: { markdown: "# old" },+ });+ expect(await lookupSpanAttributes()).toMatchObject({+ "pdf.cache_outcome": "stale",+ "pdf.cache_reason": "capped",+ });+ });++ it("records a miss with the service's reason and campaign", async () => {+ answerWith({+ outcome: "miss",+ reason: "ramp",+ key: inlineKey,+ campaign: "c1",+ });+ expect(await lookupSpanAttributes()).toEqual({+ "pdf.cache_outcome": "miss",+ "pdf.cache_reason": "ramp",+ "pdf.cache_campaign": "c1",+ });+ });++ it("records an entry the request cannot use as a miss", async () => {+ answerWith({+ outcome: "hit",+ key: rawKey,+ variant: "base",+ result: { markdown: "# doc" },+ });+ await tryGetCached(+ serviceMeta(),+ base64,+ undefined,+ undefined,+ undefined,+ false,+ true,+ );+ await provider.forceFlush();+ const [span] = exporter+ .getFinishedSpans()+ .filter(s => s.name === "pdf.cache_lookup");+ expect(span.attributes).toMatchObject({+ "pdf.cache_outcome": "miss",+ "pdf.cache_reason": "unusable",+ });+ });++ it("records a failed lookup as an errored span", async () => {+ fetchMock.mockRejectedValueOnce(new Error("timed out"));+ expect(await lookupSpanAttributes()).toEqual({+ "pdf.cache_outcome": "error",+ });+ const [span] = exporter+ .getFinishedSpans()+ .filter(s => s.name === "pdf.cache_lookup");+ expect(span.status).toEqual({+ code: SpanStatusCode.ERROR,+ message: "timed out",+ });+ });+ });});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.tsindex 25eb2ab61..01a3b9bea 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts@@ -8,6 +8,12 @@ import {} from "../../../../../lib/gcs-pdf-cache";import type { PDFMode } from "../../../../../controllers/v2/types";import { robustFetch } from "../../../lib/fetch";+import {+ recordSpanException,+ type Span,+ setSpanAttributes,+ withSpan,+} from "../../../../../lib/otel-tracer";import { safeMarkdownToHtml } from "../markdownToHtml";import type { PDFProcessorResult } from "../types";import { firePdfCacheEventsTotal } from "./metrics";@@ -56,13 +62,16 @@ const lookupOutcomeSchema = z.discriminatedUnion("outcome", [outcome: z.literal("hit"),key: z.string(),variant: z.string(),+ generation: z.string().optional(),result: cachedResultSchema,}),z.object({outcome: z.literal("stale"),key: z.string(),variant: z.string(),+ generation: z.string().optional(),campaign: z.string(),+ capped: z.boolean().optional(),result: cachedResultSchema,}),z.object({@@ -87,6 +96,17 @@ export async function lookupCachedResult(/** The variant this request would write, for the miss labels. */ownVariant: string;},+): Promise<PDFProcessorResult | null> {+ return withSpan("pdf.cache_lookup", span =>+ lookupWithinSpan(span, meta, input, args),+ );+}++async function lookupWithinSpan(+ span: Span,+ meta: Meta,+ input: PdfCacheKeyInput,+ args: Parameters<typeof lookupCachedResult>[2],): Promise<PDFProcessorResult | null> {const {mode,@@ -135,6 +155,8 @@ export async function lookupCachedResult(// The scrape's own abort is not a lookup failure.meta.abort.throwIfAborted();firePdfCacheEventsTotal.inc({ event: "lookup_error", variant: ownVariant });+ setSpanAttributes(span, { "pdf.cache_outcome": "error" });+ recordSpanException(span, error);meta.logger.warn("FirePDF cache lookup failed, proceeding", {scrapeId: meta.id,error,@@ -153,6 +175,11 @@ export async function lookupCachedResult();}if (answer.outcome === "miss") {+ setSpanAttributes(span, {+ "pdf.cache_outcome": "miss",+ "pdf.cache_reason": answer.reason,+ "pdf.cache_campaign": answer.campaign,+ });firePdfCacheEventsTotal.inc({event: answer.reason === "refresh" ? "bypass_refresh" : "miss",variant: ownVariant,@@ -160,6 +187,20 @@ export async function lookupCachedResult(return null;}const { result } = answer;+ setSpanAttributes(span, {+ "pdf.cache_outcome": answer.outcome,+ "pdf.cache_reason":+ answer.outcome === "stale"+ ? answer.capped+ ? "capped"+ : "campaign"+ : "served",+ "pdf.cache_variant": answer.variant,+ "pdf.cache_generation": answer.generation,+ ...(answer.outcome === "stale" && {+ "pdf.cache_campaign": answer.campaign,+ }),+ });// Never an unusable entry, whatever the service says: a sidecar the request// needs must be present, a marker request needs a marker entry, and an// empty result for a raster image is a stale verdict, not an answer (see@@ -170,6 +211,10 @@ export async function lookupCachedResult((pageMarkers && result.page_markers !== true) ||(sourceKind === "image" && result.markdown.trim().length === 0)) {+ setSpanAttributes(span, {+ "pdf.cache_outcome": "miss",+ "pdf.cache_reason": "unusable",+ });firePdfCacheEventsTotal.inc({ event: "miss", variant: ownVariant });return null;}