Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(pdf): trace FirePDF cache lookups with their outcome (#5116)

Abimael Martellf8105593130f
diff --git a/apps/api/src/lib/otel-tracer.test.ts b/apps/api/src/lib/otel-tracer.test.ts
index 134b6e0a1..a9332f038 100644
--- a/apps/api/src/lib/otel-tracer.test.ts
+++ b/apps/api/src/lib/otel-tracer.test.ts
@@ -10,6 +10,7 @@ import {
} from "@opentelemetry/sdk-trace-node";
import {
createTracerProvider,
+ recordSpanException,
serializeTraceContext,
setSpanAttributes,
SpanKind,
@@ -104,6 +105,20 @@ describe("otel-tracer", () => {
expect(span.events.map(event => event.name)).toContain("exception");
});
+ it("keeps the error status of a failure the span handled and returned from", async () => {
+ const result = await withSpan("test.handled", async span => {
+ recordSpanException(span, new Error("upstream down"));
+ return "fallback";
+ });
+
+ expect(result).toBe("fallback");
+ const [span] = await exportedSpans();
+ expect(span.status).toEqual({
+ code: SpanStatusCode.ERROR,
+ message: "upstream down",
+ });
+ });
+
it("parents nested and library spans under the active span", async () => {
await withSpan("test.parent", async () => {
await withSpan("test.child", async () => {});
diff --git a/apps/api/src/lib/otel-tracer.ts b/apps/api/src/lib/otel-tracer.ts
index 7efdfb9fb..f32d0b0e3 100644
--- a/apps/api/src/lib/otel-tracer.ts
+++ b/apps/api/src/lib/otel-tracer.ts
@@ -282,6 +282,9 @@ function cleanAttributes(
return cleaned;
}
+// Spans marked failed by `recordSpanException`; `withSpan` keeps their error status.
+const erroredSpans = new WeakSet<Span>();
+
interface SpanOptions {
attributes?: SpanAttributeInput;
kind?: SpanKind;
@@ -306,7 +309,9 @@ export async function withSpan<T>(
async span => {
try {
const result = await fn(span);
- span.setStatus({ code: SpanStatusCode.OK });
+ if (!erroredSpans.has(span)) {
+ span.setStatus({ code: SpanStatusCode.OK });
+ }
return result;
} catch (error) {
recordSpanException(span, error);
@@ -324,6 +329,7 @@ export async function withSpan<T>(
* failures that are handled (returned) rather than thrown out of `withSpan`.
*/
export function recordSpanException(span: Span, error: unknown): void {
+ erroredSpans.add(span);
span.recordException(error instanceof Error ? error : String(error));
span.setStatus({
code: SpanStatusCode.ERROR,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
index def6a0072..50ea24001 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
@@ -20,6 +20,17 @@ import {
firePdfCacheRefusedWritesTotal,
} from "../fire-pdf/metrics";
import crypto from "crypto";
+import {
+ context,
+ propagation,
+ SpanStatusCode,
+ trace,
+} from "@opentelemetry/api";
+import {
+ InMemorySpanExporter,
+ type NodeTracerProvider,
+} from "@opentelemetry/sdk-trace-node";
+import { createTracerProvider } from "../../../../../lib/otel-tracer";
import { config } from "../../../../../config";
import { robustFetch } from "../../../lib/fetch";
@@ -1687,4 +1698,147 @@ describe("FirePDF cache through the lookup service", () => {
});
expect(saveCached).not.toHaveBeenCalled();
});
+
+ describe("lookup span", () => {
+ const exporter = new InMemorySpanExporter();
+ let provider: NodeTracerProvider;
+
+ beforeAll(() => {
+ provider = createTracerProvider({
+ exporter,
+ serviceName: "test-service",
+ serviceInstanceId: "pod-1",
+ });
+ provider.register();
+ });
+
+ afterAll(async () => {
+ await provider.shutdown();
+ trace.disable();
+ context.disable();
+ propagation.disable();
+ });
+
+ beforeEach(() => exporter.reset());
+
+ async function lookupSpanAttributes() {
+ await tryGetCached(
+ serviceMeta(),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ false,
+ false,
+ ).catch(() => undefined);
+ await provider.forceFlush();
+ const spans = exporter
+ .getFinishedSpans()
+ .filter(span => span.name === "pdf.cache_lookup");
+ expect(spans).toHaveLength(1);
+ return spans[0].attributes;
+ }
+
+ it("records a hit with the entry's variant and generation", async () => {
+ answerWith({
+ outcome: "hit",
+ key: rawKey,
+ variant: "base",
+ generation: "2026-09-16.1",
+ result: { markdown: "# doc" },
+ });
+ expect(await lookupSpanAttributes()).toEqual({
+ "pdf.cache_outcome": "hit",
+ "pdf.cache_reason": "served",
+ "pdf.cache_variant": "base",
+ "pdf.cache_generation": "2026-09-16.1",
+ });
+ });
+
+ it("records a stale serve with its campaign, and whether the re-parse budget was spent", async () => {
+ answerWith({
+ outcome: "stale",
+ key: inlineKey,
+ variant: "base",
+ generation: "unknown",
+ campaign: "c1",
+ result: { markdown: "# old" },
+ });
+ expect(await lookupSpanAttributes()).toEqual({
+ "pdf.cache_outcome": "stale",
+ "pdf.cache_reason": "campaign",
+ "pdf.cache_variant": "base",
+ "pdf.cache_generation": "unknown",
+ "pdf.cache_campaign": "c1",
+ });
+ exporter.reset();
+ answerWith({
+ outcome: "stale",
+ key: inlineKey,
+ variant: "base",
+ generation: "unknown",
+ campaign: "c1",
+ capped: true,
+ result: { markdown: "# old" },
+ });
+ expect(await lookupSpanAttributes()).toMatchObject({
+ "pdf.cache_outcome": "stale",
+ "pdf.cache_reason": "capped",
+ });
+ });
+
+ it("records a miss with the service's reason and campaign", async () => {
+ answerWith({
+ outcome: "miss",
+ reason: "ramp",
+ key: inlineKey,
+ campaign: "c1",
+ });
+ expect(await lookupSpanAttributes()).toEqual({
+ "pdf.cache_outcome": "miss",
+ "pdf.cache_reason": "ramp",
+ "pdf.cache_campaign": "c1",
+ });
+ });
+
+ it("records an entry the request cannot use as a miss", async () => {
+ answerWith({
+ outcome: "hit",
+ key: rawKey,
+ variant: "base",
+ result: { markdown: "# doc" },
+ });
+ await tryGetCached(
+ serviceMeta(),
+ base64,
+ undefined,
+ undefined,
+ undefined,
+ false,
+ true,
+ );
+ await provider.forceFlush();
+ const [span] = exporter
+ .getFinishedSpans()
+ .filter(s => s.name === "pdf.cache_lookup");
+ expect(span.attributes).toMatchObject({
+ "pdf.cache_outcome": "miss",
+ "pdf.cache_reason": "unusable",
+ });
+ });
+
+ it("records a failed lookup as an errored span", async () => {
+ fetchMock.mockRejectedValueOnce(new Error("timed out"));
+ expect(await lookupSpanAttributes()).toEqual({
+ "pdf.cache_outcome": "error",
+ });
+ const [span] = exporter
+ .getFinishedSpans()
+ .filter(s => s.name === "pdf.cache_lookup");
+ expect(span.status).toEqual({
+ code: SpanStatusCode.ERROR,
+ message: "timed out",
+ });
+ });
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
index 25eb2ab61..01a3b9bea 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache-service.ts
@@ -8,6 +8,12 @@ import {
} from "../../../../../lib/gcs-pdf-cache";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { robustFetch } from "../../../lib/fetch";
+import {
+ recordSpanException,
+ type Span,
+ setSpanAttributes,
+ withSpan,
+} from "../../../../../lib/otel-tracer";
import { safeMarkdownToHtml } from "../markdownToHtml";
import type { PDFProcessorResult } from "../types";
import { firePdfCacheEventsTotal } from "./metrics";
@@ -56,13 +62,16 @@ const lookupOutcomeSchema = z.discriminatedUnion("outcome", [
outcome: z.literal("hit"),
key: z.string(),
variant: z.string(),
+ generation: z.string().optional(),
result: cachedResultSchema,
}),
z.object({
outcome: z.literal("stale"),
key: z.string(),
variant: z.string(),
+ generation: z.string().optional(),
campaign: z.string(),
+ capped: z.boolean().optional(),
result: cachedResultSchema,
}),
z.object({
@@ -87,6 +96,17 @@ export async function lookupCachedResult(
/** The variant this request would write, for the miss labels. */
ownVariant: string;
},
+): Promise<PDFProcessorResult | null> {
+ return withSpan("pdf.cache_lookup", span =>
+ lookupWithinSpan(span, meta, input, args),
+ );
+}
+
+async function lookupWithinSpan(
+ span: Span,
+ meta: Meta,
+ input: PdfCacheKeyInput,
+ args: Parameters<typeof lookupCachedResult>[2],
): Promise<PDFProcessorResult | null> {
const {
mode,
@@ -135,6 +155,8 @@ export async function lookupCachedResult(
// The scrape's own abort is not a lookup failure.
meta.abort.throwIfAborted();
firePdfCacheEventsTotal.inc({ event: "lookup_error", variant: ownVariant });
+ setSpanAttributes(span, { "pdf.cache_outcome": "error" });
+ recordSpanException(span, error);
meta.logger.warn("FirePDF cache lookup failed, proceeding", {
scrapeId: meta.id,
error,
@@ -153,6 +175,11 @@ export async function lookupCachedResult(
);
}
if (answer.outcome === "miss") {
+ setSpanAttributes(span, {
+ "pdf.cache_outcome": "miss",
+ "pdf.cache_reason": answer.reason,
+ "pdf.cache_campaign": answer.campaign,
+ });
firePdfCacheEventsTotal.inc({
event: answer.reason === "refresh" ? "bypass_refresh" : "miss",
variant: ownVariant,
@@ -160,6 +187,20 @@ export async function lookupCachedResult(
return null;
}
const { result } = answer;
+ setSpanAttributes(span, {
+ "pdf.cache_outcome": answer.outcome,
+ "pdf.cache_reason":
+ answer.outcome === "stale"
+ ? answer.capped
+ ? "capped"
+ : "campaign"
+ : "served",
+ "pdf.cache_variant": answer.variant,
+ "pdf.cache_generation": answer.generation,
+ ...(answer.outcome === "stale" && {
+ "pdf.cache_campaign": answer.campaign,
+ }),
+ });
// Never an unusable entry, whatever the service says: a sidecar the request
// needs must be present, a marker request needs a marker entry, and an
// empty result for a raster image is a stale verdict, not an answer (see
@@ -170,6 +211,10 @@ export async function lookupCachedResult(
(pageMarkers && result.page_markers !== true) ||
(sourceKind === "image" && result.markdown.trim().length === 0)
) {
+ setSpanAttributes(span, {
+ "pdf.cache_outcome": "miss",
+ "pdf.cache_reason": "unusable",
+ });
firePdfCacheEventsTotal.inc({ event: "miss", variant: ownVariant });
return null;
}