Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(telemetry): tag branding, prompt-injection-guard and crawl-prompt LLM spans with job ids (#4814)

Abimael Martell7efe7f924821
diff --git a/apps/api/src/__tests__/lib/branding/llm-telemetry.test.ts b/apps/api/src/__tests__/lib/branding/llm-telemetry.test.ts
new file mode 100644
index 000000000..71a4e51f9
--- /dev/null
+++ b/apps/api/src/__tests__/lib/branding/llm-telemetry.test.ts
@@ -0,0 +1,70 @@
+import { vi, describe, it, expect, beforeEach } from "vitest";
+import type { Mock } from "vitest";
+
+vi.mock("ai", () => ({
+ generateObject: vi.fn(async () => ({
+ object: {
+ cleanedFonts: [],
+ colorRoles: {
+ primaryColor: "#000000",
+ accentColor: "#000000",
+ backgroundColor: "#FFFFFF",
+ textPrimary: "#000000",
+ confidence: 0.9,
+ },
+ },
+ usage: { inputTokens: 1, outputTokens: 1 },
+ })),
+}));
+vi.mock("../../../lib/generic-ai", () => ({
+ getModel: vi.fn().mockReturnValue({}),
+}));
+
+import { generateObject } from "ai";
+import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";
+import { logger } from "../../../lib/logger";
+import { CostTracking } from "../../../lib/cost-tracking";
+
+function run(ids: { teamId?: string; scrapeId?: string }) {
+ return enhanceBrandingWithLLM({
+ jsAnalysis: {},
+ buttons: [],
+ url: "https://example.com",
+ logger,
+ costTracking: new CostTracking(),
+ ...ids,
+ });
+}
+
+function telemetry() {
+ const calls = (generateObject as Mock).mock.calls;
+ expect(calls).toHaveLength(1);
+ return calls[0][0].experimental_telemetry;
+}
+
+describe("branding LLM telemetry", () => {
+ beforeEach(() => {
+ (generateObject as Mock).mockClear();
+ });
+
+ it("tags the span with the scrape id and the branding feature", async () => {
+ await run({ teamId: "test-team", scrapeId: "test-scrape" });
+
+ expect(telemetry().isEnabled).toBe(true);
+ expect(telemetry().recordInputs).toBe(false);
+ expect(telemetry().metadata).toEqual({
+ teamId: "test-team",
+ feature: "branding",
+ scrapeId: "test-scrape",
+ });
+ });
+
+ it("leaves scrapeId out when the caller has none", async () => {
+ await run({});
+
+ expect(telemetry().metadata).toEqual({
+ teamId: "unknown",
+ feature: "branding",
+ });
+ });
+});
diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.ts
index 9f2211c32..217109481 100644
--- a/apps/api/src/lib/branding/llm.ts
+++ b/apps/api/src/lib/branding/llm.ts
@@ -221,6 +221,8 @@ export async function enhanceBrandingWithLLM(
functionId: "enhanceBrandingWithLLM",
metadata: {
teamId: input.teamId || "unknown",
+ feature: "branding",
+ ...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),
},
},
});
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
index bc77de7c4..4f36fbe65 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
@@ -5,9 +5,15 @@ vi.mock("../transformers/llmExtract", () => ({
generateSchemaFromPrompt: vi.fn(),
}));
+vi.mock("./promptInjectionGuard", () => ({
+ checkForPromptInjection: vi.fn(async () => true),
+ createPromptInjectionGuardLimiter: vi.fn(),
+}));
+
import type { Mock } from "vitest";
import { extractData } from "./extractSmartScrape";
import { generateCompletions } from "../transformers/llmExtract";
+import { checkForPromptInjection } from "./promptInjectionGuard";
import { JsonExtractionContentTooLargeError } from "../error";
import { CostTracking } from "../../../lib/cost-tracking";
@@ -41,6 +47,61 @@ describe("extractData", () => {
).rejects.toBeInstanceOf(JsonExtractionContentTooLargeError);
});
+ describe("prompt injection guard", () => {
+ function run(ids: { scrapeId?: string; extractId?: string }) {
+ (generateCompletions as Mock).mockResolvedValueOnce({
+ extract: { ok: true },
+ warning: undefined,
+ totalUsage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
+ });
+ return extractData({
+ extractOptions: {
+ logger: noopLogger,
+ options: {
+ schema: { type: "object", properties: {} },
+ checkPromptInjection: true,
+ },
+ markdown: "page",
+ costTrackingOptions: {
+ costTracking: new CostTracking(),
+ metadata: {},
+ },
+ metadata: { teamId: "test-team" },
+ } as any,
+ urls: ["https://example.com"],
+ useAgent: false,
+ ...ids,
+ metadata: { teamId: "test-team", functionId: "performLLMExtract" },
+ });
+ }
+
+ beforeEach(() => {
+ (checkForPromptInjection as Mock).mockClear();
+ });
+
+ it("passes the scrape id to the guard for its span metadata", async () => {
+ await run({ scrapeId: "test-scrape" });
+
+ expect(checkForPromptInjection).toHaveBeenCalledTimes(1);
+ expect(
+ (checkForPromptInjection as Mock).mock.calls[0][0].metadata,
+ ).toEqual({
+ teamId: "test-team",
+ functionId: "performLLMExtract",
+ scrapeId: "test-scrape",
+ extractId: undefined,
+ });
+ });
+
+ it("passes the extract id to the guard for its span metadata", async () => {
+ await run({ extractId: "test-extract" });
+
+ expect(
+ (checkForPromptInjection as Mock).mock.calls[0][0].metadata,
+ ).toMatchObject({ teamId: "test-team", extractId: "test-extract" });
+ });
+ });
+
describe("SmartScrape schema wrapping", () => {
function run(schema: any, useAgent: boolean) {
return extractData({
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 9f902b3a2..6ec4ef35f 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -403,7 +403,7 @@ export async function extractData({
markdown: extractOptions.markdown,
logger,
costTracking: extractOptions.costTrackingOptions.costTracking,
- metadata,
+ metadata: { ...metadata, scrapeId, extractId },
zeroDataRetention: !!extractOptions.zeroDataRetention,
})
: Promise.resolve(true),
@@ -542,7 +542,7 @@ export async function extractData({
markdown,
logger,
costTracking: extractOptions.costTrackingOptions.costTracking,
- metadata,
+ metadata: { ...metadata, scrapeId, extractId },
zeroDataRetention: !!extractOptions.zeroDataRetention,
limiter: guardLimiter,
});
diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts
index 199589f19..6238cf6a4 100644
--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts
@@ -187,6 +187,46 @@ describe("checkForPromptInjection", () => {
expect(maxInFlight).toBe(5);
});
+ it("tags every classifier span with the scrape and extract ids", async () => {
+ vi.mocked(generateObject).mockResolvedValue(verdict(false) as any);
+
+ await checkForPromptInjection({
+ markdown: threeChunkMarkdown,
+ logger: noopLogger,
+ costTracking: new CostTracking(),
+ metadata: {
+ teamId: "test-team",
+ scrapeId: "test-scrape",
+ extractId: "test-extract",
+ },
+ zeroDataRetention: false,
+ });
+
+ const telemetry = vi
+ .mocked(generateObject)
+ .mock.calls.map(([args]) => args.experimental_telemetry);
+ expect(telemetry).toHaveLength(3);
+ for (const t of telemetry) {
+ expect(t?.recordInputs).toBe(false);
+ expect(t?.metadata).toEqual({
+ teamId: "test-team",
+ scrapeId: "test-scrape",
+ extractId: "test-extract",
+ });
+ }
+ });
+
+ it("leaves job ids out of span metadata when the caller has none", async () => {
+ vi.mocked(generateObject).mockResolvedValue(verdict(false) as any);
+
+ await run(new CostTracking(), "short page");
+
+ const [[args]] = vi.mocked(generateObject).mock.calls;
+ expect(args.experimental_telemetry?.metadata).toEqual({
+ teamId: "test-team",
+ });
+ });
+
it("resolves true without calling the classifier for empty content", async () => {
const costTracking = new CostTracking();
diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
index f32f2cdab..e580df14d 100644
--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts
@@ -58,6 +58,13 @@ function buildGuardSystemPrompt(tagName: string): string {
].join(" ");
}
+type GuardTelemetryMetadata = {
+ teamId: string;
+ functionId?: string;
+ scrapeId?: string;
+ extractId?: string;
+};
+
// "none" marks a chunk the guard failed open on. Billing reads it: the guard
// fee is charged only when every chunk got a verdict (see scrape-billing.ts).
type GuardVerdict = "clean" | "injection" | "none";
@@ -88,7 +95,7 @@ async function classifyChunk(
modelId: string,
logger: Logger,
costTracking: CostTracking,
- metadata: { teamId: string; functionId?: string },
+ metadata: GuardTelemetryMetadata,
zeroDataRetention: boolean,
): Promise<boolean> {
const tagName = `untrusted_page_content_${crypto.randomUUID()}`;
@@ -113,7 +120,11 @@ async function classifyChunk(
functionId: metadata.functionId
? metadata.functionId + "/promptInjectionGuard"
: "promptInjectionGuard",
- metadata: { teamId: metadata.teamId },
+ metadata: {
+ teamId: metadata.teamId,
+ ...(metadata.scrapeId ? { scrapeId: metadata.scrapeId } : {}),
+ ...(metadata.extractId ? { extractId: metadata.extractId } : {}),
+ },
},
});
@@ -184,7 +195,7 @@ export async function checkForPromptInjection({
markdown: string | undefined;
logger: Logger;
costTracking: CostTracking;
- metadata: { teamId: string; functionId?: string };
+ metadata: GuardTelemetryMetadata;
zeroDataRetention: boolean;
limiter?: Semaphore;
}): Promise<boolean> {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts
index 00ff91858..04a913508 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts
@@ -14,7 +14,10 @@ vi.mock("ai", async importOriginal => {
import { generateObject } from "ai";
import { encoding_for_model } from "@dqbd/tiktoken";
import type { Mock } from "vitest";
-import { generateCompletions } from "./llmExtract";
+import {
+ generateCompletions,
+ generateCrawlerOptionsFromPrompt,
+} from "./llmExtract";
import { CostTracking } from "../../../lib/cost-tracking";
import { modelPrices } from "../../../lib/extract/usage/model-prices";
@@ -163,3 +166,49 @@ describe("generateCompletions schema normalization", () => {
});
});
});
+
+describe("generateCompletions telemetry metadata", () => {
+ it("tags crawl-prompt spans with the crawl id", async () => {
+ await generateCrawlerOptionsFromPrompt(
+ "only the blog",
+ noopLogger,
+ new CostTracking(),
+ { teamId: "test-team", crawlId: "test-crawl" },
+ );
+
+ const telemetry = lastCall().experimental_telemetry;
+ expect(telemetry.functionId).toBe("generateCrawlerOptionsFromPrompt");
+ expect(telemetry.metadata).toMatchObject({
+ teamId: "test-team",
+ crawlId: "test-crawl",
+ });
+ });
+
+ it("leaves crawlId out when previewing params without a crawl", async () => {
+ await generateCrawlerOptionsFromPrompt(
+ "only the blog",
+ noopLogger,
+ new CostTracking(),
+ { teamId: "test-team" },
+ );
+
+ expect(lastCall().experimental_telemetry.metadata).toEqual({
+ teamId: "test-team",
+ });
+ });
+
+ it("keeps tagging scrape spans with the scrape id", async () => {
+ await generateCompletions({
+ logger: noopLogger,
+ options: {},
+ markdown: "page",
+ costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },
+ metadata: { teamId: "test-team", scrapeId: "test-scrape" },
+ });
+
+ expect(lastCall().experimental_telemetry.metadata).toMatchObject({
+ teamId: "test-team",
+ scrapeId: "test-scrape",
+ });
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 5475407d1..549c82f6e 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -391,6 +391,7 @@ export type GenerateCompletionsOptions = {
scrapeId?: string;
deepResearchId?: string;
llmsTxtId?: string;
+ crawlId?: string;
};
/**
* Turns off AI SDK telemetry for the call. Also on whenever the caller runs
@@ -399,6 +400,39 @@ export type GenerateCompletionsOptions = {
*/
zeroDataRetention?: boolean;
};
+
+// Span metadata that ties each call to the job that made it.
+function telemetryMetadata(metadata: GenerateCompletionsOptions["metadata"]) {
+ return {
+ teamId: metadata.teamId,
+ ...(metadata.extractId
+ ? {
+ langfuseTraceId: "extract:" + metadata.extractId,
+ extractId: metadata.extractId,
+ }
+ : {}),
+ ...(metadata.scrapeId
+ ? {
+ langfuseTraceId: "scrape:" + metadata.scrapeId,
+ scrapeId: metadata.scrapeId,
+ }
+ : {}),
+ ...(metadata.deepResearchId
+ ? {
+ langfuseTraceId: "deepResearch:" + metadata.deepResearchId,
+ deepResearchId: metadata.deepResearchId,
+ }
+ : {}),
+ ...(metadata.llmsTxtId
+ ? {
+ langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,
+ llmsTxtId: metadata.llmsTxtId,
+ }
+ : {}),
+ ...(metadata.crawlId ? { crawlId: metadata.crawlId } : {}),
+ };
+}
+
export async function generateCompletions({
logger,
options,
@@ -482,33 +516,7 @@ export async function generateCompletions({
functionId: metadata.functionId
? metadata.functionId + "/generateText"
: "generateText",
- metadata: {
- teamId: metadata.teamId,
- ...(metadata.extractId
- ? {
- langfuseTraceId: "extract:" + metadata.extractId,
- extractId: metadata.extractId,
- }
- : {}),
- ...(metadata.scrapeId
- ? {
- langfuseTraceId: "scrape:" + metadata.scrapeId,
- scrapeId: metadata.scrapeId,
- }
- : {}),
- ...(metadata.deepResearchId
- ? {
- langfuseTraceId: "deepResearch:" + metadata.deepResearchId,
- deepResearchId: metadata.deepResearchId,
- }
- : {}),
- ...(metadata.llmsTxtId
- ? {
- langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,
- llmsTxtId: metadata.llmsTxtId,
- }
- : {}),
- },
+ metadata: telemetryMetadata(metadata),
},
});
@@ -588,34 +596,7 @@ export async function generateCompletions({
functionId: metadata.functionId
? metadata.functionId + "/generateText"
: "generateText",
- metadata: {
- teamId: metadata.teamId,
- ...(metadata.extractId
- ? {
- langfuseTraceId: "extract:" + metadata.extractId,
- extractId: metadata.extractId,
- }
- : {}),
- ...(metadata.scrapeId
- ? {
- langfuseTraceId: "scrape:" + metadata.scrapeId,
- scrapeId: metadata.scrapeId,
- }
- : {}),
- ...(metadata.deepResearchId
- ? {
- langfuseTraceId:
- "deepResearch:" + metadata.deepResearchId,
- deepResearchId: metadata.deepResearchId,
- }
- : {}),
- ...(metadata.llmsTxtId
- ? {
- langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,
- llmsTxtId: metadata.llmsTxtId,
- }
- : {}),
- },
+ metadata: telemetryMetadata(metadata),
},
});
@@ -756,34 +737,7 @@ export async function generateCompletions({
functionId: metadata.functionId
? metadata.functionId + "/repairText"
: "repairText",
- metadata: {
- teamId: metadata.teamId,
- ...(metadata.extractId
- ? {
- langfuseTraceId: "extract:" + metadata.extractId,
- extractId: metadata.extractId,
- }
- : {}),
- ...(metadata.scrapeId
- ? {
- langfuseTraceId: "scrape:" + metadata.scrapeId,
- scrapeId: metadata.scrapeId,
- }
- : {}),
- ...(metadata.deepResearchId
- ? {
- langfuseTraceId:
- "deepResearch:" + metadata.deepResearchId,
- deepResearchId: metadata.deepResearchId,
- }
- : {}),
- ...(metadata.llmsTxtId
- ? {
- langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,
- llmsTxtId: metadata.llmsTxtId,
- }
- : {}),
- },
+ metadata: telemetryMetadata(metadata),
},
});
@@ -850,33 +804,7 @@ export async function generateCompletions({
experimental_telemetry: {
isEnabled: !zeroDataRetention,
functionId: metadata.functionId,
- metadata: {
- teamId: metadata.teamId,
- ...(metadata.extractId
- ? {
- langfuseTraceId: "extract:" + metadata.extractId,
- extractId: metadata.extractId,
- }
- : {}),
- ...(metadata.scrapeId
- ? {
- langfuseTraceId: "scrape:" + metadata.scrapeId,
- scrapeId: metadata.scrapeId,
- }
- : {}),
- ...(metadata.deepResearchId
- ? {
- langfuseTraceId: "deepResearch:" + metadata.deepResearchId,
- deepResearchId: metadata.deepResearchId,
- }
- : {}),
- ...(metadata.llmsTxtId
- ? {
- langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,
- llmsTxtId: metadata.llmsTxtId,
- }
- : {}),
- },
+ metadata: telemetryMetadata(metadata),
},
...(modelId.startsWith("gpt-5")
? {