Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(telemetry): tag branding, prompt-injection-guard and crawl-prompt LLM spans with job ids (#4814)
Abimael Martell7efe7f924821
diff --git a/apps/api/src/__tests__/lib/branding/llm-telemetry.test.ts b/apps/api/src/__tests__/lib/branding/llm-telemetry.test.tsnew file mode 100644index 000000000..71a4e51f9--- /dev/null+++ b/apps/api/src/__tests__/lib/branding/llm-telemetry.test.ts@@ -0,0 +1,70 @@+import { vi, describe, it, expect, beforeEach } from "vitest";+import type { Mock } from "vitest";++vi.mock("ai", () => ({+ generateObject: vi.fn(async () => ({+ object: {+ cleanedFonts: [],+ colorRoles: {+ primaryColor: "#000000",+ accentColor: "#000000",+ backgroundColor: "#FFFFFF",+ textPrimary: "#000000",+ confidence: 0.9,+ },+ },+ usage: { inputTokens: 1, outputTokens: 1 },+ })),+}));+vi.mock("../../../lib/generic-ai", () => ({+ getModel: vi.fn().mockReturnValue({}),+}));++import { generateObject } from "ai";+import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";+import { logger } from "../../../lib/logger";+import { CostTracking } from "../../../lib/cost-tracking";++function run(ids: { teamId?: string; scrapeId?: string }) {+ return enhanceBrandingWithLLM({+ jsAnalysis: {},+ buttons: [],+ url: "https://example.com",+ logger,+ costTracking: new CostTracking(),+ ...ids,+ });+}++function telemetry() {+ const calls = (generateObject as Mock).mock.calls;+ expect(calls).toHaveLength(1);+ return calls[0][0].experimental_telemetry;+}++describe("branding LLM telemetry", () => {+ beforeEach(() => {+ (generateObject as Mock).mockClear();+ });++ it("tags the span with the scrape id and the branding feature", async () => {+ await run({ teamId: "test-team", scrapeId: "test-scrape" });++ expect(telemetry().isEnabled).toBe(true);+ expect(telemetry().recordInputs).toBe(false);+ expect(telemetry().metadata).toEqual({+ teamId: "test-team",+ feature: "branding",+ scrapeId: "test-scrape",+ });+ });++ it("leaves scrapeId out when the caller has none", async () => {+ await run({});++ expect(telemetry().metadata).toEqual({+ teamId: "unknown",+ feature: "branding",+ });+ });+});diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.tsindex 9f2211c32..217109481 100644--- a/apps/api/src/lib/branding/llm.ts+++ b/apps/api/src/lib/branding/llm.ts@@ -221,6 +221,8 @@ export async function enhanceBrandingWithLLM(functionId: "enhanceBrandingWithLLM",metadata: {teamId: input.teamId || "unknown",+ feature: "branding",+ ...(input.scrapeId ? { scrapeId: input.scrapeId } : {}),},},});diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.tsindex bc77de7c4..4f36fbe65 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts@@ -5,9 +5,15 @@ vi.mock("../transformers/llmExtract", () => ({generateSchemaFromPrompt: vi.fn(),}));+vi.mock("./promptInjectionGuard", () => ({+ checkForPromptInjection: vi.fn(async () => true),+ createPromptInjectionGuardLimiter: vi.fn(),+}));+import type { Mock } from "vitest";import { extractData } from "./extractSmartScrape";import { generateCompletions } from "../transformers/llmExtract";+import { checkForPromptInjection } from "./promptInjectionGuard";import { JsonExtractionContentTooLargeError } from "../error";import { CostTracking } from "../../../lib/cost-tracking";@@ -41,6 +47,61 @@ describe("extractData", () => {).rejects.toBeInstanceOf(JsonExtractionContentTooLargeError);});+ describe("prompt injection guard", () => {+ function run(ids: { scrapeId?: string; extractId?: string }) {+ (generateCompletions as Mock).mockResolvedValueOnce({+ extract: { ok: true },+ warning: undefined,+ totalUsage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },+ });+ return extractData({+ extractOptions: {+ logger: noopLogger,+ options: {+ schema: { type: "object", properties: {} },+ checkPromptInjection: true,+ },+ markdown: "page",+ costTrackingOptions: {+ costTracking: new CostTracking(),+ metadata: {},+ },+ metadata: { teamId: "test-team" },+ } as any,+ urls: ["https://example.com"],+ useAgent: false,+ ...ids,+ metadata: { teamId: "test-team", functionId: "performLLMExtract" },+ });+ }++ beforeEach(() => {+ (checkForPromptInjection as Mock).mockClear();+ });++ it("passes the scrape id to the guard for its span metadata", async () => {+ await run({ scrapeId: "test-scrape" });++ expect(checkForPromptInjection).toHaveBeenCalledTimes(1);+ expect(+ (checkForPromptInjection as Mock).mock.calls[0][0].metadata,+ ).toEqual({+ teamId: "test-team",+ functionId: "performLLMExtract",+ scrapeId: "test-scrape",+ extractId: undefined,+ });+ });++ it("passes the extract id to the guard for its span metadata", async () => {+ await run({ extractId: "test-extract" });++ expect(+ (checkForPromptInjection as Mock).mock.calls[0][0].metadata,+ ).toMatchObject({ teamId: "test-team", extractId: "test-extract" });+ });+ });+describe("SmartScrape schema wrapping", () => {function run(schema: any, useAgent: boolean) {return extractData({diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 9f902b3a2..6ec4ef35f 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -403,7 +403,7 @@ export async function extractData({markdown: extractOptions.markdown,logger,costTracking: extractOptions.costTrackingOptions.costTracking,- metadata,+ metadata: { ...metadata, scrapeId, extractId },zeroDataRetention: !!extractOptions.zeroDataRetention,}): Promise.resolve(true),@@ -542,7 +542,7 @@ export async function extractData({markdown,logger,costTracking: extractOptions.costTrackingOptions.costTracking,- metadata,+ metadata: { ...metadata, scrapeId, extractId },zeroDataRetention: !!extractOptions.zeroDataRetention,limiter: guardLimiter,});diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.tsindex 199589f19..6238cf6a4 100644--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.test.ts@@ -187,6 +187,46 @@ describe("checkForPromptInjection", () => {expect(maxInFlight).toBe(5);});+ it("tags every classifier span with the scrape and extract ids", async () => {+ vi.mocked(generateObject).mockResolvedValue(verdict(false) as any);++ await checkForPromptInjection({+ markdown: threeChunkMarkdown,+ logger: noopLogger,+ costTracking: new CostTracking(),+ metadata: {+ teamId: "test-team",+ scrapeId: "test-scrape",+ extractId: "test-extract",+ },+ zeroDataRetention: false,+ });++ const telemetry = vi+ .mocked(generateObject)+ .mock.calls.map(([args]) => args.experimental_telemetry);+ expect(telemetry).toHaveLength(3);+ for (const t of telemetry) {+ expect(t?.recordInputs).toBe(false);+ expect(t?.metadata).toEqual({+ teamId: "test-team",+ scrapeId: "test-scrape",+ extractId: "test-extract",+ });+ }+ });++ it("leaves job ids out of span metadata when the caller has none", async () => {+ vi.mocked(generateObject).mockResolvedValue(verdict(false) as any);++ await run(new CostTracking(), "short page");++ const [[args]] = vi.mocked(generateObject).mock.calls;+ expect(args.experimental_telemetry?.metadata).toEqual({+ teamId: "test-team",+ });+ });+it("resolves true without calling the classifier for empty content", async () => {const costTracking = new CostTracking();diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.tsindex f32f2cdab..e580df14d 100644--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts@@ -58,6 +58,13 @@ function buildGuardSystemPrompt(tagName: string): string {].join(" ");}+type GuardTelemetryMetadata = {+ teamId: string;+ functionId?: string;+ scrapeId?: string;+ extractId?: string;+};+// "none" marks a chunk the guard failed open on. Billing reads it: the guard// fee is charged only when every chunk got a verdict (see scrape-billing.ts).type GuardVerdict = "clean" | "injection" | "none";@@ -88,7 +95,7 @@ async function classifyChunk(modelId: string,logger: Logger,costTracking: CostTracking,- metadata: { teamId: string; functionId?: string },+ metadata: GuardTelemetryMetadata,zeroDataRetention: boolean,): Promise<boolean> {const tagName = `untrusted_page_content_${crypto.randomUUID()}`;@@ -113,7 +120,11 @@ async function classifyChunk(functionId: metadata.functionId? metadata.functionId + "/promptInjectionGuard": "promptInjectionGuard",- metadata: { teamId: metadata.teamId },+ metadata: {+ teamId: metadata.teamId,+ ...(metadata.scrapeId ? { scrapeId: metadata.scrapeId } : {}),+ ...(metadata.extractId ? { extractId: metadata.extractId } : {}),+ },},});@@ -184,7 +195,7 @@ export async function checkForPromptInjection({markdown: string | undefined;logger: Logger;costTracking: CostTracking;- metadata: { teamId: string; functionId?: string };+ metadata: GuardTelemetryMetadata;zeroDataRetention: boolean;limiter?: Semaphore;}): Promise<boolean> {diff --git a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.tsindex 00ff91858..04a913508 100644--- a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts@@ -14,7 +14,10 @@ vi.mock("ai", async importOriginal => {import { generateObject } from "ai";import { encoding_for_model } from "@dqbd/tiktoken";import type { Mock } from "vitest";-import { generateCompletions } from "./llmExtract";+import {+ generateCompletions,+ generateCrawlerOptionsFromPrompt,+} from "./llmExtract";import { CostTracking } from "../../../lib/cost-tracking";import { modelPrices } from "../../../lib/extract/usage/model-prices";@@ -163,3 +166,49 @@ describe("generateCompletions schema normalization", () => {});});});++describe("generateCompletions telemetry metadata", () => {+ it("tags crawl-prompt spans with the crawl id", async () => {+ await generateCrawlerOptionsFromPrompt(+ "only the blog",+ noopLogger,+ new CostTracking(),+ { teamId: "test-team", crawlId: "test-crawl" },+ );++ const telemetry = lastCall().experimental_telemetry;+ expect(telemetry.functionId).toBe("generateCrawlerOptionsFromPrompt");+ expect(telemetry.metadata).toMatchObject({+ teamId: "test-team",+ crawlId: "test-crawl",+ });+ });++ it("leaves crawlId out when previewing params without a crawl", async () => {+ await generateCrawlerOptionsFromPrompt(+ "only the blog",+ noopLogger,+ new CostTracking(),+ { teamId: "test-team" },+ );++ expect(lastCall().experimental_telemetry.metadata).toEqual({+ teamId: "test-team",+ });+ });++ it("keeps tagging scrape spans with the scrape id", async () => {+ await generateCompletions({+ logger: noopLogger,+ options: {},+ markdown: "page",+ costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },+ metadata: { teamId: "test-team", scrapeId: "test-scrape" },+ });++ expect(lastCall().experimental_telemetry.metadata).toMatchObject({+ teamId: "test-team",+ scrapeId: "test-scrape",+ });+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 5475407d1..549c82f6e 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -391,6 +391,7 @@ export type GenerateCompletionsOptions = {scrapeId?: string;deepResearchId?: string;llmsTxtId?: string;+ crawlId?: string;};/*** Turns off AI SDK telemetry for the call. Also on whenever the caller runs@@ -399,6 +400,39 @@ export type GenerateCompletionsOptions = {*/zeroDataRetention?: boolean;};++// Span metadata that ties each call to the job that made it.+function telemetryMetadata(metadata: GenerateCompletionsOptions["metadata"]) {+ return {+ teamId: metadata.teamId,+ ...(metadata.extractId+ ? {+ langfuseTraceId: "extract:" + metadata.extractId,+ extractId: metadata.extractId,+ }+ : {}),+ ...(metadata.scrapeId+ ? {+ langfuseTraceId: "scrape:" + metadata.scrapeId,+ scrapeId: metadata.scrapeId,+ }+ : {}),+ ...(metadata.deepResearchId+ ? {+ langfuseTraceId: "deepResearch:" + metadata.deepResearchId,+ deepResearchId: metadata.deepResearchId,+ }+ : {}),+ ...(metadata.llmsTxtId+ ? {+ langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,+ llmsTxtId: metadata.llmsTxtId,+ }+ : {}),+ ...(metadata.crawlId ? { crawlId: metadata.crawlId } : {}),+ };+}+export async function generateCompletions({logger,options,@@ -482,33 +516,7 @@ export async function generateCompletions({functionId: metadata.functionId? metadata.functionId + "/generateText": "generateText",- metadata: {- teamId: metadata.teamId,- ...(metadata.extractId- ? {- langfuseTraceId: "extract:" + metadata.extractId,- extractId: metadata.extractId,- }- : {}),- ...(metadata.scrapeId- ? {- langfuseTraceId: "scrape:" + metadata.scrapeId,- scrapeId: metadata.scrapeId,- }- : {}),- ...(metadata.deepResearchId- ? {- langfuseTraceId: "deepResearch:" + metadata.deepResearchId,- deepResearchId: metadata.deepResearchId,- }- : {}),- ...(metadata.llmsTxtId- ? {- langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,- llmsTxtId: metadata.llmsTxtId,- }- : {}),- },+ metadata: telemetryMetadata(metadata),},});@@ -588,34 +596,7 @@ export async function generateCompletions({functionId: metadata.functionId? metadata.functionId + "/generateText": "generateText",- metadata: {- teamId: metadata.teamId,- ...(metadata.extractId- ? {- langfuseTraceId: "extract:" + metadata.extractId,- extractId: metadata.extractId,- }- : {}),- ...(metadata.scrapeId- ? {- langfuseTraceId: "scrape:" + metadata.scrapeId,- scrapeId: metadata.scrapeId,- }- : {}),- ...(metadata.deepResearchId- ? {- langfuseTraceId:- "deepResearch:" + metadata.deepResearchId,- deepResearchId: metadata.deepResearchId,- }- : {}),- ...(metadata.llmsTxtId- ? {- langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,- llmsTxtId: metadata.llmsTxtId,- }- : {}),- },+ metadata: telemetryMetadata(metadata),},});@@ -756,34 +737,7 @@ export async function generateCompletions({functionId: metadata.functionId? metadata.functionId + "/repairText": "repairText",- metadata: {- teamId: metadata.teamId,- ...(metadata.extractId- ? {- langfuseTraceId: "extract:" + metadata.extractId,- extractId: metadata.extractId,- }- : {}),- ...(metadata.scrapeId- ? {- langfuseTraceId: "scrape:" + metadata.scrapeId,- scrapeId: metadata.scrapeId,- }- : {}),- ...(metadata.deepResearchId- ? {- langfuseTraceId:- "deepResearch:" + metadata.deepResearchId,- deepResearchId: metadata.deepResearchId,- }- : {}),- ...(metadata.llmsTxtId- ? {- langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,- llmsTxtId: metadata.llmsTxtId,- }- : {}),- },+ metadata: telemetryMetadata(metadata),},});@@ -850,33 +804,7 @@ export async function generateCompletions({experimental_telemetry: {isEnabled: !zeroDataRetention,functionId: metadata.functionId,- metadata: {- teamId: metadata.teamId,- ...(metadata.extractId- ? {- langfuseTraceId: "extract:" + metadata.extractId,- extractId: metadata.extractId,- }- : {}),- ...(metadata.scrapeId- ? {- langfuseTraceId: "scrape:" + metadata.scrapeId,- scrapeId: metadata.scrapeId,- }- : {}),- ...(metadata.deepResearchId- ? {- langfuseTraceId: "deepResearch:" + metadata.deepResearchId,- deepResearchId: metadata.deepResearchId,- }- : {}),- ...(metadata.llmsTxtId- ? {- langfuseTraceId: "llmsTxt:" + metadata.llmsTxtId,- llmsTxtId: metadata.llmsTxtId,- }- : {}),- },+ metadata: telemetryMetadata(metadata),},...(modelId.startsWith("gpt-5")? {