Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(extract): name the F0 schema analysis call in LLM telemetry (#4815)
Abimael Martell30bc9ba1faa6
diff --git a/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.test.ts b/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.test.tsnew file mode 100644index 000000000..60b5b3947--- /dev/null+++ b/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.test.ts@@ -0,0 +1,142 @@+import { vi } from "vitest";++vi.mock("ai", async importOriginal => {+ const actual = await importOriginal<typeof import("ai")>();+ return { ...actual, generateObject: vi.fn() };+});+vi.mock("../../../generic-ai", () => ({+ getModel: vi.fn((name: string) => ({ modelId: name })),+}));++import { generateObject } from "ai";+import type { Mock } from "vitest";+import { analyzeSchemaAndPrompt_F0 } from "./analyzeSchemaAndPrompt-f0";++const schema = {+ type: "object",+ properties: { products: { type: "array", items: { type: "string" } } },+};++function calls() {+ return (generateObject as Mock).mock.calls.map(([args]) => args);+}++function respond(object: unknown) {+ (generateObject as Mock).mockResolvedValueOnce({+ object,+ usage: { outputTokens: 1 },+ });+}++describe("analyzeSchemaAndPrompt_F0 telemetry", () => {+ beforeEach(() => {+ (generateObject as Mock).mockReset();+ });++ it("attributes the gpt-4.1 analysis call to the extract that ran it", async () => {+ respond({+ isMultiEntity: true,+ multiEntityKeys: ["products"],+ reasoning: "a list of products",+ keyIndicators: ["products"],+ });++ const result = await analyzeSchemaAndPrompt_F0(+ ["https://example.com"],+ schema,+ "list the products",+ {+ teamId: "team-1",+ extractId: "extract-1",+ functionId: "performExtraction_F0",+ },+ );++ expect(result.isMultiEntity).toBe(true);+ expect(calls()).toHaveLength(1);+ const [args] = calls();+ expect(args.model.modelId).toBe("gpt-4.1");+ expect(args.experimental_telemetry).toEqual({+ isEnabled: true,+ functionId: "performExtraction_F0/analyzeSchemaAndPrompt_F0",+ metadata: {+ teamId: "team-1",+ extractId: "extract-1",+ langfuseTraceId: "extract:extract-1",+ },+ });+ expect(args.providerOptions.google.labels).toMatchObject({+ functionId: "performExtraction_F0/analyzeSchemaAndPrompt_F0",+ extractId: "extract-1",+ teamId: "team-1",+ });+ });++ it("falls back to its own name when the caller passes no functionId", async () => {+ respond({+ isMultiEntity: false,+ multiEntityKeys: [],+ reasoning: "",+ keyIndicators: [],+ });++ await analyzeSchemaAndPrompt_F0(["https://example.com"], schema, "", {+ teamId: "team-1",+ });++ const [args] = calls();+ expect(args.experimental_telemetry.functionId).toBe(+ "analyzeSchemaAndPrompt_F0",+ );+ expect(args.experimental_telemetry.metadata).toEqual({ teamId: "team-1" });+ });++ it("names the schema generation call it makes when no schema is given", async () => {+ respond(schema);+ respond({+ isMultiEntity: false,+ multiEntityKeys: [],+ reasoning: "",+ keyIndicators: [],+ });++ await analyzeSchemaAndPrompt_F0(+ ["https://example.com"],+ undefined,+ "list the products",+ {+ teamId: "team-1",+ extractId: "extract-1",+ functionId: "performExtraction_F0",+ },+ );++ expect(calls().map(args => args.experimental_telemetry.functionId)).toEqual(+ [+ "performExtraction_F0/generateSchemaFromPrompt_F0",+ "performExtraction_F0/analyzeSchemaAndPrompt_F0",+ ],+ );+ });++ it("keeps the telemetry settings when the model call fails", async () => {+ (generateObject as Mock).mockRejectedValueOnce(new Error("rate limited"));++ const result = await analyzeSchemaAndPrompt_F0(+ ["https://example.com"],+ schema,+ "list the products",+ {+ teamId: "team-1",+ extractId: "extract-1",+ functionId: "performExtraction_F0",+ },+ );++ expect(result).toMatchObject({ isMultiEntity: false, multiEntityKeys: [] });+ const [args] = calls();+ expect(args.experimental_telemetry.functionId).toBe(+ "performExtraction_F0/analyzeSchemaAndPrompt_F0",+ );+ });+});diff --git a/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.ts b/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.tsindex 0aa544b2a..753a54aad 100644--- a/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.ts+++ b/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.ts@@ -15,7 +15,7 @@ export async function analyzeSchemaAndPrompt_F0(urls: string[],schema: any,prompt: string,- metadata: { teamId: string; extractId?: string },+ metadata: { teamId: string; functionId?: string; extractId?: string },): Promise<{isMultiEntity: boolean;multiEntityKeys: string[];@@ -54,7 +54,12 @@ export async function analyzeSchemaAndPrompt_F0(},markdown: "",model,- metadata,+ metadata: {+ ...metadata,+ functionId: metadata.functionId+ ? metadata.functionId + "/analyzeSchemaAndPrompt_F0"+ : "analyzeSchemaAndPrompt_F0",+ },});const { isMultiEntity, multiEntityKeys, reasoning, keyIndicators } =diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex d029fbf6f..ce73d6265 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -289,6 +289,7 @@ export async function performExtraction_F0(} = await analyzeSchemaAndPrompt_F0(links, reqSchema, request.prompt ?? "", {teamId,extractId,+ functionId: "performExtraction_F0",});logger.debug("Analyzed schema.", {diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.tsindex 85d9cc991..9056cec5d 100644--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts@@ -193,7 +193,8 @@ export async function generateCompletions_F0({mode?: "object" | "no-object";metadata: {teamId: string;- functionId?: string;+ // Required so every call is attributed in LLM telemetry.+ functionId: string;extractId?: string;scrapeId?: string;};