Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(extract): name the F0 schema analysis call in LLM telemetry (#4815)

Abimael Martell30bc9ba1faa6
diff --git a/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.test.ts b/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.test.ts
new file mode 100644
index 000000000..60b5b3947
--- /dev/null
+++ b/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.test.ts
@@ -0,0 +1,142 @@
+import { vi } from "vitest";
+
+vi.mock("ai", async importOriginal => {
+ const actual = await importOriginal<typeof import("ai")>();
+ return { ...actual, generateObject: vi.fn() };
+});
+vi.mock("../../../generic-ai", () => ({
+ getModel: vi.fn((name: string) => ({ modelId: name })),
+}));
+
+import { generateObject } from "ai";
+import type { Mock } from "vitest";
+import { analyzeSchemaAndPrompt_F0 } from "./analyzeSchemaAndPrompt-f0";
+
+const schema = {
+ type: "object",
+ properties: { products: { type: "array", items: { type: "string" } } },
+};
+
+function calls() {
+ return (generateObject as Mock).mock.calls.map(([args]) => args);
+}
+
+function respond(object: unknown) {
+ (generateObject as Mock).mockResolvedValueOnce({
+ object,
+ usage: { outputTokens: 1 },
+ });
+}
+
+describe("analyzeSchemaAndPrompt_F0 telemetry", () => {
+ beforeEach(() => {
+ (generateObject as Mock).mockReset();
+ });
+
+ it("attributes the gpt-4.1 analysis call to the extract that ran it", async () => {
+ respond({
+ isMultiEntity: true,
+ multiEntityKeys: ["products"],
+ reasoning: "a list of products",
+ keyIndicators: ["products"],
+ });
+
+ const result = await analyzeSchemaAndPrompt_F0(
+ ["https://example.com"],
+ schema,
+ "list the products",
+ {
+ teamId: "team-1",
+ extractId: "extract-1",
+ functionId: "performExtraction_F0",
+ },
+ );
+
+ expect(result.isMultiEntity).toBe(true);
+ expect(calls()).toHaveLength(1);
+ const [args] = calls();
+ expect(args.model.modelId).toBe("gpt-4.1");
+ expect(args.experimental_telemetry).toEqual({
+ isEnabled: true,
+ functionId: "performExtraction_F0/analyzeSchemaAndPrompt_F0",
+ metadata: {
+ teamId: "team-1",
+ extractId: "extract-1",
+ langfuseTraceId: "extract:extract-1",
+ },
+ });
+ expect(args.providerOptions.google.labels).toMatchObject({
+ functionId: "performExtraction_F0/analyzeSchemaAndPrompt_F0",
+ extractId: "extract-1",
+ teamId: "team-1",
+ });
+ });
+
+ it("falls back to its own name when the caller passes no functionId", async () => {
+ respond({
+ isMultiEntity: false,
+ multiEntityKeys: [],
+ reasoning: "",
+ keyIndicators: [],
+ });
+
+ await analyzeSchemaAndPrompt_F0(["https://example.com"], schema, "", {
+ teamId: "team-1",
+ });
+
+ const [args] = calls();
+ expect(args.experimental_telemetry.functionId).toBe(
+ "analyzeSchemaAndPrompt_F0",
+ );
+ expect(args.experimental_telemetry.metadata).toEqual({ teamId: "team-1" });
+ });
+
+ it("names the schema generation call it makes when no schema is given", async () => {
+ respond(schema);
+ respond({
+ isMultiEntity: false,
+ multiEntityKeys: [],
+ reasoning: "",
+ keyIndicators: [],
+ });
+
+ await analyzeSchemaAndPrompt_F0(
+ ["https://example.com"],
+ undefined,
+ "list the products",
+ {
+ teamId: "team-1",
+ extractId: "extract-1",
+ functionId: "performExtraction_F0",
+ },
+ );
+
+ expect(calls().map(args => args.experimental_telemetry.functionId)).toEqual(
+ [
+ "performExtraction_F0/generateSchemaFromPrompt_F0",
+ "performExtraction_F0/analyzeSchemaAndPrompt_F0",
+ ],
+ );
+ });
+
+ it("keeps the telemetry settings when the model call fails", async () => {
+ (generateObject as Mock).mockRejectedValueOnce(new Error("rate limited"));
+
+ const result = await analyzeSchemaAndPrompt_F0(
+ ["https://example.com"],
+ schema,
+ "list the products",
+ {
+ teamId: "team-1",
+ extractId: "extract-1",
+ functionId: "performExtraction_F0",
+ },
+ );
+
+ expect(result).toMatchObject({ isMultiEntity: false, multiEntityKeys: [] });
+ const [args] = calls();
+ expect(args.experimental_telemetry.functionId).toBe(
+ "performExtraction_F0/analyzeSchemaAndPrompt_F0",
+ );
+ });
+});
diff --git a/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.ts b/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.ts
index 0aa544b2a..753a54aad 100644
--- a/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/completions/analyzeSchemaAndPrompt-f0.ts
@@ -15,7 +15,7 @@ export async function analyzeSchemaAndPrompt_F0(
urls: string[],
schema: any,
prompt: string,
- metadata: { teamId: string; extractId?: string },
+ metadata: { teamId: string; functionId?: string; extractId?: string },
): Promise<{
isMultiEntity: boolean;
multiEntityKeys: string[];
@@ -54,7 +54,12 @@ export async function analyzeSchemaAndPrompt_F0(
},
markdown: "",
model,
- metadata,
+ metadata: {
+ ...metadata,
+ functionId: metadata.functionId
+ ? metadata.functionId + "/analyzeSchemaAndPrompt_F0"
+ : "analyzeSchemaAndPrompt_F0",
+ },
});
const { isMultiEntity, multiEntityKeys, reasoning, keyIndicators } =
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index d029fbf6f..ce73d6265 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -289,6 +289,7 @@ export async function performExtraction_F0(
} = await analyzeSchemaAndPrompt_F0(links, reqSchema, request.prompt ?? "", {
teamId,
extractId,
+ functionId: "performExtraction_F0",
});
logger.debug("Analyzed schema.", {
diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
index 85d9cc991..9056cec5d 100644
--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
@@ -193,7 +193,8 @@ export async function generateCompletions_F0({
mode?: "object" | "no-object";
metadata: {
teamId: string;
- functionId?: string;
+ // Required so every call is attributed in LLM telemetry.
+ functionId: string;
extractId?: string;
scrapeId?: string;
};