Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): surface JSON-format extraction failures instead of silently dropping the key (#4231)
Abimael Martell30e8e5050c29
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex c91cb16a6..d07cdb80e 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -378,7 +378,14 @@ export async function extractData({logger.error("failed during extractSmartScrape.ts:generateCompletions", {error,});- // console.log("failed during extractSmartScrape.ts:generateCompletions", error);+ // Surface the failure to the caller: swallowing it here made a failed+ // extraction indistinguishable from a successful-but-empty one β the+ // scrape returned 200 with the json field silently absent (and billed).+ // `warning` is provably undefined here (only assigned on the success+ // path of the try above), so assign directly β the caller merges any+ // pre-existing document warnings.+ const reason = error instanceof Error ? error.message : String(error);+ warning = `JSON extraction failed: ${reason.slice(0, 300)}`;}let extractedData = extract?.extractedData;diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.tsindex 53ab13df3..bb824997a 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts@@ -1,4 +1,18 @@-import { removeDefaultProperty } from "./llmExtract";+import { vi } from "vitest";++vi.mock("../lib/extractSmartScrape", async importOriginal => {+ const actual =+ await importOriginal<typeof import("../lib/extractSmartScrape")>();+ return {+ ...actual,+ extractData: vi.fn(async () => ({+ extractedDataArray: [],+ warning: undefined,+ costLimitExceededTokenUsage: null,+ })),+ };+});+import { performLLMExtract, removeDefaultProperty } from "./llmExtract";import { trimToTokenLimit } from "./llmExtract";import { performSummary } from "./llmExtract";import { performCleanContent } from "./llmExtract";@@ -338,3 +352,45 @@ describe("performCleanContent", () => {expect(result.warning).toBeUndefined();});});++describe("performLLMExtract empty-result handling", () => {+ it("keeps the json key (null) and adds a warning when extraction produces nothing", async () => {+ const mockMeta = {+ options: {+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: { rows: { type: "array" } },+ },+ },+ ],+ },+ internalOptions: { zeroDataRetention: false, teamId: "test-team" },+ logger: {+ child: vi.fn(() => ({+ info: vi.fn(),+ debug: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ })),+ debug: vi.fn(),+ info: vi.fn(),+ },+ costTracking: {},+ id: "test-id",+ url: "https://example.com/doc.pdf",+ } as any;++ const document = { markdown: "# some content", metadata: {} } as any;++ const result = await performLLMExtract(mockMeta, document);++ // the requested format must never vanish silently+ expect(result.json).toBeNull();+ expect(result.warning).toContain(+ "JSON extraction did not produce a result",+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex e581b2ccd..5c02c0fc2 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -1029,6 +1029,16 @@ export async function performLLMExtract(const extractedData =extractedDataArray[extractedDataArray.length - 1] ?? undefined;+ // A requested format must never vanish silently: when extraction+ // produced nothing, keep the key (null) and say why in `warning` β+ // otherwise the response is a 200 with the json field absent and no+ // signal to the caller that anything failed.+ if (extractedData === undefined && !warning) {+ document.warning =+ "JSON extraction did not produce a result." ++ (document.warning ? " " + document.warning : "");+ }+// // Prepare the schema, potentially wrapping it// const { schemaToUse, schemaWasWrapped } = prepareSmartScrapeSchema(// originalOptions.schema,@@ -1114,12 +1124,12 @@ export async function performLLMExtract(});if (meta.internalOptions.v1OriginalFormat === "extract") {- document.extract = extractedData;+ document.extract = extractedData ?? null;} else if (meta.internalOptions.v1OriginalFormat === "json") {- document.json = extractedData;+ document.json = extractedData ?? null;} else {// v2 API or no v1OriginalFormat - use json field- document.json = extractedData;+ document.json = extractedData ?? null;}// document.warning = warning;}