Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: skip summary generation when markdown is empty (#2226)
devin-ai-integration[bot]f3a224758c52
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.tsindex 1326bbe82..aece5739b 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts@@ -1,5 +1,6 @@import { removeDefaultProperty } from "./llmExtract";import { trimToTokenLimit } from "./llmExtract";+import { performSummary } from "./llmExtract";import { encoding_for_model } from "@dqbd/tiktoken";jest.mock("@dqbd/tiktoken", () => ({@@ -268,3 +269,55 @@ describe("trimToTokenLimit", () => {expect(mockFree).toHaveBeenCalled();});});++describe("performSummary", () => {+ it("should skip summary generation and add warning when markdown is empty", async () => {+ const mockMeta = {+ options: { formats: [{ type: "summary" }] },+ internalOptions: { zeroDataRetention: false, teamId: "test-team" },+ logger: {+ child: jest.fn(() => ({+ info: jest.fn(),+ })),+ },+ costTracking: {},+ id: "test-id",+ } as any;++ const document = {+ markdown: "",+ } as any;++ const result = await performSummary(mockMeta, document);++ expect(result.summary).toBeUndefined();+ expect(result.warning).toContain(+ "Summary generation was skipped because the markdown content is empty",+ );+ });++ it("should skip summary generation when markdown is whitespace-only", async () => {+ const mockMeta = {+ options: { formats: [{ type: "summary" }] },+ internalOptions: { zeroDataRetention: false, teamId: "test-team" },+ logger: {+ child: jest.fn(() => ({+ info: jest.fn(),+ })),+ },+ costTracking: {},+ id: "test-id",+ } as any;++ const document = {+ markdown: " \n\t ",+ } as any;++ const result = await performSummary(mockMeta, document);++ expect(result.summary).toBeUndefined();+ expect(result.warning).toContain(+ "Summary generation was skipped because the markdown content is empty",+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 7ca70792d..e57963882 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -1062,6 +1062,13 @@ export async function performSummary(document.warning = trimOutput.warning;+ if (!trimOutput.text || trimOutput.text.trim() === "") {+ document.warning =+ "Summary generation was skipped because the markdown content is empty." ++ (document.warning ? " " + document.warning : "");+ return document;+ }+const generationOptions: GenerateCompletionsOptions = {logger: meta.logger.child({method: "performSummary/generateCompletions",