Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scraper): skip onlyCleanContent for documents exceeding model output limit (#3240)
firecrawl-spring[bot]c7914fb955ba
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.tsindex aece5739b..bd20fc936 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts@@ -1,6 +1,7 @@import { removeDefaultProperty } from "./llmExtract";import { trimToTokenLimit } from "./llmExtract";import { performSummary } from "./llmExtract";+import { performCleanContent } from "./llmExtract";import { encoding_for_model } from "@dqbd/tiktoken";jest.mock("@dqbd/tiktoken", () => ({@@ -321,3 +322,96 @@ describe("performSummary", () => {);});});++describe("performCleanContent", () => {+ const mockEncode = jest.fn();+ const mockFree = jest.fn();+ const mockEncoder = {+ encode: mockEncode,+ free: mockFree,+ };++ beforeEach(() => {+ jest.clearAllMocks();+ (encoding_for_model as jest.Mock).mockReturnValue(mockEncoder);+ });++ const makeMeta = (onlyCleanContent: boolean) =>+ ({+ options: { onlyCleanContent },+ internalOptions: { zeroDataRetention: false, teamId: "test-team" },+ logger: {+ child: jest.fn(() => ({ info: jest.fn() })),+ info: jest.fn(),+ },+ costTracking: {},+ id: "test-id",+ }) as any;++ it("should skip cleaning when input tokens exceed model max output tokens", async () => {+ const longMarkdown = "A".repeat(200000); // simulate a very long document+ // Simulate 80,000 tokens β well above gpt-4o-mini's 16,384 output limit+ mockEncode.mockReturnValue(new Array(80000));++ const document = { markdown: longMarkdown } as any;+ const result = await performCleanContent(makeMeta(true), document);++ // Should preserve original markdown+ expect(result.markdown).toBe(longMarkdown);+ // Should have a warning about skipping+ expect(result.warning).toContain("Content cleaning was skipped");+ expect(result.warning).toContain("too long");+ expect(result.warning).toContain("80000 tokens");+ expect(result.warning).toContain("original markdown has been preserved");+ });++ it("should not skip cleaning when input tokens are within model output limit", async () => {+ // Simulate 5,000 tokens β well within gpt-4o-mini's 16,384 output limit+ mockEncode.mockReturnValue(new Array(5000));++ const document = { markdown: "Short content for cleaning" } as any;++ // Track whether logger.child was called with the generateCompletions method,+ // which only happens after the guard passes (line 1180 in llmExtract.ts).+ const childLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn() };+ const loggerChild = jest.fn(() => childLogger);+ const meta = {+ options: { onlyCleanContent: true },+ internalOptions: { zeroDataRetention: false, teamId: "test-team" },+ logger: {+ child: loggerChild,+ info: jest.fn(),+ error: jest.fn(),+ },+ costTracking: {},+ id: "test-id",+ } as any;++ // The call will fail inside generateCompletions (no LLM provider configured),+ // but if it gets that far, it proves the guard didn't fire.+ try {+ await performCleanContent(meta, document);+ } catch (_e) {+ // Expected β no LLM available in test+ }++ // Verify the guard did NOT skip: logger.child should have been called with+ // the generateCompletions method, which only happens after the guard.+ expect(loggerChild).toHaveBeenCalledWith(+ expect.objectContaining({+ method: "performCleanContent/generateCompletions",+ }),+ );+ expect(document.warning ?? "").not.toContain(+ "Content cleaning was skipped because the content is too long",+ );+ });++ it("should return document unchanged when onlyCleanContent is false", async () => {+ const document = { markdown: "Some content" } as any;+ const result = await performCleanContent(makeMeta(false), document);++ expect(result.markdown).toBe("Some content");+ expect(result.warning).toBeUndefined();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex abe4ed682..e8caa0f0d 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -1143,6 +1143,21 @@ export async function performCleanContent(document.warning = trimOutput.warning;+ const modelLimits = getModelLimits("gpt-4o-mini");+ if (trimOutput.numTokens > modelLimits.maxOutputTokens) {+ const skipWarning = `Content cleaning was skipped because the content is too long (${trimOutput.numTokens} tokens) for the model to return in full (max output: ${modelLimits.maxOutputTokens} tokens). The original markdown has been preserved.`;+ document.warning =+ skipWarning + (document.warning ? " " + document.warning : "");+ meta.logger.info(+ "Skipping onlyCleanContent: input tokens exceed model output limit",+ {+ inputTokens: trimOutput.numTokens,+ maxOutputTokens: modelLimits.maxOutputTokens,+ },+ );+ return document;+ }+if (!trimOutput.text || trimOutput.text.trim() === "") {document.warning ="Content cleaning was skipped because the markdown content is empty." +@@ -1414,8 +1429,14 @@ ${markdown}</page>`;const modelChain = [- { name: "gemini-2.5-flash-lite", model: getModel("gemini-2.5-flash-lite", "google") },- { name: "gemini-2.0-flash-lite", model: getModel("gemini-2.0-flash-lite", "google") },+ {+ name: "gemini-2.5-flash-lite",+ model: getModel("gemini-2.5-flash-lite", "google"),+ },+ {+ name: "gemini-2.0-flash-lite",+ model: getModel("gemini-2.0-flash-lite", "google"),+ },];for (const { name, model } of modelChain) {