Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(scraper): skip onlyCleanContent for documents exceeding model output limit (#3240)

firecrawl-spring[bot]c7914fb955ba
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
index aece5739b..bd20fc936 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
@@ -1,6 +1,7 @@
import { removeDefaultProperty } from "./llmExtract";
import { trimToTokenLimit } from "./llmExtract";
import { performSummary } from "./llmExtract";
+import { performCleanContent } from "./llmExtract";
import { encoding_for_model } from "@dqbd/tiktoken";
jest.mock("@dqbd/tiktoken", () => ({
@@ -321,3 +322,96 @@ describe("performSummary", () => {
);
});
});
+
+describe("performCleanContent", () => {
+ const mockEncode = jest.fn();
+ const mockFree = jest.fn();
+ const mockEncoder = {
+ encode: mockEncode,
+ free: mockFree,
+ };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ (encoding_for_model as jest.Mock).mockReturnValue(mockEncoder);
+ });
+
+ const makeMeta = (onlyCleanContent: boolean) =>
+ ({
+ options: { onlyCleanContent },
+ internalOptions: { zeroDataRetention: false, teamId: "test-team" },
+ logger: {
+ child: jest.fn(() => ({ info: jest.fn() })),
+ info: jest.fn(),
+ },
+ costTracking: {},
+ id: "test-id",
+ }) as any;
+
+ it("should skip cleaning when input tokens exceed model max output tokens", async () => {
+ const longMarkdown = "A".repeat(200000); // simulate a very long document
+ // Simulate 80,000 tokens β€” well above gpt-4o-mini's 16,384 output limit
+ mockEncode.mockReturnValue(new Array(80000));
+
+ const document = { markdown: longMarkdown } as any;
+ const result = await performCleanContent(makeMeta(true), document);
+
+ // Should preserve original markdown
+ expect(result.markdown).toBe(longMarkdown);
+ // Should have a warning about skipping
+ expect(result.warning).toContain("Content cleaning was skipped");
+ expect(result.warning).toContain("too long");
+ expect(result.warning).toContain("80000 tokens");
+ expect(result.warning).toContain("original markdown has been preserved");
+ });
+
+ it("should not skip cleaning when input tokens are within model output limit", async () => {
+ // Simulate 5,000 tokens β€” well within gpt-4o-mini's 16,384 output limit
+ mockEncode.mockReturnValue(new Array(5000));
+
+ const document = { markdown: "Short content for cleaning" } as any;
+
+ // Track whether logger.child was called with the generateCompletions method,
+ // which only happens after the guard passes (line 1180 in llmExtract.ts).
+ const childLogger = { info: jest.fn(), error: jest.fn(), warn: jest.fn() };
+ const loggerChild = jest.fn(() => childLogger);
+ const meta = {
+ options: { onlyCleanContent: true },
+ internalOptions: { zeroDataRetention: false, teamId: "test-team" },
+ logger: {
+ child: loggerChild,
+ info: jest.fn(),
+ error: jest.fn(),
+ },
+ costTracking: {},
+ id: "test-id",
+ } as any;
+
+ // The call will fail inside generateCompletions (no LLM provider configured),
+ // but if it gets that far, it proves the guard didn't fire.
+ try {
+ await performCleanContent(meta, document);
+ } catch (_e) {
+ // Expected β€” no LLM available in test
+ }
+
+ // Verify the guard did NOT skip: logger.child should have been called with
+ // the generateCompletions method, which only happens after the guard.
+ expect(loggerChild).toHaveBeenCalledWith(
+ expect.objectContaining({
+ method: "performCleanContent/generateCompletions",
+ }),
+ );
+ expect(document.warning ?? "").not.toContain(
+ "Content cleaning was skipped because the content is too long",
+ );
+ });
+
+ it("should return document unchanged when onlyCleanContent is false", async () => {
+ const document = { markdown: "Some content" } as any;
+ const result = await performCleanContent(makeMeta(false), document);
+
+ expect(result.markdown).toBe("Some content");
+ expect(result.warning).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index abe4ed682..e8caa0f0d 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -1143,6 +1143,21 @@ export async function performCleanContent(
document.warning = trimOutput.warning;
+ const modelLimits = getModelLimits("gpt-4o-mini");
+ if (trimOutput.numTokens > modelLimits.maxOutputTokens) {
+ const skipWarning = `Content cleaning was skipped because the content is too long (${trimOutput.numTokens} tokens) for the model to return in full (max output: ${modelLimits.maxOutputTokens} tokens). The original markdown has been preserved.`;
+ document.warning =
+ skipWarning + (document.warning ? " " + document.warning : "");
+ meta.logger.info(
+ "Skipping onlyCleanContent: input tokens exceed model output limit",
+ {
+ inputTokens: trimOutput.numTokens,
+ maxOutputTokens: modelLimits.maxOutputTokens,
+ },
+ );
+ return document;
+ }
+
if (!trimOutput.text || trimOutput.text.trim() === "") {
document.warning =
"Content cleaning was skipped because the markdown content is empty." +
@@ -1414,8 +1429,14 @@ ${markdown}
</page>`;
const modelChain = [
- { name: "gemini-2.5-flash-lite", model: getModel("gemini-2.5-flash-lite", "google") },
- { name: "gemini-2.0-flash-lite", model: getModel("gemini-2.0-flash-lite", "google") },
+ {
+ name: "gemini-2.5-flash-lite",
+ model: getModel("gemini-2.5-flash-lite", "google"),
+ },
+ {
+ name: "gemini-2.0-flash-lite",
+ model: getModel("gemini-2.0-flash-lite", "google"),
+ },
];
for (const { name, model } of modelChain) {