Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(llm): bound tiktoken trimming to prevent event-loop freeze (#3707)
Gergő Móricz42b46be4f75a
diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.test.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.test.tsnew file mode 100644index 000000000..d6df42399--- /dev/null+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.test.ts@@ -0,0 +1,96 @@+import { trimToTokenLimit_F0 } from "./llmExtract-f0";+import { encoding_for_model } from "@dqbd/tiktoken";++jest.mock("@dqbd/tiktoken", () => ({+ encoding_for_model: jest.fn(),+}));++describe("trimToTokenLimit_F0", () => {+ // Exercise the real tiktoken encoder/decoder rather than a mock, so the+ // encode -> slice -> decode round-trip (and the freeze regression it guards+ // against) is tested faithfully.+ const { encoding_for_model: realEncodingForModel } = jest.requireActual(+ "@dqbd/tiktoken",+ ) as typeof import("@dqbd/tiktoken");++ // Records the length of every string handed to encode(), so we can prove the+ // synchronous tokenizer never has to chew through an unbounded input.+ let encodeInputLengths: number[];++ beforeEach(() => {+ jest.clearAllMocks();+ encodeInputLengths = [];+ (encoding_for_model as jest.Mock).mockImplementation((model: any) => {+ const encoder = realEncodingForModel(model);+ const realEncode = encoder.encode.bind(encoder);+ encoder.encode = ((input: string) => {+ encodeInputLengths.push(input.length);+ return realEncode(input);+ }) as typeof encoder.encode;+ return encoder;+ });+ });++ it("returns the original text untouched when within the token limit", () => {+ const text = "This is a short piece of text";++ const result = trimToTokenLimit_F0(text, 1000, "gpt-4o");++ expect(result.text).toBe(text);+ expect(result.numTokens).toBeGreaterThan(0);+ expect(result.numTokens).toBeLessThanOrEqual(1000);+ expect(result.warning).toBeUndefined();+ });++ it("trims to exactly maxTokens and returns a byte-exact prefix", () => {+ const text = "The quick brown fox jumps over the lazy dog. ".repeat(100);+ const maxTokens = 50;++ const result = trimToTokenLimit_F0(text, maxTokens, "gpt-4o");++ expect(result.numTokens).toBe(maxTokens);+ expect(result.warning).toContain("automatically trimmed");+ // ASCII content round-trips exactly, so the result must be a clean prefix.+ expect(text.startsWith(result.text)).toBe(true);+ expect(result.text.length).toBeLessThan(text.length);+ });++ it("encodes only a bounded amount of a huge input (freeze regression)", () => {+ // Before the fix, this synchronously tokenized the entire multi-megabyte+ // string (and re-encoded it in a loop), blocking the event loop for tens of+ // seconds. The pre-trim must cap how much text the encoder ever sees.+ const maxTokens = 1000;+ const huge = "A".repeat(10_000_000);++ const start = Date.now();+ const result = trimToTokenLimit_F0(huge, maxTokens, "gpt-4o");+ const durationMs = Date.now() - start;++ expect(Math.max(...encodeInputLengths)).toBeLessThanOrEqual(maxTokens * 5);+ expect(result.numTokens).toBeLessThanOrEqual(maxTokens);+ expect(result.text.length).toBeLessThanOrEqual(maxTokens * 5);+ expect(result.warning).toBeDefined();+ expect(durationMs).toBeLessThan(2000);+ });++ it("falls back to a char-based estimate when the encoder throws", () => {+ const text = "This is some text to test fallback";+ (encoding_for_model as jest.Mock).mockImplementationOnce(() => {+ throw new Error("Encoder error");+ });++ const result = trimToTokenLimit_F0(text, 10, "gpt-4o");++ expect(result.text.length).toBeLessThanOrEqual(Math.floor(10 * 2.8));+ expect(result.numTokens).toBe(10);+ expect(result.warning).toContain("Failed to derive number of LLM tokens");+ });++ it("handles empty text", () => {+ const result = trimToTokenLimit_F0("", 10, "gpt-4o");++ expect(result.text).toBe("");+ expect(result.numTokens).toBe(0);+ expect(result.warning).toBeUndefined();+ });+});diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.tsindex 4eb31fd38..85d9cc991 100644--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts@@ -97,48 +97,63 @@ interface TrimResult {warning?: string;}-function trimToTokenLimit_F0(+// Generous upper bound on the number of characters a single token can represent.+// Real-world ratios for cl100k/o200k are ~3-4 chars/token; 5 leaves ample headroom+// while still bounding how much text the (synchronous, main-thread) tiktoken encoder+// ever has to process. Without this cap, encoding an unbounded multi-megabyte string+// can block the event loop for tens of seconds.+const MAX_CHARS_PER_TOKEN = 5;++export function trimToTokenLimit_F0(text: string,maxTokens: number,modelId: string = "gpt-4o-mini",previousWarning?: string,): TrimResult {+ // Pre-trim by characters before handing anything to tiktoken. This bounds the cost+ // of the synchronous encode() below so a huge input can't freeze the event loop.+ const maxChars = maxTokens * MAX_CHARS_PER_TOKEN;+ const preTrimmed = text.length > maxChars;+ const candidate = preTrimmed ? text.slice(0, maxChars) : text;+try {const encoder = encoding_for_model(modelId as TiktokenModel);try {- const tokens = encoder.encode(text);+ const tokens = encoder.encode(candidate);const numTokens = tokens.length;- if (numTokens <= maxTokens) {+ // The candidate fits within the token budget. If we never pre-trimmed, the+ // original text is returned untouched.+ if (numTokens <= maxTokens && !preTrimmed) {return { text, numTokens };}- const modifier = 3;- // Start with 3 chars per token estimation- let currentText = text.slice(0, Math.floor(maxTokens * modifier) - 1);-- // Keep trimming until we're under the token limit- while (true) {- const currentTokens = encoder.encode(currentText);- if (currentTokens.length <= maxTokens) {- const warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;- return {- text: currentText,- numTokens: currentTokens.length,- warning: previousWarning- ? `${warning} ${previousWarning}`- : warning,- };- }- const overflow = currentTokens.length * modifier - maxTokens - 1;- // If still over limit, remove another chunk- currentText = currentText.slice(- 0,- Math.floor(currentText.length - overflow),- );+ if (numTokens <= maxTokens) {+ // We pre-trimmed by chars but the result is already under the token limit.+ const warning = `The extraction content would have used more characters than the maximum we allow (${maxChars}). -- the input has been automatically trimmed.`;+ return {+ text: candidate,+ numTokens,+ warning: previousWarning ? `${warning} ${previousWarning}` : warning,+ };}- } catch (e) {- throw e;++ // Trim to exactly maxTokens by slicing the token array and decoding it back.+ // This is a single encode + single decode (no re-encode loop). The cut may+ // land mid-UTF-8-character; decode() returns raw bytes and TextDecoder+ // replaces any trailing partial sequence, so only the final glyph can be+ // affected -- everything before it is byte-identical to the input.+ const trimmedTokens = tokens.slice(0, maxTokens);+ const trimmedText = new TextDecoder().decode(+ encoder.decode(trimmedTokens),+ );++ const warning = `The extraction content would have used more tokens (${numTokens}${preTrimmed ? "+" : ""}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;+ return {+ text: trimmedText,+ numTokens: maxTokens,+ warning: previousWarning ? `${warning} ${previousWarning}` : warning,+ };} finally {encoder.free();}diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.tsindex bd20fc936..ba652285a 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts@@ -49,225 +49,144 @@ describe("removeDefaultProperty", () => {});describe("trimToTokenLimit", () => {- const mockEncode = jest.fn();- const mockFree = jest.fn();- const mockEncoder = {- encode: mockEncode,- free: mockFree,- };+ // Exercise the real tiktoken encoder/decoder rather than a mock. The function's+ // correctness (and the event-loop-freeze regression it guards against) depends on+ // the real encode -> slice -> decode round-trip, which a hand-rolled mock cannot+ // model faithfully.+ const { encoding_for_model: realEncodingForModel } = jest.requireActual(+ "@dqbd/tiktoken",+ ) as typeof import("@dqbd/tiktoken");++ // Records the length of every string handed to encode(), so we can prove the+ // synchronous tokenizer never has to chew through an unbounded input.+ let encodeInputLengths: number[];+ let freeCalls: number;beforeEach(() => {jest.clearAllMocks();- (encoding_for_model as jest.Mock).mockReturnValue(mockEncoder);+ encodeInputLengths = [];+ freeCalls = 0;+ (encoding_for_model as jest.Mock).mockImplementation((model: any) => {+ const encoder = realEncodingForModel(model);+ const realEncode = encoder.encode.bind(encoder);+ const realFree = encoder.free.bind(encoder);+ encoder.encode = ((input: string) => {+ encodeInputLengths.push(input.length);+ return realEncode(input);+ }) as typeof encoder.encode;+ encoder.free = (() => {+ freeCalls++;+ return realFree();+ }) as typeof encoder.free;+ return encoder;+ });});- it("should return original text if within token limit", () => {+ it("should return original text untouched if within token limit", () => {const text = "This is a test text";- mockEncode.mockReturnValue(new Array(5)); // Simulate 5 tokens- const result = trimToTokenLimit(text, 10, "gpt-4o");+ const result = trimToTokenLimit(text, 1000, "gpt-4o");- expect(result).toEqual({- text,- numTokens: 5,- warning: undefined,- });- expect(mockEncode).toHaveBeenCalledWith(text);- expect(mockFree).toHaveBeenCalled();+ expect(result.text).toBe(text);+ expect(result.numTokens).toBeGreaterThan(0);+ expect(result.numTokens).toBeLessThanOrEqual(1000);+ expect(result.warning).toBeUndefined();+ expect(freeCalls).toBe(1);});- it("should trim text and return warning when exceeding token limit", () => {- const text = "This is a longer text that needs to be trimmed";- mockEncode- .mockReturnValueOnce(new Array(20)) // First call for full text- .mockReturnValueOnce(new Array(8)); // Second call for trimmed text+ it("should trim to exactly maxTokens and return a byte-exact prefix", () => {+ const text = "The quick brown fox jumps over the lazy dog. ".repeat(100);+ const maxTokens = 50;- const result = trimToTokenLimit(text, 10, "gpt-4o");+ const result = trimToTokenLimit(text, maxTokens, "gpt-4o");- expect(result.text.length).toBeLessThan(text.length);- expect(result.numTokens).toBe(8);+ expect(result.numTokens).toBe(maxTokens);expect(result.warning).toContain("automatically trimmed");- expect(mockEncode).toHaveBeenCalledTimes(2);- expect(mockFree).toHaveBeenCalled();+ // ASCII content round-trips exactly, so the result must be a clean prefix.+ expect(text.startsWith(result.text)).toBe(true);+ expect(result.text.length).toBeLessThan(text.length);+ expect(freeCalls).toBe(1);});it("should append previous warning if provided", () => {- const text = "This is a test text that is too long";+ const text = "This is a test text that is definitely too long. ".repeat(+ 100,+ );const previousWarning = "Previous warning message";- mockEncode- .mockReturnValueOnce(new Array(15))- .mockReturnValueOnce(new Array(8));- const result = trimToTokenLimit(text, 10, "gpt-4o", previousWarning);+ const result = trimToTokenLimit(text, 20, "gpt-4o", previousWarning);expect(result.warning).toContain("automatically trimmed");expect(result.warning).toContain(previousWarning);});- it("should use fallback approach when encoder throws error", () => {+ it("should encode only a bounded amount of a huge input (freeze regression)", () => {+ // Before the fix, this synchronously tokenized the entire multi-megabyte+ // string (and re-encoded it in a loop), blocking the event loop for tens of+ // seconds. The pre-trim must cap how much text the encoder ever sees.+ const maxTokens = 1000;+ const huge = "A".repeat(10_000_000);++ const start = Date.now();+ const result = trimToTokenLimit(huge, maxTokens, "gpt-4o");+ const durationMs = Date.now() - start;++ // The encoder must never be handed more than the char-bounded candidate.+ expect(Math.max(...encodeInputLengths)).toBeLessThanOrEqual(maxTokens * 5);+ expect(result.numTokens).toBeLessThanOrEqual(maxTokens);+ expect(result.text.length).toBeLessThanOrEqual(maxTokens * 5);+ expect(result.warning).toBeDefined();+ // Generous threshold: bounded tokenization should complete near-instantly.+ expect(durationMs).toBeLessThan(2000);+ expect(freeCalls).toBe(1);+ });++ it("should use fallback approach when encoder initialization throws", () => {const text = "This is some text to test fallback";- mockEncode.mockImplementation(() => {+ (encoding_for_model as jest.Mock).mockImplementationOnce(() => {throw new Error("Encoder error");});const result = trimToTokenLimit(text, 10, "gpt-4o");- expect(result.text.length).toBeLessThanOrEqual(30); // 10 tokens * 3 chars per token+ expect(result.text.length).toBeLessThanOrEqual(Math.floor(10 * 2.8));expect(result.numTokens).toBe(10);expect(result.warning).toContain("Failed to derive number of LLM tokens");});it("should handle empty text", () => {- const text = "";- mockEncode.mockReturnValue([]);-- const result = trimToTokenLimit(text, 10, "gpt-4o");-- expect(result).toEqual({- text: "",- numTokens: 0,- warning: undefined,- });- expect(mockFree).toHaveBeenCalled();- });-- it("should handle large token limits (128k)", () => {- const text = "A".repeat(384000); // Assuming ~3 chars per token, this would be ~128k tokens- mockEncode- .mockReturnValueOnce(new Array(130000)) // First check shows it's too long- .mockReturnValueOnce(new Array(127000)); // Second check shows it's within limit after trim-- const result = trimToTokenLimit(text, 128000, "gpt-4o");-- expect(result.text.length).toBeLessThan(text.length);- expect(result.numTokens).toBe(127000);- expect(result.warning).toContain("automatically trimmed");- expect(mockEncode).toHaveBeenCalledTimes(2);- expect(mockFree).toHaveBeenCalled();- });-- it("should handle large token limits (512k) with 32k context window", () => {- const text = "A".repeat(1536000); // Assuming ~3 chars per token, this would be ~512k tokens- mockEncode- .mockReturnValueOnce(new Array(520000)) // First check shows it's too long- .mockReturnValueOnce(new Array(32000)); // Second check shows it's within context limit after trim-- const result = trimToTokenLimit(text, 32000, "gpt-4o");-- expect(result.text.length).toBeLessThan(text.length);- expect(result.numTokens).toBe(32000);- expect(result.warning).toContain("automatically trimmed");- expect(mockEncode).toHaveBeenCalledTimes(2);- expect(mockFree).toHaveBeenCalled();- });-- it("should preserve text when under token limit", () => {- const text = "Short text";- mockEncode.mockReturnValue(new Array(5)); // 5 tokens-- const result = trimToTokenLimit(text, 10, "gpt-4o");+ const result = trimToTokenLimit("", 10, "gpt-4o");- expect(result.text).toBe(text);- expect(result.numTokens).toBe(5);+ expect(result.text).toBe("");+ expect(result.numTokens).toBe(0);expect(result.warning).toBeUndefined();- expect(mockFree).toHaveBeenCalled();- });-- it("should append new warning to previous warning", () => {- const text = "A".repeat(300);- const previousWarning = "Previous warning message";- mockEncode- .mockReturnValueOnce(new Array(100))- .mockReturnValueOnce(new Array(50));-- const result = trimToTokenLimit(text, 50, "gpt-4o", previousWarning);-- expect(result.warning).toContain("automatically trimmed");- expect(result.warning).toContain(previousWarning);- expect(mockFree).toHaveBeenCalled();- });-- it("should handle encoder initialization failure gracefully", () => {- const text = "Sample text";- (encoding_for_model as jest.Mock).mockImplementationOnce(() => {- throw new Error("Encoder initialization failed");- });-- const result = trimToTokenLimit(text, 10, "gpt-4o");-- expect(result.text.length).toBeLessThanOrEqual(30); // 10 tokens * 3 chars- expect(result.warning).toContain("Failed to derive number of LLM tokens");- expect(mockFree).not.toHaveBeenCalled();- });-- it("should handle encoding errors during trimming", () => {- const text = "Sample text";- mockEncode.mockImplementation(() => {- throw new Error("Encoding failed");- });-- const result = trimToTokenLimit(text, 10, "gpt-4o");-- expect(result.text.length).toBeLessThanOrEqual(30);- expect(result.warning).toContain("Failed to derive number of LLM tokens");- expect(mockFree).toHaveBeenCalled();+ expect(freeCalls).toBe(1);});- it("should handle very small token limits", () => {- const text = "This is a test sentence that should be trimmed significantly";- mockEncode- .mockReturnValueOnce(new Array(20))- .mockReturnValueOnce(new Array(3));+ it("should not crash on unicode and stay within the token budget", () => {+ const text = "Hello 👋 World 🌍 ".repeat(500);+ const maxTokens = 5;- const result = trimToTokenLimit(text, 3, "gpt-4o");+ const result = trimToTokenLimit(text, maxTokens, "gpt-4o");- expect(result.text.length).toBeLessThan(text.length);- expect(result.numTokens).toBe(3);+ expect(typeof result.text).toBe("string");+ expect(result.numTokens).toBeLessThanOrEqual(maxTokens);expect(result.warning).toContain("automatically trimmed");- expect(mockFree).toHaveBeenCalled();+ expect(freeCalls).toBe(1);});- it("should handle unicode characters", () => {- const text = "Hello 👋 World 🌍";- mockEncode- .mockReturnValueOnce(new Array(8))- .mockReturnValueOnce(new Array(4));+ it("should pre-trim by characters even when the result fits the token budget", () => {+ // Repeated single characters tokenize into far fewer tokens than characters,+ // so a char-pre-trimmed candidate can already be under maxTokens.+ const maxTokens = 100;+ const text = "A".repeat(maxTokens * 5 * 4); // well past the char cap- const result = trimToTokenLimit(text, 4, "gpt-4o");+ const result = trimToTokenLimit(text, maxTokens, "gpt-4o");+ expect(result.text.length).toBeLessThanOrEqual(maxTokens * 5);expect(result.text.length).toBeLessThan(text.length);- expect(result.numTokens).toBe(4);+ expect(result.numTokens).toBeLessThanOrEqual(maxTokens);expect(result.warning).toContain("automatically trimmed");- expect(mockFree).toHaveBeenCalled();- });-- it("should handle multiple trimming iterations", () => {- const text = "A".repeat(1000);- mockEncode- .mockReturnValueOnce(new Array(300))- .mockReturnValueOnce(new Array(200))- .mockReturnValueOnce(new Array(100))- .mockReturnValueOnce(new Array(50));-- const result = trimToTokenLimit(text, 50, "gpt-4o");-- expect(result.text.length).toBeLessThan(text.length);- expect(result.numTokens).toBe(50);- expect(result.warning).toContain("automatically trimmed");- expect(mockEncode).toHaveBeenCalledTimes(4);- expect(mockFree).toHaveBeenCalled();- });-- it("should handle exact token limit match", () => {- const text = "Exact token limit text";- mockEncode.mockReturnValue(new Array(10));-- const result = trimToTokenLimit(text, 10, "gpt-4o");-- expect(result.text).toBe(text);- expect(result.numTokens).toBe(10);- expect(result.warning).toBeUndefined();- expect(mockFree).toHaveBeenCalled();});});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 3f7f14af7..e581b2ccd 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -159,48 +159,63 @@ interface TrimResult {warning?: string;}+// Generous upper bound on the number of characters a single token can represent.+// Real-world ratios for cl100k/o200k are ~3-4 chars/token; 5 leaves ample headroom+// while still bounding how much text the (synchronous, main-thread) tiktoken encoder+// ever has to process. Without this cap, encoding an unbounded multi-megabyte string+// can block the event loop for tens of seconds.+const MAX_CHARS_PER_TOKEN = 5;+export function trimToTokenLimit(text: string,maxTokens: number,modelId: string = "gpt-4o-mini",previousWarning?: string,): TrimResult {+ // Pre-trim by characters before handing anything to tiktoken. This bounds the cost+ // of the synchronous encode() below so a huge input can't freeze the event loop.+ const maxChars = maxTokens * MAX_CHARS_PER_TOKEN;+ const preTrimmed = text.length > maxChars;+ const candidate = preTrimmed ? text.slice(0, maxChars) : text;+try {const encoder = encoding_for_model(modelId as TiktokenModel);try {- const tokens = encoder.encode(text);+ const tokens = encoder.encode(candidate);const numTokens = tokens.length;- if (numTokens <= maxTokens) {+ // The candidate fits within the token budget. If we never pre-trimmed, the+ // original text is returned untouched.+ if (numTokens <= maxTokens && !preTrimmed) {return { text, numTokens };}- const modifier = 3;- // Start with 3 chars per token estimation- let currentText = text.slice(0, Math.floor(maxTokens * modifier) - 1);-- // Keep trimming until we're under the token limit- while (true) {- const currentTokens = encoder.encode(currentText);- if (currentTokens.length <= maxTokens) {- const warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;- return {- text: currentText,- numTokens: currentTokens.length,- warning: previousWarning- ? `${warning} ${previousWarning}`- : warning,- };- }- const overflow = currentTokens.length * modifier - maxTokens - 1;- // If still over limit, remove another chunk- currentText = currentText.slice(- 0,- Math.floor(currentText.length - overflow),- );+ if (numTokens <= maxTokens) {+ // We pre-trimmed by chars but the result is already under the token limit.+ const warning = `The extraction content would have used more characters than the maximum we allow (${maxChars}). -- the input has been automatically trimmed.`;+ return {+ text: candidate,+ numTokens,+ warning: previousWarning ? `${warning} ${previousWarning}` : warning,+ };}- } catch (e) {- throw e;++ // Trim to exactly maxTokens by slicing the token array and decoding it back.+ // This is a single encode + single decode (no re-encode loop). The cut may+ // land mid-UTF-8-character; decode() returns raw bytes and TextDecoder+ // replaces any trailing partial sequence, so only the final glyph can be+ // affected -- everything before it is byte-identical to the input.+ const trimmedTokens = tokens.slice(0, maxTokens);+ const trimmedText = new TextDecoder().decode(+ encoder.decode(trimmedTokens),+ );++ const warning = `The extraction content would have used more tokens (${numTokens}${preTrimmed ? "+" : ""}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;+ return {+ text: trimmedText,+ numTokens: maxTokens,+ warning: previousWarning ? `${warning} ${previousWarning}` : warning,+ };} finally {encoder.free();}