Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(llm): bound tiktoken trimming to prevent event-loop freeze (#3707)

Gergő Móricz42b46be4f75a
diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.test.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.test.ts
new file mode 100644
index 000000000..d6df42399
--- /dev/null
+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.test.ts
@@ -0,0 +1,96 @@
+import { trimToTokenLimit_F0 } from "./llmExtract-f0";
+import { encoding_for_model } from "@dqbd/tiktoken";
+
+jest.mock("@dqbd/tiktoken", () => ({
+ encoding_for_model: jest.fn(),
+}));
+
+describe("trimToTokenLimit_F0", () => {
+ // Exercise the real tiktoken encoder/decoder rather than a mock, so the
+ // encode -> slice -> decode round-trip (and the freeze regression it guards
+ // against) is tested faithfully.
+ const { encoding_for_model: realEncodingForModel } = jest.requireActual(
+ "@dqbd/tiktoken",
+ ) as typeof import("@dqbd/tiktoken");
+
+ // Records the length of every string handed to encode(), so we can prove the
+ // synchronous tokenizer never has to chew through an unbounded input.
+ let encodeInputLengths: number[];
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ encodeInputLengths = [];
+ (encoding_for_model as jest.Mock).mockImplementation((model: any) => {
+ const encoder = realEncodingForModel(model);
+ const realEncode = encoder.encode.bind(encoder);
+ encoder.encode = ((input: string) => {
+ encodeInputLengths.push(input.length);
+ return realEncode(input);
+ }) as typeof encoder.encode;
+ return encoder;
+ });
+ });
+
+ it("returns the original text untouched when within the token limit", () => {
+ const text = "This is a short piece of text";
+
+ const result = trimToTokenLimit_F0(text, 1000, "gpt-4o");
+
+ expect(result.text).toBe(text);
+ expect(result.numTokens).toBeGreaterThan(0);
+ expect(result.numTokens).toBeLessThanOrEqual(1000);
+ expect(result.warning).toBeUndefined();
+ });
+
+ it("trims to exactly maxTokens and returns a byte-exact prefix", () => {
+ const text = "The quick brown fox jumps over the lazy dog. ".repeat(100);
+ const maxTokens = 50;
+
+ const result = trimToTokenLimit_F0(text, maxTokens, "gpt-4o");
+
+ expect(result.numTokens).toBe(maxTokens);
+ expect(result.warning).toContain("automatically trimmed");
+ // ASCII content round-trips exactly, so the result must be a clean prefix.
+ expect(text.startsWith(result.text)).toBe(true);
+ expect(result.text.length).toBeLessThan(text.length);
+ });
+
+ it("encodes only a bounded amount of a huge input (freeze regression)", () => {
+ // Before the fix, this synchronously tokenized the entire multi-megabyte
+ // string (and re-encoded it in a loop), blocking the event loop for tens of
+ // seconds. The pre-trim must cap how much text the encoder ever sees.
+ const maxTokens = 1000;
+ const huge = "A".repeat(10_000_000);
+
+ const start = Date.now();
+ const result = trimToTokenLimit_F0(huge, maxTokens, "gpt-4o");
+ const durationMs = Date.now() - start;
+
+ expect(Math.max(...encodeInputLengths)).toBeLessThanOrEqual(maxTokens * 5);
+ expect(result.numTokens).toBeLessThanOrEqual(maxTokens);
+ expect(result.text.length).toBeLessThanOrEqual(maxTokens * 5);
+ expect(result.warning).toBeDefined();
+ expect(durationMs).toBeLessThan(2000);
+ });
+
+ it("falls back to a char-based estimate when the encoder throws", () => {
+ const text = "This is some text to test fallback";
+ (encoding_for_model as jest.Mock).mockImplementationOnce(() => {
+ throw new Error("Encoder error");
+ });
+
+ const result = trimToTokenLimit_F0(text, 10, "gpt-4o");
+
+ expect(result.text.length).toBeLessThanOrEqual(Math.floor(10 * 2.8));
+ expect(result.numTokens).toBe(10);
+ expect(result.warning).toContain("Failed to derive number of LLM tokens");
+ });
+
+ it("handles empty text", () => {
+ const result = trimToTokenLimit_F0("", 10, "gpt-4o");
+
+ expect(result.text).toBe("");
+ expect(result.numTokens).toBe(0);
+ expect(result.warning).toBeUndefined();
+ });
+});
diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
index 4eb31fd38..85d9cc991 100644
--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
@@ -97,48 +97,63 @@ interface TrimResult {
warning?: string;
}
-function trimToTokenLimit_F0(
+// Generous upper bound on the number of characters a single token can represent.
+// Real-world ratios for cl100k/o200k are ~3-4 chars/token; 5 leaves ample headroom
+// while still bounding how much text the (synchronous, main-thread) tiktoken encoder
+// ever has to process. Without this cap, encoding an unbounded multi-megabyte string
+// can block the event loop for tens of seconds.
+const MAX_CHARS_PER_TOKEN = 5;
+
+export function trimToTokenLimit_F0(
text: string,
maxTokens: number,
modelId: string = "gpt-4o-mini",
previousWarning?: string,
): TrimResult {
+ // Pre-trim by characters before handing anything to tiktoken. This bounds the cost
+ // of the synchronous encode() below so a huge input can't freeze the event loop.
+ const maxChars = maxTokens * MAX_CHARS_PER_TOKEN;
+ const preTrimmed = text.length > maxChars;
+ const candidate = preTrimmed ? text.slice(0, maxChars) : text;
+
try {
const encoder = encoding_for_model(modelId as TiktokenModel);
try {
- const tokens = encoder.encode(text);
+ const tokens = encoder.encode(candidate);
const numTokens = tokens.length;
- if (numTokens <= maxTokens) {
+ // The candidate fits within the token budget. If we never pre-trimmed, the
+ // original text is returned untouched.
+ if (numTokens <= maxTokens && !preTrimmed) {
return { text, numTokens };
}
- const modifier = 3;
- // Start with 3 chars per token estimation
- let currentText = text.slice(0, Math.floor(maxTokens * modifier) - 1);
-
- // Keep trimming until we're under the token limit
- while (true) {
- const currentTokens = encoder.encode(currentText);
- if (currentTokens.length <= maxTokens) {
- const warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;
- return {
- text: currentText,
- numTokens: currentTokens.length,
- warning: previousWarning
- ? `${warning} ${previousWarning}`
- : warning,
- };
- }
- const overflow = currentTokens.length * modifier - maxTokens - 1;
- // If still over limit, remove another chunk
- currentText = currentText.slice(
- 0,
- Math.floor(currentText.length - overflow),
- );
+ if (numTokens <= maxTokens) {
+ // We pre-trimmed by chars but the result is already under the token limit.
+ const warning = `The extraction content would have used more characters than the maximum we allow (${maxChars}). -- the input has been automatically trimmed.`;
+ return {
+ text: candidate,
+ numTokens,
+ warning: previousWarning ? `${warning} ${previousWarning}` : warning,
+ };
}
- } catch (e) {
- throw e;
+
+ // Trim to exactly maxTokens by slicing the token array and decoding it back.
+ // This is a single encode + single decode (no re-encode loop). The cut may
+ // land mid-UTF-8-character; decode() returns raw bytes and TextDecoder
+ // replaces any trailing partial sequence, so only the final glyph can be
+ // affected -- everything before it is byte-identical to the input.
+ const trimmedTokens = tokens.slice(0, maxTokens);
+ const trimmedText = new TextDecoder().decode(
+ encoder.decode(trimmedTokens),
+ );
+
+ const warning = `The extraction content would have used more tokens (${numTokens}${preTrimmed ? "+" : ""}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;
+ return {
+ text: trimmedText,
+ numTokens: maxTokens,
+ warning: previousWarning ? `${warning} ${previousWarning}` : warning,
+ };
} finally {
encoder.free();
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
index bd20fc936..ba652285a 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
@@ -49,225 +49,144 @@ describe("removeDefaultProperty", () => {
});
describe("trimToTokenLimit", () => {
- const mockEncode = jest.fn();
- const mockFree = jest.fn();
- const mockEncoder = {
- encode: mockEncode,
- free: mockFree,
- };
+ // Exercise the real tiktoken encoder/decoder rather than a mock. The function's
+ // correctness (and the event-loop-freeze regression it guards against) depends on
+ // the real encode -> slice -> decode round-trip, which a hand-rolled mock cannot
+ // model faithfully.
+ const { encoding_for_model: realEncodingForModel } = jest.requireActual(
+ "@dqbd/tiktoken",
+ ) as typeof import("@dqbd/tiktoken");
+
+ // Records the length of every string handed to encode(), so we can prove the
+ // synchronous tokenizer never has to chew through an unbounded input.
+ let encodeInputLengths: number[];
+ let freeCalls: number;
beforeEach(() => {
jest.clearAllMocks();
- (encoding_for_model as jest.Mock).mockReturnValue(mockEncoder);
+ encodeInputLengths = [];
+ freeCalls = 0;
+ (encoding_for_model as jest.Mock).mockImplementation((model: any) => {
+ const encoder = realEncodingForModel(model);
+ const realEncode = encoder.encode.bind(encoder);
+ const realFree = encoder.free.bind(encoder);
+ encoder.encode = ((input: string) => {
+ encodeInputLengths.push(input.length);
+ return realEncode(input);
+ }) as typeof encoder.encode;
+ encoder.free = (() => {
+ freeCalls++;
+ return realFree();
+ }) as typeof encoder.free;
+ return encoder;
+ });
});
- it("should return original text if within token limit", () => {
+ it("should return original text untouched if within token limit", () => {
const text = "This is a test text";
- mockEncode.mockReturnValue(new Array(5)); // Simulate 5 tokens
- const result = trimToTokenLimit(text, 10, "gpt-4o");
+ const result = trimToTokenLimit(text, 1000, "gpt-4o");
- expect(result).toEqual({
- text,
- numTokens: 5,
- warning: undefined,
- });
- expect(mockEncode).toHaveBeenCalledWith(text);
- expect(mockFree).toHaveBeenCalled();
+ expect(result.text).toBe(text);
+ expect(result.numTokens).toBeGreaterThan(0);
+ expect(result.numTokens).toBeLessThanOrEqual(1000);
+ expect(result.warning).toBeUndefined();
+ expect(freeCalls).toBe(1);
});
- it("should trim text and return warning when exceeding token limit", () => {
- const text = "This is a longer text that needs to be trimmed";
- mockEncode
- .mockReturnValueOnce(new Array(20)) // First call for full text
- .mockReturnValueOnce(new Array(8)); // Second call for trimmed text
+ it("should trim to exactly maxTokens and return a byte-exact prefix", () => {
+ const text = "The quick brown fox jumps over the lazy dog. ".repeat(100);
+ const maxTokens = 50;
- const result = trimToTokenLimit(text, 10, "gpt-4o");
+ const result = trimToTokenLimit(text, maxTokens, "gpt-4o");
- expect(result.text.length).toBeLessThan(text.length);
- expect(result.numTokens).toBe(8);
+ expect(result.numTokens).toBe(maxTokens);
expect(result.warning).toContain("automatically trimmed");
- expect(mockEncode).toHaveBeenCalledTimes(2);
- expect(mockFree).toHaveBeenCalled();
+ // ASCII content round-trips exactly, so the result must be a clean prefix.
+ expect(text.startsWith(result.text)).toBe(true);
+ expect(result.text.length).toBeLessThan(text.length);
+ expect(freeCalls).toBe(1);
});
it("should append previous warning if provided", () => {
- const text = "This is a test text that is too long";
+ const text = "This is a test text that is definitely too long. ".repeat(
+ 100,
+ );
const previousWarning = "Previous warning message";
- mockEncode
- .mockReturnValueOnce(new Array(15))
- .mockReturnValueOnce(new Array(8));
- const result = trimToTokenLimit(text, 10, "gpt-4o", previousWarning);
+ const result = trimToTokenLimit(text, 20, "gpt-4o", previousWarning);
expect(result.warning).toContain("automatically trimmed");
expect(result.warning).toContain(previousWarning);
});
- it("should use fallback approach when encoder throws error", () => {
+ it("should encode only a bounded amount of a huge input (freeze regression)", () => {
+ // Before the fix, this synchronously tokenized the entire multi-megabyte
+ // string (and re-encoded it in a loop), blocking the event loop for tens of
+ // seconds. The pre-trim must cap how much text the encoder ever sees.
+ const maxTokens = 1000;
+ const huge = "A".repeat(10_000_000);
+
+ const start = Date.now();
+ const result = trimToTokenLimit(huge, maxTokens, "gpt-4o");
+ const durationMs = Date.now() - start;
+
+ // The encoder must never be handed more than the char-bounded candidate.
+ expect(Math.max(...encodeInputLengths)).toBeLessThanOrEqual(maxTokens * 5);
+ expect(result.numTokens).toBeLessThanOrEqual(maxTokens);
+ expect(result.text.length).toBeLessThanOrEqual(maxTokens * 5);
+ expect(result.warning).toBeDefined();
+ // Generous threshold: bounded tokenization should complete near-instantly.
+ expect(durationMs).toBeLessThan(2000);
+ expect(freeCalls).toBe(1);
+ });
+
+ it("should use fallback approach when encoder initialization throws", () => {
const text = "This is some text to test fallback";
- mockEncode.mockImplementation(() => {
+ (encoding_for_model as jest.Mock).mockImplementationOnce(() => {
throw new Error("Encoder error");
});
const result = trimToTokenLimit(text, 10, "gpt-4o");
- expect(result.text.length).toBeLessThanOrEqual(30); // 10 tokens * 3 chars per token
+ expect(result.text.length).toBeLessThanOrEqual(Math.floor(10 * 2.8));
expect(result.numTokens).toBe(10);
expect(result.warning).toContain("Failed to derive number of LLM tokens");
});
it("should handle empty text", () => {
- const text = "";
- mockEncode.mockReturnValue([]);
-
- const result = trimToTokenLimit(text, 10, "gpt-4o");
-
- expect(result).toEqual({
- text: "",
- numTokens: 0,
- warning: undefined,
- });
- expect(mockFree).toHaveBeenCalled();
- });
-
- it("should handle large token limits (128k)", () => {
- const text = "A".repeat(384000); // Assuming ~3 chars per token, this would be ~128k tokens
- mockEncode
- .mockReturnValueOnce(new Array(130000)) // First check shows it's too long
- .mockReturnValueOnce(new Array(127000)); // Second check shows it's within limit after trim
-
- const result = trimToTokenLimit(text, 128000, "gpt-4o");
-
- expect(result.text.length).toBeLessThan(text.length);
- expect(result.numTokens).toBe(127000);
- expect(result.warning).toContain("automatically trimmed");
- expect(mockEncode).toHaveBeenCalledTimes(2);
- expect(mockFree).toHaveBeenCalled();
- });
-
- it("should handle large token limits (512k) with 32k context window", () => {
- const text = "A".repeat(1536000); // Assuming ~3 chars per token, this would be ~512k tokens
- mockEncode
- .mockReturnValueOnce(new Array(520000)) // First check shows it's too long
- .mockReturnValueOnce(new Array(32000)); // Second check shows it's within context limit after trim
-
- const result = trimToTokenLimit(text, 32000, "gpt-4o");
-
- expect(result.text.length).toBeLessThan(text.length);
- expect(result.numTokens).toBe(32000);
- expect(result.warning).toContain("automatically trimmed");
- expect(mockEncode).toHaveBeenCalledTimes(2);
- expect(mockFree).toHaveBeenCalled();
- });
-
- it("should preserve text when under token limit", () => {
- const text = "Short text";
- mockEncode.mockReturnValue(new Array(5)); // 5 tokens
-
- const result = trimToTokenLimit(text, 10, "gpt-4o");
+ const result = trimToTokenLimit("", 10, "gpt-4o");
- expect(result.text).toBe(text);
- expect(result.numTokens).toBe(5);
+ expect(result.text).toBe("");
+ expect(result.numTokens).toBe(0);
expect(result.warning).toBeUndefined();
- expect(mockFree).toHaveBeenCalled();
- });
-
- it("should append new warning to previous warning", () => {
- const text = "A".repeat(300);
- const previousWarning = "Previous warning message";
- mockEncode
- .mockReturnValueOnce(new Array(100))
- .mockReturnValueOnce(new Array(50));
-
- const result = trimToTokenLimit(text, 50, "gpt-4o", previousWarning);
-
- expect(result.warning).toContain("automatically trimmed");
- expect(result.warning).toContain(previousWarning);
- expect(mockFree).toHaveBeenCalled();
- });
-
- it("should handle encoder initialization failure gracefully", () => {
- const text = "Sample text";
- (encoding_for_model as jest.Mock).mockImplementationOnce(() => {
- throw new Error("Encoder initialization failed");
- });
-
- const result = trimToTokenLimit(text, 10, "gpt-4o");
-
- expect(result.text.length).toBeLessThanOrEqual(30); // 10 tokens * 3 chars
- expect(result.warning).toContain("Failed to derive number of LLM tokens");
- expect(mockFree).not.toHaveBeenCalled();
- });
-
- it("should handle encoding errors during trimming", () => {
- const text = "Sample text";
- mockEncode.mockImplementation(() => {
- throw new Error("Encoding failed");
- });
-
- const result = trimToTokenLimit(text, 10, "gpt-4o");
-
- expect(result.text.length).toBeLessThanOrEqual(30);
- expect(result.warning).toContain("Failed to derive number of LLM tokens");
- expect(mockFree).toHaveBeenCalled();
+ expect(freeCalls).toBe(1);
});
- it("should handle very small token limits", () => {
- const text = "This is a test sentence that should be trimmed significantly";
- mockEncode
- .mockReturnValueOnce(new Array(20))
- .mockReturnValueOnce(new Array(3));
+ it("should not crash on unicode and stay within the token budget", () => {
+ const text = "Hello 👋 World 🌍 ".repeat(500);
+ const maxTokens = 5;
- const result = trimToTokenLimit(text, 3, "gpt-4o");
+ const result = trimToTokenLimit(text, maxTokens, "gpt-4o");
- expect(result.text.length).toBeLessThan(text.length);
- expect(result.numTokens).toBe(3);
+ expect(typeof result.text).toBe("string");
+ expect(result.numTokens).toBeLessThanOrEqual(maxTokens);
expect(result.warning).toContain("automatically trimmed");
- expect(mockFree).toHaveBeenCalled();
+ expect(freeCalls).toBe(1);
});
- it("should handle unicode characters", () => {
- const text = "Hello 👋 World 🌍";
- mockEncode
- .mockReturnValueOnce(new Array(8))
- .mockReturnValueOnce(new Array(4));
+ it("should pre-trim by characters even when the result fits the token budget", () => {
+ // Repeated single characters tokenize into far fewer tokens than characters,
+ // so a char-pre-trimmed candidate can already be under maxTokens.
+ const maxTokens = 100;
+ const text = "A".repeat(maxTokens * 5 * 4); // well past the char cap
- const result = trimToTokenLimit(text, 4, "gpt-4o");
+ const result = trimToTokenLimit(text, maxTokens, "gpt-4o");
+ expect(result.text.length).toBeLessThanOrEqual(maxTokens * 5);
expect(result.text.length).toBeLessThan(text.length);
- expect(result.numTokens).toBe(4);
+ expect(result.numTokens).toBeLessThanOrEqual(maxTokens);
expect(result.warning).toContain("automatically trimmed");
- expect(mockFree).toHaveBeenCalled();
- });
-
- it("should handle multiple trimming iterations", () => {
- const text = "A".repeat(1000);
- mockEncode
- .mockReturnValueOnce(new Array(300))
- .mockReturnValueOnce(new Array(200))
- .mockReturnValueOnce(new Array(100))
- .mockReturnValueOnce(new Array(50));
-
- const result = trimToTokenLimit(text, 50, "gpt-4o");
-
- expect(result.text.length).toBeLessThan(text.length);
- expect(result.numTokens).toBe(50);
- expect(result.warning).toContain("automatically trimmed");
- expect(mockEncode).toHaveBeenCalledTimes(4);
- expect(mockFree).toHaveBeenCalled();
- });
-
- it("should handle exact token limit match", () => {
- const text = "Exact token limit text";
- mockEncode.mockReturnValue(new Array(10));
-
- const result = trimToTokenLimit(text, 10, "gpt-4o");
-
- expect(result.text).toBe(text);
- expect(result.numTokens).toBe(10);
- expect(result.warning).toBeUndefined();
- expect(mockFree).toHaveBeenCalled();
});
});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 3f7f14af7..e581b2ccd 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -159,48 +159,63 @@ interface TrimResult {
warning?: string;
}
+// Generous upper bound on the number of characters a single token can represent.
+// Real-world ratios for cl100k/o200k are ~3-4 chars/token; 5 leaves ample headroom
+// while still bounding how much text the (synchronous, main-thread) tiktoken encoder
+// ever has to process. Without this cap, encoding an unbounded multi-megabyte string
+// can block the event loop for tens of seconds.
+const MAX_CHARS_PER_TOKEN = 5;
+
export function trimToTokenLimit(
text: string,
maxTokens: number,
modelId: string = "gpt-4o-mini",
previousWarning?: string,
): TrimResult {
+ // Pre-trim by characters before handing anything to tiktoken. This bounds the cost
+ // of the synchronous encode() below so a huge input can't freeze the event loop.
+ const maxChars = maxTokens * MAX_CHARS_PER_TOKEN;
+ const preTrimmed = text.length > maxChars;
+ const candidate = preTrimmed ? text.slice(0, maxChars) : text;
+
try {
const encoder = encoding_for_model(modelId as TiktokenModel);
try {
- const tokens = encoder.encode(text);
+ const tokens = encoder.encode(candidate);
const numTokens = tokens.length;
- if (numTokens <= maxTokens) {
+ // The candidate fits within the token budget. If we never pre-trimmed, the
+ // original text is returned untouched.
+ if (numTokens <= maxTokens && !preTrimmed) {
return { text, numTokens };
}
- const modifier = 3;
- // Start with 3 chars per token estimation
- let currentText = text.slice(0, Math.floor(maxTokens * modifier) - 1);
-
- // Keep trimming until we're under the token limit
- while (true) {
- const currentTokens = encoder.encode(currentText);
- if (currentTokens.length <= maxTokens) {
- const warning = `The extraction content would have used more tokens (${numTokens}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;
- return {
- text: currentText,
- numTokens: currentTokens.length,
- warning: previousWarning
- ? `${warning} ${previousWarning}`
- : warning,
- };
- }
- const overflow = currentTokens.length * modifier - maxTokens - 1;
- // If still over limit, remove another chunk
- currentText = currentText.slice(
- 0,
- Math.floor(currentText.length - overflow),
- );
+ if (numTokens <= maxTokens) {
+ // We pre-trimmed by chars but the result is already under the token limit.
+ const warning = `The extraction content would have used more characters than the maximum we allow (${maxChars}). -- the input has been automatically trimmed.`;
+ return {
+ text: candidate,
+ numTokens,
+ warning: previousWarning ? `${warning} ${previousWarning}` : warning,
+ };
}
- } catch (e) {
- throw e;
+
+ // Trim to exactly maxTokens by slicing the token array and decoding it back.
+ // This is a single encode + single decode (no re-encode loop). The cut may
+ // land mid-UTF-8-character; decode() returns raw bytes and TextDecoder
+ // replaces any trailing partial sequence, so only the final glyph can be
+ // affected -- everything before it is byte-identical to the input.
+ const trimmedTokens = tokens.slice(0, maxTokens);
+ const trimmedText = new TextDecoder().decode(
+ encoder.decode(trimmedTokens),
+ );
+
+ const warning = `The extraction content would have used more tokens (${numTokens}${preTrimmed ? "+" : ""}) than the maximum we allow (${maxTokens}). -- the input has been automatically trimmed.`;
+ return {
+ text: trimmedText,
+ numTokens: maxTokens,
+ warning: previousWarning ? `${warning} ${previousWarning}` : warning,
+ };
} finally {
encoder.free();
}