Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(tokenisation): drop the unicode pass that never changes its input (#17347)

Mohil Ahujadf3eaaa36d61
diff --git a/worker/src/__tests__/tokenisationUnicode.unit.test.ts b/worker/src/__tests__/tokenisationUnicode.unit.test.ts
new file mode 100644
index 0000000000..e571d34090
--- /dev/null
+++ b/worker/src/__tests__/tokenisationUnicode.unit.test.ts
@@ -0,0 +1,42 @@
+import { encoding_for_model } from "tiktoken";
+import { afterAll, describe, expect, it } from "vitest";
+
+import { type Model } from "@langfuse/shared";
+
+import { tokenCount } from "../features/tokenisation/usage";
+
+const gpt4o = {
+ id: "model-gpt-4o",
+ tokenizerId: "openai",
+ tokenizerConfig: {
+ tokenizerModel: "gpt-4o",
+ tokensPerMessage: 3,
+ tokensPerName: 1,
+ },
+} as unknown as Model;
+
+const encoding = encoding_for_model("gpt-4o");
+
+afterAll(() => {
+ encoding.free();
+});
+
+describe("token count of text outside the basic plane", () => {
+ // Text was passed through a pass that hex-encoded astral characters before
+ // counting. It never actually did so, because indexing a string yields UTF-16
+ // code units and an astral character arrives as a lone surrogate. These pin
+ // the counts to what the tokeniser itself reports, so a later change that
+ // revives the encoding is caught rather than silently repricing every
+ // observation containing an emoji.
+ it.each([
+ ["an emoji", "hello 😀 world"],
+ ["a musical symbol", "𝄞 clef"],
+ ["a multi-person emoji sequence", "👨‍👩‍👧‍👦 family"],
+ ["a regional indicator pair", "🇮🇳 flag"],
+ ["text with no astral characters", "plain ascii text"],
+ ])("counts %s as the tokeniser does", (_label, text) => {
+ expect(tokenCount({ model: gpt4o, text })).toBe(
+ encoding.encode(text, "all").length,
+ );
+ });
+});
diff --git a/worker/src/features/tokenisation/usage.ts b/worker/src/features/tokenisation/usage.ts
index e35e329236..f43e3f5e1d 100644
--- a/worker/src/features/tokenisation/usage.ts
+++ b/worker/src/features/tokenisation/usage.ts
@@ -189,10 +189,8 @@ const getTokensByModel = (model: TiktokenModel, text: string) => {
encoding = get_encoding("cl100k_base");
}
- const cleandedText = unicodeToBytesInString(text);
-
logger.debug(`Tokenized data for model: ${model}`);
- return encoding?.encode(cleandedText, "all").length;
+ return encoding?.encode(text, "all").length;
};
interface Tokenizer {
@@ -226,24 +224,3 @@ function isChatMessageArray(value: unknown): value is ChatMessage[] {
(!("name" in item) || typeof item.name === "string"),
);
}
-
-function unicodeToBytesInString(input: string): string {
- let result = "";
- for (let i = 0; i < input.length; i++) {
- const char = input[i];
- if (char && /[\u{10000}-\u{10FFFF}]/u.test(char)) {
- const bytes = unicodeToBytes(char);
- result += Array.from(bytes)
- .map((b) => b.toString(16))
- .join("");
- } else {
- result += char;
- }
- }
- return result;
-}
-
-function unicodeToBytes(input: string): Uint8Array {
- const encoder = new TextEncoder();
- return encoder.encode(input);
-}