🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(tokenisation): drop the unicode pass that never changes its input (#17347)
Mohil Ahujadf3eaaa36d61
diff --git a/worker/src/__tests__/tokenisationUnicode.unit.test.ts b/worker/src/__tests__/tokenisationUnicode.unit.test.tsnew file mode 100644index 0000000000..e571d34090--- /dev/null+++ b/worker/src/__tests__/tokenisationUnicode.unit.test.ts@@ -0,0 +1,42 @@+import { encoding_for_model } from "tiktoken";+import { afterAll, describe, expect, it } from "vitest";++import { type Model } from "@langfuse/shared";++import { tokenCount } from "../features/tokenisation/usage";++const gpt4o = {+ id: "model-gpt-4o",+ tokenizerId: "openai",+ tokenizerConfig: {+ tokenizerModel: "gpt-4o",+ tokensPerMessage: 3,+ tokensPerName: 1,+ },+} as unknown as Model;++const encoding = encoding_for_model("gpt-4o");++afterAll(() => {+ encoding.free();+});++describe("token count of text outside the basic plane", () => {+ // Text was passed through a pass that hex-encoded astral characters before+ // counting. It never actually did so, because indexing a string yields UTF-16+ // code units and an astral character arrives as a lone surrogate. These pin+ // the counts to what the tokeniser itself reports, so a later change that+ // revives the encoding is caught rather than silently repricing every+ // observation containing an emoji.+ it.each([+ ["an emoji", "hello 😀 world"],+ ["a musical symbol", "𝄞 clef"],+ ["a multi-person emoji sequence", "👨👩👧👦 family"],+ ["a regional indicator pair", "🇮🇳 flag"],+ ["text with no astral characters", "plain ascii text"],+ ])("counts %s as the tokeniser does", (_label, text) => {+ expect(tokenCount({ model: gpt4o, text })).toBe(+ encoding.encode(text, "all").length,+ );+ });+});diff --git a/worker/src/features/tokenisation/usage.ts b/worker/src/features/tokenisation/usage.tsindex e35e329236..f43e3f5e1d 100644--- a/worker/src/features/tokenisation/usage.ts+++ b/worker/src/features/tokenisation/usage.ts@@ -189,10 +189,8 @@ const getTokensByModel = (model: TiktokenModel, text: string) => {encoding = get_encoding("cl100k_base");}- const cleandedText = unicodeToBytesInString(text);-logger.debug(`Tokenized data for model: ${model}`);- return encoding?.encode(cleandedText, "all").length;+ return encoding?.encode(text, "all").length;};interface Tokenizer {@@ -226,24 +224,3 @@ function isChatMessageArray(value: unknown): value is ChatMessage[] {(!("name" in item) || typeof item.name === "string"),);}--function unicodeToBytesInString(input: string): string {- let result = "";- for (let i = 0; i < input.length; i++) {- const char = input[i];- if (char && /[\u{10000}-\u{10FFFF}]/u.test(char)) {- const bytes = unicodeToBytes(char);- result += Array.from(bytes)- .map((b) => b.toString(16))- .join("");- } else {- result += char;- }- }- return result;-}--function unicodeToBytes(input: string): Uint8Array {- const encoder = new TextEncoder();- return encoder.encode(input);-}