Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

6.6 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-use-before-define */
import { Model, parseJsonPrioritised } from "@langfuse/shared";
import { isChatModel, isTiktokenModel } from "./types";
import { countTokens } from "@anthropic-ai/tokenizer";

import {
  type TiktokenModel,
  type Tiktoken,
  get_encoding,
  encoding_for_model,
} from "tiktoken";

import { z } from "zod";
import { logger, recordIncrement } from "@langfuse/shared/src/server";

const OpenAiTokenConfig = z.object({
  tokenizerModel: z.string().refine(isTiktokenModel, {
    message: "Unknown tiktoken model",
  }),
});

const OpenAiChatTokenConfig = z.object({
  tokenizerModel: z
    .string()
    .refine((m) => isTiktokenModel(m) && isChatModel(m), {
      message: "Chat model expected",
    }),
  tokensPerMessage: z.number(),
  tokensPerName: z.number(),
});

// An invalid tokenizer config is a static, per-model configuration problem, not
// a per-event condition. Log once per model id so the signal survives without
// one line per observation. tokenCount runs inside the token-count worker-thread
// pool (each pool worker is its own module instance), so this dedup is per
// worker-thread, not strictly per process — still a collapse from per-event to a
// handful of lines per pod.
const warnedInvalidTokenizerConfigModelIds = new Set<string>();

export function tokenCount(p: {
  model: Model;
  text: unknown;
}): number | undefined {
  if (
    p.text === null ||
    p.text === undefined ||
    (Array.isArray(p.text) && p.text.length === 0)
  ) {
    return undefined;
  }

  if (p.model.tokenizerId === "openai") {
    return openAiTokenCount({
      model: p.model,
      text: p.text,
    });
  } else if (p.model.tokenizerId === "claude") {
    return claudeTokenCount(p.text);
  }
  if (p.model.tokenizerId) {
    logger.error(`Unknown tokenizer ${p.model.tokenizerId}`);
  }
  return undefined;
}

type ChatMessage = {
  role: string;
  name?: string;
  content: string;
};

function openAiTokenCount(p: { model: Model; text: unknown }) {
  const config = OpenAiTokenConfig.safeParse(p.model.tokenizerConfig);
  if (!config.success) {
    // Counter fires per occurrence so the ongoing rate stays visible; the log
    // line dedupes to once per model id (the config itself is static).
    recordIncrement("langfuse.ingestion.tokenisation.invalid_config", 1, {
      tokenizer: "openai",
    });
    if (!warnedInvalidTokenizerConfigModelIds.has(p.model.id)) {
      warnedInvalidTokenizerConfigModelIds.add(p.model.id);
      logger.warn(
        `Invalid tokenizer config for model ${p.model.id}: ${JSON.stringify(
          p.model.tokenizerConfig,
        )}, ${JSON.stringify(config.error)}`,
      );
    }
    return undefined;
  }

  let result = undefined;
  const parsedText =
    typeof p.text === "string" ? parseJsonPrioritised(p.text) : p.text; // Clickhouse stores ChatMessage array as string

  if (
    isChatMessageArray(parsedText) &&
    isChatModel(config.data.tokenizerModel)
  ) {
    // check if the tokenizerConfig is a valid chat config
    const parsedConfig = OpenAiChatTokenConfig.safeParse(
      p.model.tokenizerConfig,
    );
    if (!parsedConfig.success) {
      recordIncrement("langfuse.ingestion.tokenisation.invalid_config", 1, {
        tokenizer: "openai_chat",
      });
      if (!warnedInvalidTokenizerConfigModelIds.has(p.model.id)) {
        warnedInvalidTokenizerConfigModelIds.add(p.model.id);
        logger.error(
          `Invalid tokenizer config for chat model ${
            p.model.id
          }: ${JSON.stringify(p.model.tokenizerConfig)}`,
        );
      }
      return undefined;
    }
    result = openAiChatTokenCount({
      messages: parsedText,
      config: parsedConfig.data,
    });
  } else {
    result = isString(parsedText)
      ? getTokensByModel(
          config.data.tokenizerModel as TiktokenModel,
          parsedText,
        )
      : getTokensByModel(
          config.data.tokenizerModel as TiktokenModel,
          JSON.stringify(parsedText),
        );
  }
  return result;
}

function claudeTokenCount(text: unknown) {
  return isString(text) ? countTokens(text) : countTokens(JSON.stringify(text));
}

function openAiChatTokenCount(params: {
  messages: ChatMessage[];
  config: z.infer<typeof OpenAiChatTokenConfig>;
}) {
  const model = params.config.tokenizerModel;
  if (!isTiktokenModel(model)) return undefined;

  let numTokens = 0;
  params.messages.forEach((message) => {
    numTokens += params.config.tokensPerMessage;

    Object.keys(message).forEach((key) => {
      const value = message[key as keyof typeof message];
      if (
        // check API docs for available keys: https://platform.openai.com/docs/api-reference/chat/create?lang=node.js
        // memory access out of bounds error in tiktoken if unexpected key and value of type boolean
        // expected keys with booleans work
        value &&
        [
          "content",
          "role",
          "name",
          "tool_calls",
          "function_call",
          "toolCalls",
          "functionCall",
        ].some((k) => k === key)
      ) {
        const tokens = getTokensByModel(model, value);
        if (tokens) numTokens += tokens;
      }
      if (key === "name") {
        numTokens += params.config.tokensPerName;
      }
    });
  });
  numTokens += 3; // every reply is primed with <| start |> assistant <| message |>

  return numTokens;
}

const getTokensByModel = (model: TiktokenModel, text: string) => {
  // encoding should be kept in memory to avoid re-creating it
  let encoding: Tiktoken | undefined;
  try {
    cachedTokenizerByModel[model] =
      cachedTokenizerByModel[model] || encoding_for_model(model);

    encoding = cachedTokenizerByModel[model];
  } catch {
    logger.warn("Model not found. Using cl100k_base encoding.");

    encoding = get_encoding("cl100k_base");
  }
  logger.debug(`Tokenized data for model: ${model}`);
  return encoding?.encode(text, "all").length;
};

interface Tokenizer {
  [model: string]: Tiktoken;
}
const cachedTokenizerByModel: Tokenizer = {};

export function freeAllTokenizers() {
  Object.values(cachedTokenizerByModel).forEach((tokenizer) => {
    tokenizer.free();
  });
}

function isString(value: unknown): value is string {
  return typeof value === "string";
}

function isChatMessageArray(value: unknown): value is ChatMessage[] {
  if (!Array.isArray(value)) {
    return false;
  }

  return value.every(
    (item: unknown) =>
      typeof item === "object" &&
      item !== null &&
      "role" in item &&
      typeof item.role === "string" &&
      "content" in item &&
      typeof item.content === "string" &&
      (!("name" in item) || typeof item.name === "string"),
  );
}