Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy
Rust
share normalization between request and count paths
Raine Virta1a33a6f9ba4c
diff --git a/src/count-tokens.ts b/src/count-tokens.tsindex 217fd3d..e976d24 100644--- a/src/count-tokens.ts+++ b/src/count-tokens.ts@@ -1,38 +1,35 @@import { encode } from "gpt-tokenizer/model/gpt-4o"import type { AnthropicRequest } from "./anthropic/schema.ts"+import { buildInstructions, normalizeContent, toolResultToString } from "./translate/request.ts"export function countTokens(req: AnthropicRequest): number {let total = 0- if (req.system) {- const text =- typeof req.system === "string"- ? req.system- : req.system.map((b) => b.text || "").join("\n")- total += encode(text).length- }+ const instructions = buildInstructions(req.system)+ if (instructions) total += encode(instructions).length+for (const msg of req.messages) {- if (typeof msg.content === "string") {- total += encode(msg.content).length- continue- }- for (const block of msg.content) {- if (block.type === "text") total += encode(block.text).length- else if (block.type === "tool_use") total += encode(JSON.stringify(block.input ?? {})).length + encode(block.name).length- else if (block.type === "tool_result") {- const text =- typeof block.content === "string"- ? block.content- : block.content.map((b) => (b.type === "text" ? b.text : "")).join("\n")- total += encode(text).length+ const blocks = normalizeContent(msg.content)+ for (const block of blocks) {+ if (block.type === "text") {+ total += encode(block.text).length+ } else if (block.type === "image") {+ const value = block.source.type === "url" ? block.source.url : `data:${block.source.media_type};base64,${block.source.data}`+ total += encode(value).length+ } else if (block.type === "tool_use") {+ total += encode(block.name).length+ total += encode(JSON.stringify(block.input ?? {})).length+ } else if (block.type === "tool_result") {+ total += encode(toolResultToString(block.content)).length}}}+for (const tool of req.tools ?? []) {total += encode(tool.name).lengthif (tool.description) total += encode(tool.description).lengthtotal += encode(JSON.stringify(tool.input_schema ?? {})).length}- // Rough per-message overhead+total += req.messages.length * 4return total}diff --git a/src/translate/request.ts b/src/translate/request.tsindex 326455e..ec5a393 100644--- a/src/translate/request.ts+++ b/src/translate/request.ts@@ -117,7 +117,7 @@ function mapToolChoice(}}-function buildInstructions(system: AnthropicRequest["system"]): string | undefined {+export function buildInstructions(system: AnthropicRequest["system"]): string | undefined {if (!system) return undefinedconst blocks: AnthropicTextBlock[] =typeof system === "string" ? [{ type: "text", text: system }] : system@@ -181,7 +181,7 @@ function buildInput(messages: AnthropicMessage[]): ResponsesInputItem[] {return out}-function normalizeContent(content: AnthropicMessage["content"]): AnthropicContentBlock[] {+export function normalizeContent(content: AnthropicMessage["content"]): AnthropicContentBlock[] {if (typeof content === "string") return [{ type: "text", text: content }]return content}@@ -191,7 +191,7 @@ function imageToUrl(block: Extract<AnthropicContentBlock, { type: "image" }>): sreturn `data:${block.source.media_type};base64,${block.source.data}`}-function toolResultToString(+export function toolResultToString(content: string | Array<AnthropicTextBlock | AnthropicImageBlock>,): string {if (typeof content === "string") return content