An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(chat): count gpt-5.6-luna tokens with the o200k tokenizer (#1021)
Yoshiki Miura437704010d33
diff --git a/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts b/lib/streaming/helpers/__tests__/trim-cold-start-history.test.tsindex 1c98ff47..cf0cbd6d 100644--- a/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts+++ b/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts@@ -234,6 +234,30 @@ describe('trimColdStartHistory', () => {).toBe(true)})+ it('counts gpt-5.6-luna history with its own tokenizer', () => {+ const arabicText = 'مرحبا بك في هذا البحث'.repeat(20)+ const coldAt = timestamp(3 * WARM_GAP_MS + COLD_GAP_MS)+ const messages = Array.from({ length: 5 }, (_, i) => {+ const createdAt = i === 4 ? coldAt : timestamp(i * WARM_GAP_MS)+ return [+ message(`u${i}`, 'user', createdAt, [+ { type: 'text', text: arabicText }+ ]),+ message(`a${i}`, 'assistant', createdAt)+ ]+ }).flat()+ const options = { now: coldAt, limit: 1_200 }++ expect(+ trimColdStartHistory(messages, { ...options, modelId: 'gpt-4o-mini' })+ .trimmedAtCurrentTurn+ ).toBe(true)+ expect(+ trimColdStartHistory(messages, { ...options, modelId: 'gpt-5.6-luna' })+ .messages+ ).toBe(messages)+ })+it('disables trimming when the limit is zero', () => {const messages = Array.from({ length: 6 }, (_, i) =>turn(i, timestamp(i * COLD_GAP_MS))diff --git a/lib/utils/__tests__/context-window.test.ts b/lib/utils/__tests__/context-window.test.tsindex 8c64195a..19cf9fd0 100644--- a/lib/utils/__tests__/context-window.test.ts+++ b/lib/utils/__tests__/context-window.test.ts@@ -1,15 +1,29 @@import { ModelMessage } from 'ai'+import { getEncoding } from 'js-tiktoken'import { describe, expect, test } from 'vitest'import { Model } from '@/lib/types/models'import {+ countTextTokens,getMaxAllowedTokens,shouldTruncateMessages,truncateMessages} from '../context-window'describe('context-window', () => {+ describe('countTextTokens', () => {+ test('counts gpt-5.6-luna text with the o200k tokenizer', () => {+ const text = 'مرحبا بك في هذا البحث عن الطاقة المتجددة. '.repeat(50)+ const tokens = countTextTokens(text, 'gpt-5.6-luna')++ expect(tokens).toBe(getEncoding('o200k_base').encode(text).length)+ expect(tokens).toBeLessThan(+ getEncoding('cl100k_base').encode(text).length+ )+ })+ })+const mockModel: Model = {id: 'gpt-4o-mini',name: 'GPT-4o mini',diff --git a/lib/utils/context-window.ts b/lib/utils/context-window.tsindex 9dec37cf..d6549fb2 100644--- a/lib/utils/context-window.ts+++ b/lib/utils/context-window.ts@@ -51,13 +51,13 @@ const SAFETY_BUFFER_RATIO = 0.1const encoderCache = new Map<string, any>()// Mapping of our model IDs to tiktoken encoding names-// js-tiktoken supports 'cl100k_base' (for GPT-4), 'p50k_base', 'r50k_base'+// js-tiktoken supports 'o200k_base', 'cl100k_base', 'p50k_base', 'r50k_base'const MODEL_TO_ENCODING: Record<string, TiktokenEncoding> = {'gpt-4.1': 'cl100k_base','gpt-4.1-mini': 'cl100k_base','gpt-4.1-nano': 'cl100k_base','gpt-4o-mini': 'cl100k_base',- 'gpt-5.6-luna': 'cl100k_base',+ 'gpt-5.6-luna': 'o200k_base','claude-opus-4': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Claude'claude-sonnet-4': 'cl100k_base','claude-3-7-sonnet': 'cl100k_base',