Skip to content

An AI-powered search engine with a generative UI

fix(chat): count gpt-5.6-luna tokens with the o200k tokenizer (#1021)

Yoshiki Miura437704010d33
diff --git a/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts b/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts
index 1c98ff47..cf0cbd6d 100644
--- a/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts
+++ b/lib/streaming/helpers/__tests__/trim-cold-start-history.test.ts
@@ -234,6 +234,30 @@ describe('trimColdStartHistory', () => {
).toBe(true)
})
+ it('counts gpt-5.6-luna history with its own tokenizer', () => {
+ const arabicText = 'مرحبا بك في هذا البحث'.repeat(20)
+ const coldAt = timestamp(3 * WARM_GAP_MS + COLD_GAP_MS)
+ const messages = Array.from({ length: 5 }, (_, i) => {
+ const createdAt = i === 4 ? coldAt : timestamp(i * WARM_GAP_MS)
+ return [
+ message(`u${i}`, 'user', createdAt, [
+ { type: 'text', text: arabicText }
+ ]),
+ message(`a${i}`, 'assistant', createdAt)
+ ]
+ }).flat()
+ const options = { now: coldAt, limit: 1_200 }
+
+ expect(
+ trimColdStartHistory(messages, { ...options, modelId: 'gpt-4o-mini' })
+ .trimmedAtCurrentTurn
+ ).toBe(true)
+ expect(
+ trimColdStartHistory(messages, { ...options, modelId: 'gpt-5.6-luna' })
+ .messages
+ ).toBe(messages)
+ })
+
it('disables trimming when the limit is zero', () => {
const messages = Array.from({ length: 6 }, (_, i) =>
turn(i, timestamp(i * COLD_GAP_MS))
diff --git a/lib/utils/__tests__/context-window.test.ts b/lib/utils/__tests__/context-window.test.ts
index 8c64195a..19cf9fd0 100644
--- a/lib/utils/__tests__/context-window.test.ts
+++ b/lib/utils/__tests__/context-window.test.ts
@@ -1,15 +1,29 @@
import { ModelMessage } from 'ai'
+import { getEncoding } from 'js-tiktoken'
import { describe, expect, test } from 'vitest'
import { Model } from '@/lib/types/models'
import {
+ countTextTokens,
getMaxAllowedTokens,
shouldTruncateMessages,
truncateMessages
} from '../context-window'
describe('context-window', () => {
+ describe('countTextTokens', () => {
+ test('counts gpt-5.6-luna text with the o200k tokenizer', () => {
+ const text = 'مرحبا بك في هذا البحث عن الطاقة المتجددة. '.repeat(50)
+ const tokens = countTextTokens(text, 'gpt-5.6-luna')
+
+ expect(tokens).toBe(getEncoding('o200k_base').encode(text).length)
+ expect(tokens).toBeLessThan(
+ getEncoding('cl100k_base').encode(text).length
+ )
+ })
+ })
+
const mockModel: Model = {
id: 'gpt-4o-mini',
name: 'GPT-4o mini',
diff --git a/lib/utils/context-window.ts b/lib/utils/context-window.ts
index 9dec37cf..d6549fb2 100644
--- a/lib/utils/context-window.ts
+++ b/lib/utils/context-window.ts
@@ -51,13 +51,13 @@ const SAFETY_BUFFER_RATIO = 0.1
const encoderCache = new Map<string, any>()
// Mapping of our model IDs to tiktoken encoding names
-// js-tiktoken supports 'cl100k_base' (for GPT-4), 'p50k_base', 'r50k_base'
+// js-tiktoken supports 'o200k_base', 'cl100k_base', 'p50k_base', 'r50k_base'
const MODEL_TO_ENCODING: Record<string, TiktokenEncoding> = {
'gpt-4.1': 'cl100k_base',
'gpt-4.1-mini': 'cl100k_base',
'gpt-4.1-nano': 'cl100k_base',
'gpt-4o-mini': 'cl100k_base',
- 'gpt-5.6-luna': 'cl100k_base',
+ 'gpt-5.6-luna': 'o200k_base',
'claude-opus-4': 'cl100k_base', // Use GPT-4 tokenizer as approximation for Claude
'claude-sonnet-4': 'cl100k_base',
'claude-3-7-sonnet': 'cl100k_base',