OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(api): AI chat follow-ups no longer fail on Claude models with an invalid thinking signature (#595)
Carl-Gerhard Lindesvärdde94ef95ae58
diff --git a/apps/start/e2e/chat.spec.ts b/apps/start/e2e/chat.spec.tsindex 7f1a6d900..c868bae54 100644--- a/apps/start/e2e/chat.spec.ts+++ b/apps/start/e2e/chat.spec.ts@@ -1,4 +1,17 @@+import {+ type ChatModelEntry,+ MODEL_STORAGE_KEY,+} from '@openpanel/core/modules/assistant/assistant.constants';import { expect as baseExpect, test } from './fixtures';+import { shot, trpcQuery } from './org-helpers';++/** The model list the API offers the dashboard, and whether a self-hoster may pick one. */+async function chatModels(page: Parameters<typeof trpcQuery>[0]) {+ return trpcQuery<{+ models: ChatModelEntry[];+ canChooseModel: boolean;+ }>(page, 'chat.models');+}// A real model answers the question, so the budget is generous.const expect = baseExpect.configure({ timeout: 60_000 });@@ -79,4 +92,62 @@ test.describe('AI chat', () => {// The list shows the chat under "Today" with a generated title.await expect(page.getByText('Today', { exact: true })).toBeVisible();});++ // The history of a turn that ran tools is replayed to the model on the next+ // turn; it has to come back from storage exactly as the model produced it.+ // Claude rejects a replayed turn whose stored parts differ from what it+ // produced, so the test runs on Claude whenever the API offers it and on the+ // default model otherwise (the storage round trip matters to both).+ test('a follow-up after a tool-using answer gets an answer', async ({+ page,+ }) => {+ const { models, canChooseModel } = await chatModels(page);+ const claude = canChooseModel+ ? models.find((model) => model.provider === 'anthropic')+ : undefined;+ test.info().annotations.push({+ type: 'model',+ description: claude?.id ?? 'default',+ });+ if (claude) {+ await page.addInitScript(+ ([key, id]) => window.localStorage.setItem(key, id),+ [MODEL_STORAGE_KEY, claude.id]+ );+ }+ await page.goto(CHAT);+ if (await page.getByText(NOT_CONFIGURED).isVisible()) {+ test.skip();+ return;+ }+ if (claude) {+ await expect(+ page.getByRole('button', { name: 'Select model' })+ ).toHaveText(new RegExp(claude.label));+ }+ const input = page.getByRole('textbox', { name: 'Message' });+ const send = page.getByRole('button', { name: 'Send message' });+ await input.fill(+ 'Show me the top 3 pages by views in the last 7 days as a bar chart'+ );+ await input.press('Enter');+ await expect(page).toHaveURL(/\/chat\?.*c=/);+ await expect(send).toBeVisible({ timeout: 120_000 });+ // The first answer must have run the chart tool, or the follow-up proves nothing.+ await expect(+ page.getByRole('button', { name: 'Expand chart' }).first()+ ).toBeVisible();++ await input.fill('And which countries do those views come from?');+ await input.press('Enter');+ await expect(+ page.getByRole('button', { name: 'Stop generating' })+ ).toBeVisible();+ await expect(send).toBeVisible({ timeout: 120_000 });+ await shot(page, 'chat-follow-up');+ await expect(+ page.getByText(/Something went wrong|Invalid `signature`/)+ ).toHaveCount(0);+ await expect(page.getByText(/countr/i).nth(1)).toBeVisible();+ });});diff --git a/packages/core/src/modules/assistant/src/persistence.postgres.test.ts b/packages/core/src/modules/assistant/src/persistence.postgres.test.tsnew file mode 100644index 000000000..ee76cdae7--- /dev/null+++ b/packages/core/src/modules/assistant/src/persistence.postgres.test.ts@@ -0,0 +1,77 @@+import { afterAll, beforeAll, expect, test } from 'bun:test';+import { testServiceDeps } from '../../../../test/service-deps';+import type { ServiceDeps } from '../../../services';+import type { AssistantUIMessage } from './message';+import { loadMessages, saveMessages } from './persistence';++const ORGANIZATION_ID = 'chat-persistence-org';+const PROJECT_ID = 'chat-persistence-project';+const USER_ID = 'chat-persistence-user';+const CONVERSATION_ID = 'chat-persistence-conversation';++let deps: ServiceDeps;++beforeAll(async () => {+ deps = await testServiceDeps();+ await deps.db.organization.upsert({+ where: { id: ORGANIZATION_ID },+ create: { id: ORGANIZATION_ID, name: 'Chat org', timezone: 'UTC' },+ update: {},+ });+ await deps.db.project.upsert({+ where: { id: PROJECT_ID },+ create: { id: PROJECT_ID, name: 'Chat', organizationId: ORGANIZATION_ID },+ update: {},+ });+ await deps.db.user.upsert({+ where: { id: USER_ID },+ create: { id: USER_ID, email: '[email protected]' },+ update: {},+ });+});++afterAll(async () => {+ await deps.db.conversation.deleteMany({ where: { id: CONVERSATION_ID } });+ await deps.db.project.deleteMany({ where: { id: PROJECT_ID } });+ await deps.db.organization.deleteMany({ where: { id: ORGANIZATION_ID } });+ await deps.db.user.deleteMany({ where: { id: USER_ID } });+});++// Anthropic binds a thinking block to the bytes of the conversation before+// it, so a tool input or output must come back exactly as the model saw it.+test('message parts survive storage byte for byte, including key order', async () => {+ const message: AssistantUIMessage = {+ id: 'chat-persistence-assistant',+ role: 'assistant',+ parts: [+ { type: 'step-start' },+ {+ type: 'reasoning',+ text: 'Look at pages first.',+ providerMetadata: { anthropic: { signature: 'sig-1' } },+ },+ {+ type: 'tool-get_top_pages',+ toolCallId: 'call-1',+ state: 'output-available',+ input: { startDate: '2026-10-01', endDate: '2026-10-23', limit: 5 },+ output: {+ rows: [{ views: 12, path: '/pricing' }],+ longerKeyAfterShort: true,+ a: 1,+ },+ } as unknown as AssistantUIMessage['parts'][number],+ ],+ };++ await saveMessages(deps, {+ conversationId: CONVERSATION_ID,+ projectId: PROJECT_ID,+ organizationId: ORGANIZATION_ID,+ userId: USER_ID,+ messages: [message],+ });+ const [loaded] = await loadMessages(deps, CONVERSATION_ID);++ expect(JSON.stringify(loaded?.parts)).toBe(JSON.stringify(message.parts));+});diff --git a/packages/core/src/modules/assistant/src/providers.test.ts b/packages/core/src/modules/assistant/src/providers.test.tsnew file mode 100644index 000000000..d08f9b7dc--- /dev/null+++ b/packages/core/src/modules/assistant/src/providers.test.ts@@ -0,0 +1,57 @@+import { afterAll, beforeAll, expect, test } from 'bun:test';+import { streamText } from 'ai';+import { testCoreConfig } from '../../../../test/config-fixture';+import { CHAT_MODELS } from '../assistant.constants';+import { reasoningProviderOptions, resolveChatModel } from './providers';++const NOT_FOUND = 404;++let server: ReturnType<typeof Bun.serve>;+let captured: { body: Record<string, unknown>; headers: Headers } | null = null;++beforeAll(() => {+ server = Bun.serve({+ port: 0,+ fetch: async (request) => {+ captured = {+ body: (await request.json()) as Record<string, unknown>,+ headers: request.headers,+ };+ return new Response('', { status: NOT_FOUND });+ },+ });+});++afterAll(() => {+ server.stop(true);+});++// A thinking block replayed after any change to the conversation before it is+// a 400 unless the request opts into dropping it instead.+test('a Claude turn asks Anthropic to drop stale thinking blocks', async () => {+ const config = testCoreConfig();+ config.ai.anthropic.apiKey = 'test-key';+ config.ai.anthropic.baseUrl = server.url.origin;+ const entry = CHAT_MODELS.find((model) => model.provider === 'anthropic');+ if (!entry) {+ throw new Error('no Anthropic chat model in the catalog');+ }++ const result = streamText({+ model: resolveChatModel(config, entry),+ prompt: 'hi',+ providerOptions: reasoningProviderOptions(entry),+ onError: () => undefined,+ });+ // The stub answers 404 on purpose; only the request matters.+ await result.consumeStream();++ expect(captured?.body.thinking).toEqual({+ type: 'adaptive',+ display: 'summarized',+ block_binding: { prefix_mismatch_behavior: 'drop_block' },+ });+ expect(captured?.headers.get('anthropic-beta')?.split(',')).toContain(+ 'thinking-binding-controls-2026-08-01'+ );+});diff --git a/packages/core/src/modules/assistant/src/providers.ts b/packages/core/src/modules/assistant/src/providers.tsindex b19242e8f..1d17a2c7a 100644--- a/packages/core/src/modules/assistant/src/providers.ts+++ b/packages/core/src/modules/assistant/src/providers.ts@@ -93,7 +93,14 @@ type ProviderOptions = NonNullable<Parameters<typeof streamText>[0]['providerOptions']>;-/** Reasoning summaries are off by default on both providers; the chat shows them. */+/**+ * Reasoning summaries are off by default on both providers; the chat shows them.+ *+ * Anthropic binds each thinking block to the conversation prefix that produced+ * it and rejects the whole request when that prefix changed (the system prompt+ * carries today's date and the launch hint, so it can). `drop_block` makes the+ * API drop the stale blocks instead of failing the turn.+ */export function reasoningProviderOptions(entry: ChatModelEntry): ProviderOptions {@@ -102,7 +109,11 @@ export function reasoningProviderOptions(}return {anthropic: {- thinking: { type: 'adaptive' as const, display: 'summarized' as const },+ thinking: {+ type: 'adaptive' as const,+ display: 'summarized' as const,+ blockBinding: { prefixMismatchBehavior: 'drop_block' as const },+ },},};}diff --git a/packages/db/prisma/migrations/20261023120000_chat_message_parts_json/migration.sql b/packages/db/prisma/migrations/20261023120000_chat_message_parts_json/migration.sqlnew file mode 100644index 000000000..b13d2492f--- /dev/null+++ b/packages/db/prisma/migrations/20261023120000_chat_message_parts_json/migration.sql@@ -0,0 +1,4 @@+-- jsonb reorders object keys and normalizes numbers, so a replayed turn no+-- longer matched what the model saw and Anthropic rejected its thinking+-- signatures. json stores the parts verbatim.+ALTER TABLE "chat_messages" ALTER COLUMN "parts" TYPE JSON USING "parts"::text::json;diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prismaindex 1c9ee0b64..b5860f969 100644--- a/packages/db/prisma/schema.prisma+++ b/packages/db/prisma/schema.prisma@@ -57,8 +57,9 @@ model ChatMessage {conversationId Stringconversation Conversation @relation(fields: [conversationId], references: [id], onDelete: Cascade)role String+ // json, not jsonb: the model replays these verbatim and jsonb reorders keys./// [IPrismaUIMessageParts]- parts Json+ parts Json @db.Json/// [IPrismaChatMessageMetadata]metadata Json?createdAt DateTime @default(now())