Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(api): AI chat follow-ups no longer fail on Claude models with an invalid thinking signature (#595)

Carl-Gerhard Lindesvärdde94ef95ae58
diff --git a/apps/start/e2e/chat.spec.ts b/apps/start/e2e/chat.spec.ts
index 7f1a6d900..c868bae54 100644
--- a/apps/start/e2e/chat.spec.ts
+++ b/apps/start/e2e/chat.spec.ts
@@ -1,4 +1,17 @@
+import {
+ type ChatModelEntry,
+ MODEL_STORAGE_KEY,
+} from '@openpanel/core/modules/assistant/assistant.constants';
import { expect as baseExpect, test } from './fixtures';
+import { shot, trpcQuery } from './org-helpers';
+
+/** The model list the API offers the dashboard, and whether a self-hoster may pick one. */
+async function chatModels(page: Parameters<typeof trpcQuery>[0]) {
+ return trpcQuery<{
+ models: ChatModelEntry[];
+ canChooseModel: boolean;
+ }>(page, 'chat.models');
+}
// A real model answers the question, so the budget is generous.
const expect = baseExpect.configure({ timeout: 60_000 });
@@ -79,4 +92,62 @@ test.describe('AI chat', () => {
// The list shows the chat under "Today" with a generated title.
await expect(page.getByText('Today', { exact: true })).toBeVisible();
});
+
+ // The history of a turn that ran tools is replayed to the model on the next
+ // turn; it has to come back from storage exactly as the model produced it.
+ // Claude rejects a replayed turn whose stored parts differ from what it
+ // produced, so the test runs on Claude whenever the API offers it and on the
+ // default model otherwise (the storage round trip matters to both).
+ test('a follow-up after a tool-using answer gets an answer', async ({
+ page,
+ }) => {
+ const { models, canChooseModel } = await chatModels(page);
+ const claude = canChooseModel
+ ? models.find((model) => model.provider === 'anthropic')
+ : undefined;
+ test.info().annotations.push({
+ type: 'model',
+ description: claude?.id ?? 'default',
+ });
+ if (claude) {
+ await page.addInitScript(
+ ([key, id]) => window.localStorage.setItem(key, id),
+ [MODEL_STORAGE_KEY, claude.id]
+ );
+ }
+ await page.goto(CHAT);
+ if (await page.getByText(NOT_CONFIGURED).isVisible()) {
+ test.skip();
+ return;
+ }
+ if (claude) {
+ await expect(
+ page.getByRole('button', { name: 'Select model' })
+ ).toHaveText(new RegExp(claude.label));
+ }
+ const input = page.getByRole('textbox', { name: 'Message' });
+ const send = page.getByRole('button', { name: 'Send message' });
+ await input.fill(
+ 'Show me the top 3 pages by views in the last 7 days as a bar chart'
+ );
+ await input.press('Enter');
+ await expect(page).toHaveURL(/\/chat\?.*c=/);
+ await expect(send).toBeVisible({ timeout: 120_000 });
+ // The first answer must have run the chart tool, or the follow-up proves nothing.
+ await expect(
+ page.getByRole('button', { name: 'Expand chart' }).first()
+ ).toBeVisible();
+
+ await input.fill('And which countries do those views come from?');
+ await input.press('Enter');
+ await expect(
+ page.getByRole('button', { name: 'Stop generating' })
+ ).toBeVisible();
+ await expect(send).toBeVisible({ timeout: 120_000 });
+ await shot(page, 'chat-follow-up');
+ await expect(
+ page.getByText(/Something went wrong|Invalid `signature`/)
+ ).toHaveCount(0);
+ await expect(page.getByText(/countr/i).nth(1)).toBeVisible();
+ });
});
diff --git a/packages/core/src/modules/assistant/src/persistence.postgres.test.ts b/packages/core/src/modules/assistant/src/persistence.postgres.test.ts
new file mode 100644
index 000000000..ee76cdae7
--- /dev/null
+++ b/packages/core/src/modules/assistant/src/persistence.postgres.test.ts
@@ -0,0 +1,77 @@
+import { afterAll, beforeAll, expect, test } from 'bun:test';
+import { testServiceDeps } from '../../../../test/service-deps';
+import type { ServiceDeps } from '../../../services';
+import type { AssistantUIMessage } from './message';
+import { loadMessages, saveMessages } from './persistence';
+
+const ORGANIZATION_ID = 'chat-persistence-org';
+const PROJECT_ID = 'chat-persistence-project';
+const USER_ID = 'chat-persistence-user';
+const CONVERSATION_ID = 'chat-persistence-conversation';
+
+let deps: ServiceDeps;
+
+beforeAll(async () => {
+ deps = await testServiceDeps();
+ await deps.db.organization.upsert({
+ where: { id: ORGANIZATION_ID },
+ create: { id: ORGANIZATION_ID, name: 'Chat org', timezone: 'UTC' },
+ update: {},
+ });
+ await deps.db.project.upsert({
+ where: { id: PROJECT_ID },
+ create: { id: PROJECT_ID, name: 'Chat', organizationId: ORGANIZATION_ID },
+ update: {},
+ });
+ await deps.db.user.upsert({
+ where: { id: USER_ID },
+ create: { id: USER_ID, email: '[email protected]' },
+ update: {},
+ });
+});
+
+afterAll(async () => {
+ await deps.db.conversation.deleteMany({ where: { id: CONVERSATION_ID } });
+ await deps.db.project.deleteMany({ where: { id: PROJECT_ID } });
+ await deps.db.organization.deleteMany({ where: { id: ORGANIZATION_ID } });
+ await deps.db.user.deleteMany({ where: { id: USER_ID } });
+});
+
+// Anthropic binds a thinking block to the bytes of the conversation before
+// it, so a tool input or output must come back exactly as the model saw it.
+test('message parts survive storage byte for byte, including key order', async () => {
+ const message: AssistantUIMessage = {
+ id: 'chat-persistence-assistant',
+ role: 'assistant',
+ parts: [
+ { type: 'step-start' },
+ {
+ type: 'reasoning',
+ text: 'Look at pages first.',
+ providerMetadata: { anthropic: { signature: 'sig-1' } },
+ },
+ {
+ type: 'tool-get_top_pages',
+ toolCallId: 'call-1',
+ state: 'output-available',
+ input: { startDate: '2026-10-01', endDate: '2026-10-23', limit: 5 },
+ output: {
+ rows: [{ views: 12, path: '/pricing' }],
+ longerKeyAfterShort: true,
+ a: 1,
+ },
+ } as unknown as AssistantUIMessage['parts'][number],
+ ],
+ };
+
+ await saveMessages(deps, {
+ conversationId: CONVERSATION_ID,
+ projectId: PROJECT_ID,
+ organizationId: ORGANIZATION_ID,
+ userId: USER_ID,
+ messages: [message],
+ });
+ const [loaded] = await loadMessages(deps, CONVERSATION_ID);
+
+ expect(JSON.stringify(loaded?.parts)).toBe(JSON.stringify(message.parts));
+});
diff --git a/packages/core/src/modules/assistant/src/providers.test.ts b/packages/core/src/modules/assistant/src/providers.test.ts
new file mode 100644
index 000000000..d08f9b7dc
--- /dev/null
+++ b/packages/core/src/modules/assistant/src/providers.test.ts
@@ -0,0 +1,57 @@
+import { afterAll, beforeAll, expect, test } from 'bun:test';
+import { streamText } from 'ai';
+import { testCoreConfig } from '../../../../test/config-fixture';
+import { CHAT_MODELS } from '../assistant.constants';
+import { reasoningProviderOptions, resolveChatModel } from './providers';
+
+const NOT_FOUND = 404;
+
+let server: ReturnType<typeof Bun.serve>;
+let captured: { body: Record<string, unknown>; headers: Headers } | null = null;
+
+beforeAll(() => {
+ server = Bun.serve({
+ port: 0,
+ fetch: async (request) => {
+ captured = {
+ body: (await request.json()) as Record<string, unknown>,
+ headers: request.headers,
+ };
+ return new Response('', { status: NOT_FOUND });
+ },
+ });
+});
+
+afterAll(() => {
+ server.stop(true);
+});
+
+// A thinking block replayed after any change to the conversation before it is
+// a 400 unless the request opts into dropping it instead.
+test('a Claude turn asks Anthropic to drop stale thinking blocks', async () => {
+ const config = testCoreConfig();
+ config.ai.anthropic.apiKey = 'test-key';
+ config.ai.anthropic.baseUrl = server.url.origin;
+ const entry = CHAT_MODELS.find((model) => model.provider === 'anthropic');
+ if (!entry) {
+ throw new Error('no Anthropic chat model in the catalog');
+ }
+
+ const result = streamText({
+ model: resolveChatModel(config, entry),
+ prompt: 'hi',
+ providerOptions: reasoningProviderOptions(entry),
+ onError: () => undefined,
+ });
+ // The stub answers 404 on purpose; only the request matters.
+ await result.consumeStream();
+
+ expect(captured?.body.thinking).toEqual({
+ type: 'adaptive',
+ display: 'summarized',
+ block_binding: { prefix_mismatch_behavior: 'drop_block' },
+ });
+ expect(captured?.headers.get('anthropic-beta')?.split(',')).toContain(
+ 'thinking-binding-controls-2026-08-01'
+ );
+});
diff --git a/packages/core/src/modules/assistant/src/providers.ts b/packages/core/src/modules/assistant/src/providers.ts
index b19242e8f..1d17a2c7a 100644
--- a/packages/core/src/modules/assistant/src/providers.ts
+++ b/packages/core/src/modules/assistant/src/providers.ts
@@ -93,7 +93,14 @@ type ProviderOptions = NonNullable<
Parameters<typeof streamText>[0]['providerOptions']
>;
-/** Reasoning summaries are off by default on both providers; the chat shows them. */
+/**
+ * Reasoning summaries are off by default on both providers; the chat shows them.
+ *
+ * Anthropic binds each thinking block to the conversation prefix that produced
+ * it and rejects the whole request when that prefix changed (the system prompt
+ * carries today's date and the launch hint, so it can). `drop_block` makes the
+ * API drop the stale blocks instead of failing the turn.
+ */
export function reasoningProviderOptions(
entry: ChatModelEntry
): ProviderOptions {
@@ -102,7 +109,11 @@ export function reasoningProviderOptions(
}
return {
anthropic: {
- thinking: { type: 'adaptive' as const, display: 'summarized' as const },
+ thinking: {
+ type: 'adaptive' as const,
+ display: 'summarized' as const,
+ blockBinding: { prefixMismatchBehavior: 'drop_block' as const },
+ },
},
};
}
diff --git a/packages/db/prisma/migrations/20261023120000_chat_message_parts_json/migration.sql b/packages/db/prisma/migrations/20261023120000_chat_message_parts_json/migration.sql
new file mode 100644
index 000000000..b13d2492f
--- /dev/null
+++ b/packages/db/prisma/migrations/20261023120000_chat_message_parts_json/migration.sql
@@ -0,0 +1,4 @@
+-- jsonb reorders object keys and normalizes numbers, so a replayed turn no
+-- longer matched what the model saw and Anthropic rejected its thinking
+-- signatures. json stores the parts verbatim.
+ALTER TABLE "chat_messages" ALTER COLUMN "parts" TYPE JSON USING "parts"::text::json;
diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma
index 1c9ee0b64..b5860f969 100644
--- a/packages/db/prisma/schema.prisma
+++ b/packages/db/prisma/schema.prisma
@@ -57,8 +57,9 @@ model ChatMessage {
conversationId String
conversation Conversation @relation(fields: [conversationId], references: [id], onDelete: Cascade)
role String
+ // json, not jsonb: the model replays these verbatim and jsonb reorders keys.
/// [IPrismaUIMessageParts]
- parts Json
+ parts Json @db.Json
/// [IPrismaChatMessageMetadata]
metadata Json?
createdAt DateTime @default(now())