Skip to content

Build your personal knowledge base with Trilium Notes

fix(llm): cancel the completion and its web search when a turn is stopped

Elian Doran1cc43142ca74
diff --git a/packages/trilium-core/src/services/llm/chat.spec.ts b/packages/trilium-core/src/services/llm/chat.spec.ts
index 9876ecb852..59e7dfe189 100644
--- a/packages/trilium-core/src/services/llm/chat.spec.ts
+++ b/packages/trilium-core/src/services/llm/chat.spec.ts
@@ -13,14 +13,17 @@ const state = vi.hoisted(() => ({
providerIdRequested: undefined as string | undefined,
providerTypeRequested: undefined as string | undefined,
/** The config the provider's `chat()` was handed. */
- chatConfig: undefined as unknown
+ chatConfig: undefined as unknown,
+ /** The abort signal the provider's `chat()` was handed. */
+ chatSignal: undefined as AbortSignal | undefined
}));
vi.mock("./index.js", () => {
const makeProvider = () => ({
name: "fake",
- chat: (_messages: unknown, config: unknown) => {
+ chat: (_messages: unknown, config: unknown, signal?: AbortSignal) => {
state.chatConfig = config;
+ state.chatSignal = signal;
if (state.chatThrows !== undefined) throw state.chatThrows;
return {};
},
@@ -156,6 +159,12 @@ describe("runChat", () => {
expect(state.chunkSignal).toBe(controller.signal);
});
+ it("hands the abort signal to an AI SDK provider too, so its tool calls stop with the turn", async () => {
+ const controller = new AbortController();
+ await collect(HELLO, {}, controller.signal);
+ expect(state.chatSignal).toBe(controller.signal);
+ });
+
it("stops at the next chunk once aborted", async () => {
state.chunks = [{ type: "text", content: "a" }, { type: "text", content: "b" }];
const controller = new AbortController();
diff --git a/packages/trilium-core/src/services/llm/chat.ts b/packages/trilium-core/src/services/llm/chat.ts
index 764d46dcc9..4c80ff95cb 100644
--- a/packages/trilium-core/src/services/llm/chat.ts
+++ b/packages/trilium-core/src/services/llm/chat.ts
@@ -24,8 +24,8 @@ import { resolveToolRegistries } from "./tools/index.js";
* ends with a `done` chunk only when the turn completed — an errored one stops
* after its `error` chunk, so a caller that needs a terminator must add its own.
*
- * @param abortSignal stops the turn early; for a chunk-native provider it aborts
- * the agent loop itself, otherwise the stream is simply left at the next chunk.
+ * @param abortSignal stops the turn early: the provider aborts its completion and any tool
+ * call in progress, and the stream is left at the next chunk.
*/
export async function* runChat(
messages: LlmMessage[],
@@ -70,7 +70,7 @@ export async function* runChat(
// and produces LlmStreamChunks directly, including honouring the abort.
const chunks = provider.chatChunks
? provider.chatChunks(messages, config, abortSignal)
- : streamToChunks(provider.chat(messages, config), { model: modelDisplayName, provider: provider.name, pricing });
+ : streamToChunks(provider.chat(messages, config, abortSignal), { model: modelDisplayName, provider: provider.name, pricing });
for await (const chunk of chunks) {
if (abortSignal?.aborted) {
diff --git a/packages/trilium-core/src/services/llm/providers/anthropic.spec.ts b/packages/trilium-core/src/services/llm/providers/anthropic.spec.ts
index dff952a486..f8c52dce2e 100644
--- a/packages/trilium-core/src/services/llm/providers/anthropic.spec.ts
+++ b/packages/trilium-core/src/services/llm/providers/anthropic.spec.ts
@@ -197,6 +197,15 @@ describe("AnthropicProvider tool handling", () => {
expect(opts.stopWhen).toBeDefined();
});
+ it("passes the turn's abort signal to streamText, with or without extended thinking", () => {
+ const provider = new AnthropicProvider("sk-ant-test");
+ const signal = new AbortController().signal;
+ provider.chat([{ role: "user", content: "hi" }], {}, signal);
+ provider.chat([{ role: "user", content: "hi" }], { enableExtendedThinking: true }, signal);
+
+ expect(streamTextMock.mock.calls.map(([ opts ]) => (opts as any).abortSignal)).toEqual([ signal, signal ]);
+ });
+
it("defaults the thinking budget and floors maxOutputTokens when unset", () => {
const provider = new AnthropicProvider("sk-ant-test");
provider.chat([{ role: "user", content: "hi" }], {
diff --git a/packages/trilium-core/src/services/llm/providers/anthropic.ts b/packages/trilium-core/src/services/llm/providers/anthropic.ts
index f57a4dab55..735040e84f 100644
--- a/packages/trilium-core/src/services/llm/providers/anthropic.ts
+++ b/packages/trilium-core/src/services/llm/providers/anthropic.ts
@@ -125,9 +125,9 @@ export class AnthropicProvider extends BaseProvider {
/**
* Override chat to add Anthropic-specific extended thinking support.
*/
- override chat(messages: LlmMessage[], config: LlmProviderConfig): StreamResult {
+ override chat(messages: LlmMessage[], config: LlmProviderConfig, signal?: AbortSignal): StreamResult {
if (!config.enableExtendedThinking) {
- return super.chat(messages, config);
+ return super.chat(messages, config, signal);
}
const systemPrompt = this.buildSystemPrompt(messages, config);
@@ -152,6 +152,7 @@ export class AnthropicProvider extends BaseProvider {
// Reject any system message smuggled into `messages` (prompt injection guard).
allowSystemInMessages: false,
telemetry: TELEMETRY_OFF,
+ abortSignal: signal,
providerOptions: {
anthropic: { thinking }
}
diff --git a/packages/trilium-core/src/services/llm/providers/base_provider.ts b/packages/trilium-core/src/services/llm/providers/base_provider.ts
index c39e321cc3..43504e45f3 100644
--- a/packages/trilium-core/src/services/llm/providers/base_provider.ts
+++ b/packages/trilium-core/src/services/llm/providers/base_provider.ts
@@ -414,7 +414,7 @@ export abstract class BaseProvider implements LlmProvider {
return tools;
}
- chat(messages: LlmMessage[], config: LlmProviderConfig): StreamResult {
+ chat(messages: LlmMessage[], config: LlmProviderConfig, signal?: AbortSignal): StreamResult {
const systemPrompt = this.buildSystemPrompt(messages, config);
const chatMessages = this.applyNoteHint(messages.filter(m => m.role !== "system"), config);
const modelId = config.model || this.defaultModel;
@@ -432,7 +432,8 @@ export abstract class BaseProvider implements LlmProvider {
// `fullStream`, where `streamToChunks` turns it into a detailed message that
// the chat route logs — so suppress the unstructured stdout dump here.
onError: () => {},
- telemetry: TELEMETRY_OFF
+ telemetry: TELEMETRY_OFF,
+ abortSignal: signal
};
const providerOptions = this.chatProviderOptions(config);
if (providerOptions) {
diff --git a/packages/trilium-core/src/services/llm/providers/google.spec.ts b/packages/trilium-core/src/services/llm/providers/google.spec.ts
index 59cb0d3d08..2148651990 100644
--- a/packages/trilium-core/src/services/llm/providers/google.spec.ts
+++ b/packages/trilium-core/src/services/llm/providers/google.spec.ts
@@ -128,6 +128,14 @@ describe("GoogleProvider message building", () => {
// Extended thinking forwards Gemini's thinkingConfig.
expect(opts.providerOptions.google.thinkingConfig.thinkingBudget).toBe(10000);
});
+
+ it("chat() with extended thinking passes the turn's abort signal to streamText", () => {
+ const provider = new GoogleProvider("test-key");
+ const signal = new AbortController().signal;
+ provider.chat([{ role: "user", content: "hello" }], { enableExtendedThinking: true }, signal);
+
+ expect((streamTextMock.mock.calls[0][0] as any).abortSignal).toBe(signal);
+ });
});
describe("GoogleProvider tool handling", () => {
diff --git a/packages/trilium-core/src/services/llm/providers/google.ts b/packages/trilium-core/src/services/llm/providers/google.ts
index 535be91d52..0f0419975a 100644
--- a/packages/trilium-core/src/services/llm/providers/google.ts
+++ b/packages/trilium-core/src/services/llm/providers/google.ts
@@ -89,9 +89,9 @@ export class GoogleProvider extends BaseProvider {
* Override chat to add Google-specific extended thinking support.
* Gemini 2.5 uses thinkingBudget, Gemini 3.x uses thinkingLevel.
*/
- override chat(messages: LlmMessage[], config: LlmProviderConfig): StreamResult {
+ override chat(messages: LlmMessage[], config: LlmProviderConfig, signal?: AbortSignal): StreamResult {
if (!config.enableExtendedThinking) {
- return super.chat(messages, config);
+ return super.chat(messages, config, signal);
}
const systemPrompt = this.buildSystemPrompt(messages, config);
@@ -107,6 +107,7 @@ export class GoogleProvider extends BaseProvider {
// Reject any system message smuggled into `messages` (prompt injection guard).
allowSystemInMessages: false,
telemetry: TELEMETRY_OFF,
+ abortSignal: signal,
providerOptions: {
google: {
thinkingConfig: {
diff --git a/packages/trilium-core/src/services/llm/types.ts b/packages/trilium-core/src/services/llm/types.ts
index 10ec9c596b..d62179f829 100644
--- a/packages/trilium-core/src/services/llm/types.ts
+++ b/packages/trilium-core/src/services/llm/types.ts
@@ -77,10 +77,13 @@ export interface LlmProvider {
/**
* Create a streaming chat completion.
* Returns the AI SDK StreamResult which is provider-agnostic.
+ *
+ * @param signal stops the completion and any tool call in progress when the user stops the turn
*/
chat(
messages: LlmMessage[],
- config: LlmProviderConfig
+ config: LlmProviderConfig,
+ signal?: AbortSignal
): StreamResult;
/**
diff --git a/packages/trilium-core/src/services/llm/web_search.spec.ts b/packages/trilium-core/src/services/llm/web_search.spec.ts
index e0997c1cb0..ce35ffc283 100644
--- a/packages/trilium-core/src/services/llm/web_search.spec.ts
+++ b/packages/trilium-core/src/services/llm/web_search.spec.ts
@@ -127,6 +127,18 @@ describe("createWebSearchTool", () => {
respond({}, 403);
expect(await run(tool, "trilium")).toEqual({ error: expect.stringContaining("HTTP 403") });
});
+
+ it("cancels the search request when the turn is stopped", async () => {
+ const tool = createWebSearchTool({ provider: "tavily", apiKey: "tk" });
+ const controller = new AbortController();
+ respond({ results: [] });
+ await tool.execute?.({ query: "q" }, { toolCallId: "1", messages: [], context: {}, abortSignal: controller.signal });
+
+ const { signal } = request().init;
+ expect(signal?.aborted).toBe(false);
+ controller.abort();
+ expect(signal?.aborted).toBe(true);
+ });
});
describe("readWebPage", () => {
diff --git a/packages/trilium-core/src/services/llm/web_search.ts b/packages/trilium-core/src/services/llm/web_search.ts
index f5a951c8ea..a4a2cc44cc 100644
--- a/packages/trilium-core/src/services/llm/web_search.ts
+++ b/packages/trilium-core/src/services/llm/web_search.ts
@@ -43,12 +43,12 @@ export function createWebSearchTool(setup: WebSearchSetup) {
inputSchema: z.object({
query: z.string().describe("The search query")
}),
- execute: async ({ query }) => {
+ execute: async ({ query }, options) => {
try {
return {
query,
searchProvider: { type: setup.provider, name: setup.name ?? setup.provider },
- sources: await searchWeb(setup, query)
+ sources: await searchWeb(setup, query, options.abortSignal)
};
} catch (e) {
return { error: e instanceof Error ? e.message : String(e) };
@@ -58,19 +58,20 @@ export function createWebSearchTool(setup: WebSearchSetup) {
}
/**
- * Run `query` against the search provider in `setup`.
+ * Run `query` against the search provider in `setup`. `signal` cancels the request along with the
+ * chat turn that made it.
*
* A switch over literal cases, as `createProviderInstance()` is, because the provider type is
* user-controlled.
*/
-export async function searchWeb(setup: WebSearchSetup, query: string): Promise<WebSearchSource[]> {
+export async function searchWeb(setup: WebSearchSetup, query: string, signal?: AbortSignal): Promise<WebSearchSource[]> {
switch (setup.provider) {
case "brave": {
const base = setup.baseURL ?? "https://api.search.brave.com/res/v1";
const params = new URLSearchParams({ q: query, count: String(MAX_RESULTS) });
const payload = await requestJson(`${base}/web/search?${params}`, {
headers: { "Accept": "application/json", "X-Subscription-Token": setup.apiKey }
- }) as { web?: { results?: unknown[] } };
+ }, signal) as { web?: { results?: unknown[] } };
return toSources(payload.web?.results, "description");
}
case "tavily": {
@@ -78,7 +79,7 @@ export async function searchWeb(setup: WebSearchSetup, query: string): Promise<W
method: "POST",
headers: { "Content-Type": "application/json", "Authorization": `Bearer ${setup.apiKey}` },
body: JSON.stringify({ query, max_results: MAX_RESULTS })
- }) as { results?: unknown[] };
+ }, signal) as { results?: unknown[] };
return toSources(payload.results, "content");
}
case "exa": {
@@ -86,7 +87,7 @@ export async function searchWeb(setup: WebSearchSetup, query: string): Promise<W
method: "POST",
headers: { "Content-Type": "application/json", "x-api-key": setup.apiKey },
body: JSON.stringify({ query, numResults: MAX_RESULTS, contents: { text: { maxCharacters: 1000 } } })
- }) as { results?: unknown[] };
+ }, signal) as { results?: unknown[] };
return toSources(payload.results, "text");
}
case "serper": {
@@ -94,7 +95,7 @@ export async function searchWeb(setup: WebSearchSetup, query: string): Promise<W
method: "POST",
headers: { "Content-Type": "application/json", "X-API-KEY": setup.apiKey },
body: JSON.stringify({ q: query, num: MAX_RESULTS })
- }) as { organic?: unknown[] };
+ }, signal) as { organic?: unknown[] };
return toSources(payload.organic, "snippet", "link");
}
case "perplexity": {
@@ -102,7 +103,7 @@ export async function searchWeb(setup: WebSearchSetup, query: string): Promise<W
method: "POST",
headers: { "Content-Type": "application/json", "Authorization": `Bearer ${setup.apiKey}` },
body: JSON.stringify({ query, max_results: MAX_RESULTS })
- }) as { results?: unknown[] };
+ }, signal) as { results?: unknown[] };
return toSources(payload.results, "snippet");
}
case "searxng": {
@@ -112,7 +113,7 @@ export async function searchWeb(setup: WebSearchSetup, query: string): Promise<W
const params = new URLSearchParams({ q: query, format: "json" });
const payload = await requestJson(`${trimTrailingSlashes(setup.baseURL)}/search?${params}`, {
headers: { "Accept": "application/json", ...(setup.apiKey && { "Authorization": `Bearer ${setup.apiKey}` }) }
- }) as { results?: unknown[] };
+ }, signal) as { results?: unknown[] };
return toSources(payload.results, "content").slice(0, MAX_RESULTS);
}
default:
@@ -221,8 +222,9 @@ function htmlToMarkdown(html: string): string {
return title ? `# ${title}\n\n${markdown}` : markdown;
}
-async function requestJson(url: string, init: RequestInit): Promise<unknown> {
- const response = await llmFetch(url, { ...init, signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS) });
+async function requestJson(url: string, init: RequestInit, signal: AbortSignal | undefined): Promise<unknown> {
+ const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+ const response = await llmFetch(url, { ...init, signal: signal ? AbortSignal.any([ signal, timeout ]) : timeout });
if (!response.ok) {
if (response.status === 401 || response.status === 403) {
throw new Error(`The search provider refused the request (HTTP ${response.status}); check its API key in the AI settings.`);