A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
test(ai): follow live Kimi coding catalog (#465)
Sebastian Müller8454794b4878
diff --git a/packages/ai/test/abort.test.ts b/packages/ai/test/abort.test.tsindex 57ef078af2..cffc14c0a9 100644--- a/packages/ai/test/abort.test.ts+++ b/packages/ai/test/abort.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";import { getModel } from "../src/models.js";import { complete, stream } from "../src/stream.js";import type { Api, Context, Model, StreamOptions } from "../src/types.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;@@ -239,7 +240,7 @@ describe("AI Providers Abort Tests", () => {});describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Abort", () => {- const llm = getModel("kimi-coding", "kimi-k2-thinking");+ const llm = getKimiCodingTestModel();it("should abort mid-stream", { retry: 3 }, async () => {await testAbortSignal(llm);diff --git a/packages/ai/test/context-overflow.test.ts b/packages/ai/test/context-overflow.test.tsindex 6ee8a60687..aed7d1cd0b 100644--- a/packages/ai/test/context-overflow.test.ts+++ b/packages/ai/test/context-overflow.test.ts@@ -20,6 +20,7 @@ import type { AssistantMessage, Context, Model, Usage } from "../src/types.js";import { isContextOverflow } from "../src/utils/overflow.js";import { hasAzureOpenAICredentials } from "./azure-utils.js";import { hasBedrockCredentials } from "./bedrock-utils.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";import { resolveApiKey } from "./oauth.js";// Resolve OAuth tokens at module level (async, runs before tests)@@ -449,8 +450,8 @@ describe("Context overflow error handling", () => {// =============================================================================describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding", () => {- it("kimi-k2-thinking - should detect overflow via isContextOverflow", async () => {- const model = getModel("kimi-coding", "kimi-k2-thinking");+ it("should detect overflow via isContextOverflow", async () => {+ const model = getKimiCodingTestModel();const result = await testContextOverflow(model, process.env.KIMI_API_KEY!);logResult(result);diff --git a/packages/ai/test/cross-provider-handoff.test.ts b/packages/ai/test/cross-provider-handoff.test.tsindex ca4635f5ec..9407119b37 100644--- a/packages/ai/test/cross-provider-handoff.test.ts+++ b/packages/ai/test/cross-provider-handoff.test.ts@@ -30,6 +30,7 @@ import { completeSimple, getEnvApiKey } from "../src/stream.js";import type { Api, AssistantMessage, Message, Model, Tool, ToolResultMessage } from "../src/types.js";import { hasAzureOpenAICredentials } from "./azure-utils.js";import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";import { resolveApiKey } from "./oauth.js";// Simple tool for testing@@ -110,7 +111,7 @@ const PROVIDER_MODEL_PAIRS: ProviderModelPair[] = [// Hugging Face{ provider: "huggingface", model: "moonshotai/Kimi-K2.5", label: "huggingface-kimi-k2.5" },// Kimi For Coding- { provider: "kimi-coding", model: "kimi-k2-thinking", label: "kimi-coding-k2-thinking" },+ { provider: "kimi-coding", model: getKimiCodingTestModel().id, label: "kimi-coding" },// Mistral{ provider: "mistral", model: "devstral-medium-latest", label: "mistral-devstral-medium" },// MiniMaxdiff --git a/packages/ai/test/empty.test.ts b/packages/ai/test/empty.test.tsindex 211d372178..372d183909 100644--- a/packages/ai/test/empty.test.ts+++ b/packages/ai/test/empty.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";import { getModel } from "../src/models.js";import { complete } from "../src/stream.js";import type { Api, AssistantMessage, Context, Model, StreamOptions, UserMessage } from "../src/types.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;@@ -517,7 +518,7 @@ describe("AI Providers Empty Message Tests", () => {);describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Empty Messages", () => {- const llm = getModel("kimi-coding", "kimi-k2-thinking");+ const llm = getKimiCodingTestModel();it("should handle empty content array", { retry: 3, timeout: 30000 }, async () => {await testEmptyMessage(llm);diff --git a/packages/ai/test/image-tool-result.test.ts b/packages/ai/test/image-tool-result.test.tsindex a77714255e..e467a478b6 100644--- a/packages/ai/test/image-tool-result.test.ts+++ b/packages/ai/test/image-tool-result.test.ts@@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest";import type { Api, Context, Model, Tool, ToolResultMessage } from "../src/index.js";import { complete, getModel } from "../src/index.js";import type { StreamOptions } from "../src/types.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;@@ -368,8 +369,8 @@ describe("Tool Results with Images", () => {},);- describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider (kimi-for-coding)", () => {- const llm = getModel("kimi-coding", "kimi-for-coding");+ describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider (image-capable model)", () => {+ const llm = getKimiCodingTestModel({ image: true });it("should handle tool result with only image", { retry: 3, timeout: 30000 }, async () => {await handleToolWithImageResult(llm);diff --git a/packages/ai/test/kimi-test-model.ts b/packages/ai/test/kimi-test-model.tsnew file mode 100644index 0000000000..43e9b35ad7--- /dev/null+++ b/packages/ai/test/kimi-test-model.ts@@ -0,0 +1,18 @@+import { getModels } from "../src/models.js";+import type { Model } from "../src/types.js";++const KIMI_TEST_MODEL_PREFERENCE = ["kimi-k2-thinking", "kimi-for-coding", "k2p7", "k3", "kimi-for-coding-highspeed"];++export function getKimiCodingTestModel(options: { image?: boolean } = {}): Model<"anthropic-messages"> {+ const models = getModels("kimi-coding") as Model<"anthropic-messages">[];+ const eligible = options.image ? models.filter((model) => model.input.includes("image")) : models;+ for (const id of KIMI_TEST_MODEL_PREFERENCE) {+ const model = eligible.find((candidate) => candidate.id === id);+ if (model) return model;+ }+ const model = eligible[0];+ if (!model) {+ throw new Error(`No ${options.image ? "image-capable " : ""}Kimi Coding model is available`);+ }+ return model;+}diff --git a/packages/ai/test/stream.test.ts b/packages/ai/test/stream.test.tsindex 7ba61f93b9..76876c19b3 100644--- a/packages/ai/test/stream.test.ts+++ b/packages/ai/test/stream.test.ts@@ -8,6 +8,7 @@ import { getEnvApiKey } from "../src/env-api-keys.js";import { getModel } from "../src/models.js";import { complete, stream } from "../src/stream.js";import type { Api, Context, ImageContent, Model, StreamOptions, Tool, ToolResultMessage } from "../src/types.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;@@ -988,32 +989,29 @@ describe("Generate E2E Tests", () => {});});- describe.skipIf(!process.env.KIMI_API_KEY)(- "Kimi For Coding Provider (kimi-k2-thinking via Anthropic Messages)",- () => {- const llm = getModel("kimi-coding", "kimi-k2-thinking");+ describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider (Anthropic Messages)", () => {+ const llm = getKimiCodingTestModel();- it("should complete basic text generation", { retry: 3 }, async () => {- await basicTextGeneration(llm);- });+ it("should complete basic text generation", { retry: 3 }, async () => {+ await basicTextGeneration(llm);+ });- it("should handle tool calling", { retry: 3 }, async () => {- await handleToolCall(llm);- });+ it("should handle tool calling", { retry: 3 }, async () => {+ await handleToolCall(llm);+ });- it("should handle streaming", { retry: 3 }, async () => {- await handleStreaming(llm);- });+ it("should handle streaming", { retry: 3 }, async () => {+ await handleStreaming(llm);+ });- it("should handle thinking mode", { retry: 3 }, async () => {- await handleThinking(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });- });+ it("should handle thinking mode", { retry: 3 }, async () => {+ await handleThinking(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });+ });- it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {- await multiTurn(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });- });- },- );+ it("should handle multi-turn with thinking and tools", { retry: 3 }, async () => {+ await multiTurn(llm, { thinkingEnabled: true, thinkingBudgetTokens: 2048 });+ });+ });describe.skipIf(!process.env.XIAOMI_API_KEY)("Xiaomi MiMo (API billing) Provider (Xiaomi MiMo-V2.5-Pro via Anthropic Messages)",diff --git a/packages/ai/test/tokens.test.ts b/packages/ai/test/tokens.test.tsindex 9ff8088407..e7515a1500 100644--- a/packages/ai/test/tokens.test.ts+++ b/packages/ai/test/tokens.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";import { getModel } from "../src/models.js";import { stream } from "../src/stream.js";import type { Api, Context, Model, StreamOptions } from "../src/types.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;@@ -205,7 +206,7 @@ describe("Token Statistics on Abort", () => {});describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider", () => {- const llm = getModel("kimi-coding", "kimi-for-coding");+ const llm = getKimiCodingTestModel();it("should include token stats when aborted mid-stream", { retry: 3, timeout: 30000 }, async () => {await testTokensOnAbort(llm);diff --git a/packages/ai/test/tool-call-without-result.test.ts b/packages/ai/test/tool-call-without-result.test.tsindex e864d2638f..76e3d7a6ec 100644--- a/packages/ai/test/tool-call-without-result.test.ts+++ b/packages/ai/test/tool-call-without-result.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";import { getModel } from "../src/models.js";import { complete } from "../src/stream.js";import type { Api, Context, Model, StreamOptions, Tool } from "../src/types.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;@@ -248,7 +249,7 @@ describe("Tool Call Without Result Tests", () => {});describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider", () => {- const model = getModel("kimi-coding", "kimi-k2-thinking");+ const model = getKimiCodingTestModel();it("should filter out tool calls without corresponding tool results", { retry: 3, timeout: 30000 }, async () => {await testToolCallWithoutResult(model);diff --git a/packages/ai/test/total-tokens.test.ts b/packages/ai/test/total-tokens.test.tsindex 73aee05097..a787f64237 100644--- a/packages/ai/test/total-tokens.test.ts+++ b/packages/ai/test/total-tokens.test.ts@@ -16,6 +16,7 @@ import { describe, expect, it } from "vitest";import { getModel } from "../src/models.js";import { complete } from "../src/stream.js";import type { Api, Context, Model, StreamOptions, Usage } from "../src/types.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;@@ -542,22 +543,18 @@ describe("totalTokens field", () => {// =========================================================================describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding", () => {- it(- "kimi-k2-thinking - should return totalTokens equal to sum of components",- { retry: 3, timeout: 60000 },- async () => {- const llm = getModel("kimi-coding", "kimi-k2-thinking");+ it("should return totalTokens equal to sum of components", { retry: 3, timeout: 60000 }, async () => {+ const llm = getKimiCodingTestModel();- console.log(`\nKimi For Coding / ${llm.id}:`);- const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.KIMI_API_KEY });+ console.log(`\nKimi For Coding / ${llm.id}:`);+ const { first, second } = await testTotalTokensWithCache(llm, { apiKey: process.env.KIMI_API_KEY });- logUsage("First request", first);- logUsage("Second request", second);+ logUsage("First request", first);+ logUsage("Second request", second);- assertTotalTokensEqualsComponents(first);- assertTotalTokensEqualsComponents(second);- },- );+ assertTotalTokensEqualsComponents(first);+ assertTotalTokensEqualsComponents(second);+ });});// =========================================================================diff --git a/packages/ai/test/unicode-surrogate.test.ts b/packages/ai/test/unicode-surrogate.test.tsindex df855712d1..18349553e6 100644--- a/packages/ai/test/unicode-surrogate.test.ts+++ b/packages/ai/test/unicode-surrogate.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";import { getModel } from "../src/models.js";import { complete } from "../src/stream.js";import type { Api, Context, Model, StreamOptions, ToolResultMessage } from "../src/types.js";+import { getKimiCodingTestModel } from "./kimi-test-model.js";type StreamOptionsWithExtras = StreamOptions & Record<string, unknown>;@@ -680,7 +681,7 @@ describe("AI Providers Unicode Surrogate Pair Tests", () => {);describe.skipIf(!process.env.KIMI_API_KEY)("Kimi For Coding Provider Unicode Handling", () => {- const llm = getModel("kimi-coding", "kimi-k2-thinking");+ const llm = getKimiCodingTestModel();it("should handle emoji in tool results", { retry: 3, timeout: 30000 }, async () => {await testEmojiInToolResults(llm);