Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

enable image input for vision-capable prime inference models (#261)

kt6c116e9f44cc
diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts
index 443220fb98..4f6a3037a1 100644
--- a/packages/ai/scripts/generate-models.ts
+++ b/packages/ai/scripts/generate-models.ts
@@ -117,6 +117,7 @@ interface PrimeInferenceCatalogEntry {
interface PrimeInferenceModelMetadata {
contextWindow: number;
maxTokens: number;
+ vision?: boolean;
}
// Prime Inference intentionally exposes a curated subset of the catalog in the
@@ -125,31 +126,31 @@ interface PrimeInferenceModelMetadata {
const PRIME_INFERENCE_MODEL_METADATA: Record<string, PrimeInferenceModelMetadata> = {
// prime-inference proxies Anthropic models through OpenRouter without the
// context-1m beta header, so the enforced window is the standard 200k, not 1M.
- "anthropic/claude-haiku-4.5": { contextWindow: 200000, maxTokens: 64000 },
- "anthropic/claude-opus-4.6": { contextWindow: 200000, maxTokens: 128000 },
- "anthropic/claude-opus-4.7": { contextWindow: 200000, maxTokens: 128000 },
- "anthropic/claude-opus-4.8": { contextWindow: 200000, maxTokens: 128000 },
- "anthropic/claude-sonnet-4.5": { contextWindow: 200000, maxTokens: 64000 },
- "anthropic/claude-sonnet-4.6": { contextWindow: 200000, maxTokens: 128000 },
+ "anthropic/claude-haiku-4.5": { contextWindow: 200000, maxTokens: 64000, vision: true },
+ "anthropic/claude-opus-4.6": { contextWindow: 200000, maxTokens: 128000, vision: true },
+ "anthropic/claude-opus-4.7": { contextWindow: 200000, maxTokens: 128000, vision: true },
+ "anthropic/claude-opus-4.8": { contextWindow: 200000, maxTokens: 128000, vision: true },
+ "anthropic/claude-sonnet-4.5": { contextWindow: 200000, maxTokens: 64000, vision: true },
+ "anthropic/claude-sonnet-4.6": { contextWindow: 200000, maxTokens: 128000, vision: true },
"deepseek/deepseek-v3.2": { contextWindow: 128000, maxTokens: 8000 },
"deepseek/deepseek-v4-flash": { contextWindow: 1000000, maxTokens: 384000 },
"deepseek/deepseek-v4-pro": { contextWindow: 1000000, maxTokens: 384000 },
"minimax/minimax-m3": { contextWindow: 204800, maxTokens: 131072 },
- "moonshotai/kimi-k2.7-code": { contextWindow: 262144, maxTokens: 16000 },
+ "moonshotai/kimi-k2.7-code": { contextWindow: 262144, maxTokens: 16000, vision: true },
"nvidia/nemotron-3-nano-30b-a3b": { contextWindow: 1000000, maxTokens: 228000 },
"nvidia/nemotron-3-super-120b-a12b": { contextWindow: 1000000, maxTokens: 4096 },
- "openai/gpt-5.3-codex": { contextWindow: 400000, maxTokens: 128000 },
- "openai/gpt-5.4": { contextWindow: 1050000, maxTokens: 128000 },
- "openai/gpt-5.4-mini": { contextWindow: 400000, maxTokens: 128000 },
- "openai/gpt-5.4-pro": { contextWindow: 1050000, maxTokens: 128000 },
- "openai/gpt-5.5": { contextWindow: 1050000, maxTokens: 128000 },
+ "openai/gpt-5.3-codex": { contextWindow: 400000, maxTokens: 128000, vision: true },
+ "openai/gpt-5.4": { contextWindow: 1050000, maxTokens: 128000, vision: true },
+ "openai/gpt-5.4-mini": { contextWindow: 400000, maxTokens: 128000, vision: true },
+ "openai/gpt-5.4-pro": { contextWindow: 1050000, maxTokens: 128000, vision: true },
+ "openai/gpt-5.5": { contextWindow: 1050000, maxTokens: 128000, vision: true },
"prime-intellect/intellect-3": { contextWindow: 131072, maxTokens: 131072 },
"qwen/qwen3-235b-a22b-thinking-2507": { contextWindow: 262144, maxTokens: 4096 },
"qwen/qwen3-coder-next": { contextWindow: 262144, maxTokens: 65536 },
"qwen/qwen3-max": { contextWindow: 262144, maxTokens: 65536 },
- "qwen/qwen3-vl-235b-a22b-thinking": { contextWindow: 262144, maxTokens: 32768 },
- "x-ai/grok-4.20": { contextWindow: 2000000, maxTokens: 30000 },
- "x-ai/grok-4.20-multi-agent": { contextWindow: 2000000, maxTokens: 30000 },
+ "qwen/qwen3-vl-235b-a22b-thinking": { contextWindow: 262144, maxTokens: 32768, vision: true },
+ "x-ai/grok-4.20": { contextWindow: 2000000, maxTokens: 30000, vision: true },
+ "x-ai/grok-4.20-multi-agent": { contextWindow: 2000000, maxTokens: 30000, vision: true },
"z-ai/glm-5": { contextWindow: 202752, maxTokens: 131072 },
"z-ai/glm-5.1": { contextWindow: 202800, maxTokens: 131072 },
"z-ai/glm-5.2": { contextWindow: 202800, maxTokens: 131072 },
@@ -435,7 +436,7 @@ function createPrimeInferenceModel(
provider: "prime-inference",
baseUrl: PRIME_INFERENCE_BASE_URL,
reasoning: isPrimeInferenceReasoningModel(entry.id, entry.reasoning),
- input: ["text"],
+ input: metadata.vision ? ["text", "image"] : ["text"],
cost: {
input: entry.input,
output: entry.output,
diff --git a/packages/ai/src/models.generated.ts b/packages/ai/src/models.generated.ts
index 78ca58e603..78dc2edf37 100644
--- a/packages/ai/src/models.generated.ts
+++ b/packages/ai/src/models.generated.ts
@@ -12824,7 +12824,7 @@ export const MODELS = {
baseUrl: "https://api.pinference.ai/api/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: false,
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 1,
output: 5,
@@ -12843,7 +12843,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"max":"max"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 5,
output: 25,
@@ -12862,7 +12862,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh","max":"max"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 5,
output: 25,
@@ -12881,7 +12881,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh","max":"max"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 5,
output: 25,
@@ -12899,7 +12899,7 @@ export const MODELS = {
baseUrl: "https://api.pinference.ai/api/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 3,
output: 15,
@@ -12918,7 +12918,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"max":"max"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 3,
output: 15,
@@ -13010,7 +13010,7 @@ export const MODELS = {
baseUrl: "https://api.pinference.ai/api/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 1.0925,
output: 4.6,
@@ -13065,7 +13065,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 1.75,
output: 14,
@@ -13084,7 +13084,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 2.5,
output: 15,
@@ -13103,7 +13103,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 0.75,
output: 4.5,
@@ -13122,7 +13122,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 30,
output: 180,
@@ -13141,7 +13141,7 @@ export const MODELS = {
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh"},
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 5,
output: 30,
@@ -13213,7 +13213,7 @@ export const MODELS = {
baseUrl: "https://api.pinference.ai/api/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 0.784,
output: 3.16,
@@ -13231,7 +13231,7 @@ export const MODELS = {
baseUrl: "https://api.pinference.ai/api/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 2,
output: 6,
@@ -13249,7 +13249,7 @@ export const MODELS = {
baseUrl: "https://api.pinference.ai/api/v1",
compat: {"supportsStore":false,"supportsDeveloperRole":false,"supportsReasoningEffort":true,"maxTokensField":"max_tokens","supportsStrictMode":false},
reasoning: true,
- input: ["text"],
+ input: ["text", "image"],
cost: {
input: 2,
output: 6,
diff --git a/packages/ai/test/prime-inference-models.test.ts b/packages/ai/test/prime-inference-models.test.ts
index 2c51ac82a4..3427609657 100644
--- a/packages/ai/test/prime-inference-models.test.ts
+++ b/packages/ai/test/prime-inference-models.test.ts
@@ -49,7 +49,7 @@ describe("Prime Inference models", () => {
expect(model.reasoning).toBe(true);
expect(model.thinkingLevelMap).toEqual({ xhigh: "xhigh" });
expect(getSupportedThinkingLevels(model)).toContain("xhigh");
- expect(model.input).toEqual(["text"]);
+ expect(model.input).toEqual(["text", "image"]);
expect(model.contextWindow).toBe(1050000);
expect(model.maxTokens).toBe(128000);
expect(model.cost).toEqual({