Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(discovery): infer vision from the model id when an upstream advertises none (#1083)

Qatadaha Bin Matloobbe12fe0b187b
diff --git a/server/src/__tests__/services/model-discovery.test.ts b/server/src/__tests__/services/model-discovery.test.ts
index 453dc540..f145741c 100644
--- a/server/src/__tests__/services/model-discovery.test.ts
+++ b/server/src/__tests__/services/model-discovery.test.ts
@@ -187,3 +187,63 @@ describe('readCappedBody (#488)', () => {
await expect(readCappedBody(new Response(body))).rejects.toBeInstanceOf(ModelDiscoveryError);
});
});
+
+// #1051: an upstream that returns bare ids gives visionOf nothing to read, so
+// a VL model arrives looking exactly like a chat one. The id is the only
+// signal left.
+describe('vision inferred from the model id (#1051)', () => {
+ it('flags a VL model an upstream ships with no modality metadata', () => {
+ expect(parseModelCatalog({
+ data: [{ id: 'Qwen/Qwen2.5-VL-72B-Instruct' }],
+ })).toEqual([
+ { id: 'Qwen/Qwen2.5-VL-72B-Instruct', ownedBy: null, vision: true },
+ ]);
+ });
+
+ it('reads the version digits some VL ids carry', () => {
+ const [m] = parseModelCatalog({ data: [{ id: 'deepseek-ai/deepseek-vl2' }] });
+ expect(m.vision).toBe(true);
+ });
+
+ it('flags the named vision families', () => {
+ const ids = ['llava-1.5-7b', 'OpenGVLab/InternVL2-8B', 'mistralai/Pixtral-12B', 'llama-3.2-11b-vision'];
+ for (const id of ids) {
+ const [m] = parseModelCatalog({ data: [{ id }] });
+ expect(m.vision, id).toBe(true);
+ }
+ });
+
+ it('flags a bare string entry the same way', () => {
+ expect(parseModelCatalog(['Qwen2-VL-7B'])).toEqual([
+ { id: 'Qwen2-VL-7B', ownedBy: null, vision: true },
+ ]);
+ });
+
+ it('does not read `vl` out of a longer token', () => {
+ // A relay that puts its own stack in the id must not read as multimodal.
+ for (const id of ['vllm-proxy/qwen3-4b', 'acme-vlab-7b']) {
+ const [m] = parseModelCatalog({ data: [{ id }] });
+ expect(m.vision, id).toBeUndefined();
+ }
+ });
+
+ it('leaves a text-only id with no vision key at all', () => {
+ expect(parseModelCatalog({ data: [{ id: 'qwen3-4b' }] })).toEqual([
+ { id: 'qwen3-4b', ownedBy: null },
+ ]);
+ });
+
+ it('keeps an explicit upstream false over the id marker', () => {
+ const [m] = parseModelCatalog({
+ data: [{ id: 'some-vl-router', vision: false }],
+ });
+ expect(m.vision).toBe(false);
+ });
+
+ it('keeps upstream modality metadata authoritative when present', () => {
+ const [m] = parseModelCatalog({
+ data: [{ id: 'plain-chat-model', architecture: { input_modalities: ['text', 'image'] } }],
+ });
+ expect(m.vision).toBe(true);
+ });
+});
diff --git a/server/src/services/model-discovery.ts b/server/src/services/model-discovery.ts
index 017f8646..22cc74e0 100644
--- a/server/src/services/model-discovery.ts
+++ b/server/src/services/model-discovery.ts
@@ -76,6 +76,13 @@ const VISION_KEYS = ['vision', 'supports_vision', 'supportsVision', 'image_input
const MODALITY_KEYS = ['modalities', 'input_modalities', 'inputModalities', 'modality'] as const;
// A modality entry (or a substring of the modality string) that means images.
const VISION_MODALITIES = ['image', 'vision', 'image-input'] as const;
+// #1051: some OpenAI-compatible upstreams (SiliconFlow among them) answer
+// /v1/models with entries that carry an id and nothing else, so visionOf finds
+// no signal at all and a VL model is indistinguishable from a chat one. These
+// id markers are the last resort. They are matched per token (the id split on
+// non-alphanumerics), never as a substring, so a relay that ships `vllm` in an
+// id cannot read as `vl`.
+const VISION_ID_MARKERS = ['llava', 'internvl', 'pixtral', 'moondream', 'cogvlm', 'vision'] as const;
// A price hint sits in a chip next to the model id, so a chatty relay must not
// be able to squeeze the id out of the row.
const MAX_PRICE_NOTE_LENGTH = 40;
@@ -205,10 +212,28 @@ function visionOf(record: Record<string, unknown>): boolean | undefined {
return architecture ? modalityVision(architecture) : undefined;
}
+/** Vision read off the model id, for upstreams that advertise no modality
+ * metadata whatsoever (#1051). Returns true or undefined and never false: a
+ * missing marker is not evidence that a model lacks vision, and a false here
+ * would stamp `vision: false` onto every bare `{ id }` entry. */
+function visionFromId(id: string): true | undefined {
+ for (const token of id.toLowerCase().split(/[^a-z0-9]+/)) {
+ // `vl`/`vlm` plus the version digits some ids carry (`vl2`), anchored to a
+ // whole token so `vllm` and `vlab` stay out.
+ if (/^vlm?\d*$/.test(token)) return true;
+ if (VISION_ID_MARKERS.some(marker => token.includes(marker))) return true;
+ }
+ return undefined;
+}
+
function toDiscovered(entry: unknown): DiscoveredModel | null {
if (typeof entry === 'string') {
const id = entry.trim();
- return id ? { id, ownedBy: null } : null;
+ if (!id) return null;
+ const bare: DiscoveredModel = { id, ownedBy: null };
+ const bareVision = visionFromId(id);
+ if (bareVision !== undefined) bare.vision = bareVision;
+ return bare;
}
const record = asRecord(entry);
if (!record) return null;
@@ -225,7 +250,8 @@ function toDiscovered(entry: unknown): DiscoveredModel | null {
model.priceNote = price.note;
model.isFree = price.isFree;
}
- const vision = visionOf(record);
+ // `??` not `||`: an upstream that explicitly says false keeps saying false.
+ const vision = visionOf(record) ?? visionFromId(id);
if (vision !== undefined) model.vision = vision;
return model;
}