Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

19.4 KB
Raw Blame
/**
 * Custom-model protocol detection UI logic: the generic protocol client-type family
 * (selector visibility / value mapping), when saving must detect first, what the failure
 * popup explains, the guarantee that no entry is persisted without a protocol, and the
 * protocol-path suffix for the new client types. The probing itself is server-side (see
 * the server package's protocol-detect tests); these are the pure helpers the dialog
 * composes.
 */
import { describe, expect, it } from "vitest";
import { MODEL_CATALOG, modelEnvPreviewKey, resolveModelEnv } from "@adelie/core/model-catalog";
import { zh as ZH } from "../src/lib/strings";
import { en as EN } from "../src/lib/strings-en";
import { clientTypeAfterProviderChange, rowToEntry } from "../src/features/models/models-page";
import type { RowState } from "../src/features/models/models-page";
import {
  PROTOCOL_CLIENT_TYPES,
  detectableBaseUrl,
  displayWidthCh,
  envHintClientType,
  envHintKeyFor,
  isCustomLikeGroup,
  isGenericProtocolClientType,
  needsProtocolDetectOnSave,
  protocolForPersist,
  protocolSelectorValue,
} from "../src/features/models/protocol-types";
import { protocolPathForModel } from "../src/features/models/protocol-path";

describe("PROTOCOL_CLIENT_TYPES", () => {
  it("lists the three generic clients in the required detection order (responses first, chat completions last)", () => {
    expect(PROTOCOL_CLIENT_TYPES).toEqual(["openai-responses", "ant-messages", "openai-chat"]);
  });
});

describe("isGenericProtocolClientType", () => {
  it("accepts the protocol trio, the bare openai alias, and empty; rejects vendor-pinned types", () => {
    expect(isGenericProtocolClientType("openai-responses")).toBe(true);
    expect(isGenericProtocolClientType("ant-messages")).toBe(true);
    expect(isGenericProtocolClientType("openai-chat")).toBe(true);
    expect(isGenericProtocolClientType("openai")).toBe(true);
    expect(isGenericProtocolClientType("")).toBe(true);
    expect(isGenericProtocolClientType(" OpenAI-Responses ")).toBe(true);
    expect(isGenericProtocolClientType("deepseek-v4")).toBe(false);
    expect(isGenericProtocolClientType("claude-4-8")).toBe(false);
    expect(isGenericProtocolClientType("minimax-m3")).toBe(false);
  });
});

describe("protocolSelectorValue", () => {
  it("maps openai / unknown to openai-chat for display without rewriting the stored value", () => {
    expect(protocolSelectorValue("openai-responses")).toBe("openai-responses");
    expect(protocolSelectorValue("ant-messages")).toBe("ant-messages");
    expect(protocolSelectorValue("openai-chat")).toBe("openai-chat");
    expect(protocolSelectorValue("openai")).toBe("openai-chat");
  });

  it("reports NOTHING selected for an unset protocol, so the control cannot imply a default", () => {
    // A fresh custom model: no checked row in the menu, no path in the field. Returning
    // "openai-chat" here (as it once did) rendered a choice the user never made.
    expect(protocolSelectorValue("")).toBeNull();
    expect(protocolSelectorValue("   ")).toBeNull();
  });
});

describe("displayWidthCh (padding reserved for the in-field suffix)", () => {
  it("counts ASCII as one column, so the protocol paths reserve exactly their length", () => {
    expect(displayWidthCh("/chat/completions")).toBe(17);
    expect(displayWidthCh("/responses")).toBe(10);
    expect(displayWidthCh("Select protocol")).toBe(15);
  });

  it("counts CJK as two, so a localized placeholder is not under-reserved by half", () => {
    expect(displayWidthCh("选择协议")).toBe(8);
    expect(displayWidthCh("")).toBe(0);
  });
});

describe("detectableBaseUrl", () => {
  it("only absolute http(s) URLs are probeable (mirrors the server-side validation)", () => {
    expect(detectableBaseUrl("https://api.example.com/v1")).toBe(true);
    expect(detectableBaseUrl(" http://127.0.0.1:8000/v1 ")).toBe(true);
    expect(detectableBaseUrl("api.example.com/v1")).toBe(false);
    expect(detectableBaseUrl("ftp://example.com")).toBe(false);
    expect(detectableBaseUrl("")).toBe(false);
    expect(detectableBaseUrl("https://")).toBe(false);
  });
});

describe("clientTypeAfterProviderChange (protocol family kept on move to Custom)", () => {
  it("keeps generic protocol client types when moving to Custom", () => {
    expect(clientTypeAfterProviderChange("custom", "openai-responses")).toBe("openai-responses");
    expect(clientTypeAfterProviderChange("custom", "ant-messages")).toBe("ant-messages");
    expect(clientTypeAfterProviderChange("custom", "openai-chat")).toBe("openai-chat");
    expect(clientTypeAfterProviderChange("custom", "openai")).toBe("openai");
  });

  it("still pins vendor-specific or empty types to openai-chat when moving to Custom, and never touches other groups", () => {
    expect(clientTypeAfterProviderChange("custom", "")).toBe("openai-chat");
    expect(clientTypeAfterProviderChange("custom", "claude-5")).toBe("openai-chat");
    expect(clientTypeAfterProviderChange("google", "openai")).toBe("openai");
    expect(clientTypeAfterProviderChange("my-group", "ant-messages")).toBe("ant-messages");
  });

  it("rewrites to the group's own protocol when one is pinned, whatever the entry carried", () => {
    // The pin is the group's semantics, not a fallback: an entry dragged into vLLM speaks
    // what the rest of the group speaks, including one that had already picked a protocol.
    expect(clientTypeAfterProviderChange("vllm", "")).toBe("openai-chat-vllm-adapter");
    expect(clientTypeAfterProviderChange("vllm", "openai-chat")).toBe("openai-chat-vllm-adapter");
    expect(clientTypeAfterProviderChange("vllm", "ant-messages")).toBe("openai-chat-vllm-adapter");
    expect(clientTypeAfterProviderChange("vllm", "deepseek-v4")).toBe("openai-chat-vllm-adapter");
    // A gateway can pin too: OpenRouter speaks the Responses API for every upstream it
    // serves, so an entry dragged in is rewritten off whatever it carried before.
    expect(clientTypeAfterProviderChange("openrouter", "")).toBe("openai-responses");
    expect(clientTypeAfterProviderChange("openrouter", "openai-chat")).toBe("openai-responses");
    // And moving back out of it does not carry the pin along.
    expect(clientTypeAfterProviderChange("custom", "openai-chat-vllm-adapter")).toBe("openai-chat");
  });
});

describe("isCustomLikeGroup", () => {
  it("covers custom and every unknown (user-defined) group, but no catalog vendor group", () => {
    expect(isCustomLikeGroup("custom")).toBe(true);
    expect(isCustomLikeGroup("my-group")).toBe(true);
    expect(isCustomLikeGroup("openai")).toBe(false);
    expect(isCustomLikeGroup("anthropic")).toBe(false);
    // A group that pins its protocol picks nothing: the choice is already made for it.
    expect(isCustomLikeGroup("vllm")).toBe(false);
  });
});

describe("needsProtocolDetectOnSave (save detects a still-unset protocol first)", () => {
  it("fires for saving a custom-like entry with no protocol yet", () => {
    expect(needsProtocolDetectOnSave("save", "custom", "")).toBe(true);
    expect(needsProtocolDetectOnSave("save", "my-group", "   ")).toBe(true);
  });

  it("does not fire once a protocol is chosen", () => {
    expect(needsProtocolDetectOnSave("save", "custom", "ant-messages")).toBe(false);
    expect(needsProtocolDetectOnSave("save", "custom", "openai-chat")).toBe(false);
  });

  it("never probes for vendor groups, nor for actions other than save", () => {
    // A vendor group auto-routes by model id; an empty protocol there is correct.
    expect(needsProtocolDetectOnSave("save", "openai", "")).toBe(false);
    // A pinned group has nothing to probe for — the group already knows the answer.
    expect(needsProtocolDetectOnSave("save", "vllm", "")).toBe(false);
    expect(needsProtocolDetectOnSave("remove", "custom", "")).toBe(false);
    expect(needsProtocolDetectOnSave("setDefault", "custom", "")).toBe(false);
    expect(needsProtocolDetectOnSave("setVisionModel", "custom", "")).toBe(false);
  });
});

describe("envHintClientType (custom groups never infer a client from the model id)", () => {
  const envFor = (provider: string, modelId: string, clientType: string) =>
    resolveModelEnv(modelId, envHintClientType(provider, clientType))?.envKey;

  it("keeps a vendor-looking model id in a custom group on the compatible client's env var", () => {
    // The bug this pins: `claude-sonnet-5` typed into a custom group used to resolve to
    // ANTHROPIC_API_KEY, implying an Anthropic client that group never routes to.
    expect(envFor("custom", "claude-sonnet-5", "")).toBe("OPENAI_API_KEY");
    expect(envFor("custom", "gpt-5.6", "")).toBe("OPENAI_API_KEY");
    expect(envFor("my-group", "claude-sonnet-5", "")).toBe("OPENAI_API_KEY");
  });

  it("still honours a protocol the user actually chose", () => {
    expect(envFor("custom", "whatever", "ant-messages")).toBe("ANTHROPIC_API_KEY");
    expect(envFor("custom", "whatever", "openai-responses")).toBe("OPENAI_API_KEY");
  });

  it("resolves a pinned group against its pin rather than the model id", () => {
    // deepseek-ai/DeepSeek-V4-Pro would otherwise route by id to DEEPSEEK_API_KEY, which is
    // not what an entry saved on the vLLM client reads.
    expect(envHintClientType("vllm", "")).toBe("openai-chat-vllm-adapter");
    expect(envFor("vllm", "deepseek-ai/DeepSeek-V4-Pro", "")).toBe("OPENAI_API_KEY");
  });

  it("leaves vendor groups routing by model id, which is how they really work", () => {
    expect(envHintClientType("anthropic", "")).toBeUndefined();
    expect(envFor("anthropic", "claude-sonnet-5", "")).toBe("ANTHROPIC_API_KEY");
    expect(envFor("google", "gemini-3.1-pro", "")).toBe("GEMINI_API_KEY");
  });
});

describe("envHintKeyFor (the API-key field promises a variable only where the entry may fall back to it)", () => {
  it("names nothing for a gateway row: its preset endpoint is not the vendor's, so the vendor key is not its key", () => {
    // The #786 finding: once an Anthropic row had proved OPENAI_API_KEY / ANTHROPIC_API_KEY
    // set, a TokenDance or OpenCode Go row read "leave empty to use the ... env var".
    expect(
      envHintKeyFor("tokendance", "glm-5.3", "openai-chat", "https://tokendance.space/gateway/v1"),
    ).toBeUndefined();
    expect(
      envHintKeyFor(
        "openrouter",
        "openai/gpt-5.5",
        "openai-responses",
        "https://openrouter.ai/api/v1",
      ),
    ).toBeUndefined();
    expect(
      envHintKeyFor("custom", "claude-sonnet-5", "ant-messages", "https://gw.example.com"),
    ).toBeUndefined();
    expect(envHintKeyFor("vllm", "Qwen/Qwen3.8-27B", "", "http://gpu-box:8000/v1")).toBeUndefined();
    expect(
      envHintKeyFor("my-group", "some-model", "", "https://gw.example.com/v1"),
    ).toBeUndefined();
  });

  it("names the vendor's variable for a first-party row, and the relay's own for Penguin Go", () => {
    expect(envHintKeyFor("anthropic", "claude-sonnet-4-6", "", "")).toBe("ANTHROPIC_API_KEY");
    expect(envHintKeyFor("google", "gemini-3.1-pro", "", "")).toBe("GEMINI_API_KEY");
    expect(
      envHintKeyFor("deepseek", "deepseek-flash", "deepseek-v4", "https://api.deepseek.com"),
    ).toBe("DEEPSEEK_API_KEY");
    // A custom row pointed at the vendor's own endpoint gets the vendor's key: it goes to the vendor.
    expect(envHintKeyFor("custom", "gpt-5.6", "openai-chat", "https://api.openai.com/v1")).toBe(
      "OPENAI_API_KEY",
    );
    expect(
      envHintKeyFor(
        "penguin-go",
        "gemini-3.8-flash",
        "gemini-3.8",
        "https://token.penguin.ooo/api",
      ),
    ).toBe("PENGUIN_GO_API_KEY");
  });

  it("follows the base URL as it is edited: re-pointing a vendor row at a proxy drops the hint", () => {
    expect(
      envHintKeyFor("anthropic", "claude-sonnet-4-6", "", "https://proxy.example/anthropic"),
    ).toBeUndefined();
    expect(envHintKeyFor("anthropic", "claude-sonnet-4-6", "", "https://api.anthropic.com/")).toBe(
      "ANTHROPIC_API_KEY",
    );
  });

  it("promises nothing for a keyless vLLM preset or custom row with no base URL, and agrees with the server's preview rule", () => {
    // The vLLM presets ship with no base URL; the server refuses them a masked preview for the
    // same reason the field must not read "leave empty to use OPENAI_API_KEY": a self-hosted
    // id would be run against api.openai.com. Both sides read core's modelEnvPreviewKey.
    const cases: Array<[string, string, string, string]> = [
      ...MODEL_CATALOG.filter((m) => m.provider === "vllm").map(
        (m): [string, string, string, string] => [
          m.provider,
          m.modelId,
          m.clientType ?? "",
          m.baseUrl ?? "",
        ],
      ),
      ["custom", "local-model", "openai-chat", ""],
      ["custom", "gpt-5.6", "openai-chat", "https://api.openai.com/v1"],
      ["anthropic", "claude-sonnet-4-6", "", ""],
      ["anthropic", "claude-sonnet-4-6", "", "https://proxy.example/anthropic"],
      ["tokendance", "glm-5.3", "openai-chat", "https://tokendance.space/gateway/v1"],
      ["penguin-go", "gemini-3.8-flash", "gemini-3.8", "https://token.penguin.ooo/api"],
    ];
    for (const [provider, modelId, clientType, baseUrl] of cases) {
      expect(envHintKeyFor(provider, modelId, clientType, baseUrl), `${provider}/${modelId}`).toBe(
        modelEnvPreviewKey({
          provider,
          modelId,
          clientType: envHintClientType(provider, clientType),
          baseUrl,
        }),
      );
    }
    for (const m of MODEL_CATALOG.filter((v) => v.provider === "vllm")) {
      expect(envHintKeyFor("vllm", m.modelId, "", ""), m.modelId).toBeUndefined();
    }
    expect(envHintKeyFor("custom", "local-model", "openai-chat", "")).toBeUndefined();
  });
});

describe("protocolForPersist (an empty protocol must never reach the config)", () => {
  it("falls back to openai-chat for a custom-like entry that still has none", () => {
    // AutoLLMClient THROWS on an unmatched client type, so persisting "" would save a
    // model that cannot start.
    expect(protocolForPersist("custom", "")).toBe("openai-chat");
    expect(protocolForPersist("my-group", "   ")).toBe("openai-chat");
  });

  it("keeps an explicit protocol exactly as chosen", () => {
    expect(protocolForPersist("custom", "ant-messages")).toBe("ant-messages");
    expect(protocolForPersist("custom", "openai-responses")).toBe("openai-responses");
    expect(protocolForPersist("my-group", " openai-chat ")).toBe("openai-chat");
  });

  it("writes a pinned group's protocol rather than leaving it to inference", () => {
    expect(protocolForPersist("vllm", "")).toBe("openai-chat-vllm-adapter");
    expect(protocolForPersist("vllm", "   ")).toBe("openai-chat-vllm-adapter");
    // An explicit value still wins — this is the last-resort net, not an override.
    expect(protocolForPersist("vllm", "openai-chat")).toBe("openai-chat");
  });

  it("leaves preset / vendor groups empty so AgentHub still infers from the model id", () => {
    expect(protocolForPersist("openai", "")).toBe("");
    expect(protocolForPersist("anthropic", "")).toBe("");
    expect(protocolForPersist("google", "")).toBe("");
  });
});

describe("rowToEntry (the persistence funnel)", () => {
  const row = (over: Partial<RowState>): RowState => ({
    provider: "custom",
    modelId: "my-model",
    original: null,
    vision: true,
    contextWindow: "",
    maxTokens: "",
    fastMode: false,
    clientType: "",
    cacheRead: "",
    cacheWrite: "",
    output: "",
    baseUrl: "",
    originalBaseUrl: "",
    apiKeyInput: "",
    clearApiKey: false,
    ...over,
  });

  it("never writes a custom entry without a protocol, even when the form left it empty", () => {
    expect(rowToEntry(row({})).clientType).toBe("openai-chat");
    expect(rowToEntry(row({ provider: "my-group" })).clientType).toBe("openai-chat");
  });

  it("writes the chosen protocol verbatim", () => {
    expect(rowToEntry(row({ clientType: "ant-messages" })).clientType).toBe("ant-messages");
  });

  it("omits clientType for a vendor group so AgentHub keeps inferring it", () => {
    expect(rowToEntry(row({ provider: "openai", modelId: "gpt-5.6" })).clientType).toBeUndefined();
  });

  it("sends a promotion only when the preset sync declared one", () => {
    // A promotion the page merely loaded is left to the server, which keeps it unless the entry
    // renames the row or changes its price.
    expect(rowToEntry(row({ provider: "penguin-go", discount: 0.5 }))).not.toHaveProperty(
      "discount",
    );
    expect(rowToEntry(row({ discount: 0.2, discountDeclared: true })).discount).toBe(0.2);
    // Declared with no promotion: an explicit clear.
    expect(rowToEntry(row({ discountDeclared: true })).discount).toBeNull();
  });
});

describe("detection copy", () => {
  it("has exactly one user-facing failure message, in both locales", () => {
    // Every failure mode collapses to this: the maintainer's point is that a user cannot
    // act on "the endpoint responded but serves no protocol", only on the key and the URL.
    for (const catalog of [EN.models, ZH.models] as const) {
      expect(catalog.detectFailedBody).toBeTruthy();
      // No protocol names to parse, and no "pick one manually" instruction.
      expect(catalog.detectFailedBody).not.toContain("OpenAI Responses");
      expect(catalog.detectFailedBody).not.toContain("Anthropic Messages");
    }
    // The distinguishing strings are gone, not merely unused.
    expect("detectNone" in EN.models).toBe(false);
    expect("detectUnreachable" in EN.models).toBe(false);
    expect("detectNone" in ZH.models).toBe(false);
    expect("detectUnreachable" in ZH.models).toBe(false);
  });

  it("names the unset protocol as a placeholder rather than a protocol", () => {
    for (const catalog of [EN.models, ZH.models] as const) {
      expect(catalog.protocolUnset).toBeTruthy();
      expect(Object.values(catalog.protocolNames)).not.toContain(catalog.protocolUnset);
    }
  });

  it("carries no dialog-era copy: both outcomes are toasts, so no popup titles remain", () => {
    // The verdict used to be a blocking AlertModal (which needed an accessible name) and,
    // before that, a line rendered under the base URL field. Both are gone: a detection
    // result is transient and must occupy no space in the form.
    for (const catalog of [EN.models, ZH.models] as const) {
      expect("detectOkTitle" in catalog).toBe(false);
      expect("detectFailedTitle" in catalog).toBe(false);
    }
  });

  it("keeps both toast messages to a single short line", () => {
    for (const catalog of [EN.models, ZH.models] as const) {
      const success = catalog.detectedProtocol("OpenAI Responses");
      for (const text of [success, catalog.detectFailedBody]) {
        expect(text.length).toBeLessThanOrEqual(80);
        expect(text).not.toContain("\n");
      }
    }
  });
});

describe("protocolPathForModel (generic protocol client types)", () => {
  it("maps each protocol client to the path its AgentHub client appends", () => {
    expect(protocolPathForModel("custom", "openai-responses")).toBe("/responses");
    expect(protocolPathForModel("custom", "ant-messages")).toBe("/v1/messages");
    expect(protocolPathForModel("custom", "openai-chat")).toBe("/chat/completions");
    // The bare alias and user-defined groups behave the same as before.
    expect(protocolPathForModel("my-group", "openai")).toBe("/chat/completions");
  });

  it("keeps the legacy explicit-type and group fallbacks intact", () => {
    expect(protocolPathForModel("custom", "claude-4-8")).toBe("/v1/messages");
    expect(protocolPathForModel("openai", "")).toBe("/responses");
    expect(protocolPathForModel("anthropic", "")).toBe("/v1/messages");
    expect(protocolPathForModel("custom", "")).toBe("/chat/completions");
  });
});