Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix(inference): only set JSON response_format for schema requests (#2926)

eviaaaaa545e6f80fc66
diff --git a/packages/shared/inference.test.ts b/packages/shared/inference.test.ts
new file mode 100644
index 00000000..109958d4
--- /dev/null
+++ b/packages/shared/inference.test.ts
@@ -0,0 +1,94 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { z } from "zod";
+
+import { OpenAIInferenceClient } from "./inference";
+import type { OpenAIInferenceConfig } from "./inference";
+
+const capturedBodies: Record<string, unknown>[] = [];
+const tagSchema = z.object({ tags: z.array(z.string()) });
+
+vi.mock("openai", () => {
+ const OpenAIMock = vi.fn().mockImplementation(() => ({
+ chat: {
+ completions: {
+ create: vi.fn(async (body: Record<string, unknown>) => {
+ capturedBodies.push(body);
+ return {
+ choices: [{ message: { content: "{}" } }],
+ usage: { total_tokens: 1 },
+ };
+ }),
+ },
+ },
+ }));
+
+ return { default: OpenAIMock };
+});
+
+vi.mock("openai/helpers/zod", () => ({
+ zodResponseFormat: (schema: unknown, name: string) => ({
+ type: "json_schema",
+ json_schema: { name, schema },
+ }),
+}));
+
+function makeConfig(
+ outputSchema: OpenAIInferenceConfig["outputSchema"],
+): OpenAIInferenceConfig {
+ return {
+ apiKey: "test-key",
+ textModel: "test-text-model",
+ imageModel: "test-image-model",
+ contextLength: 2048,
+ maxOutputTokens: 1024,
+ useMaxCompletionTokens: false,
+ outputSchema,
+ };
+}
+
+describe("OpenAIInferenceClient response_format", () => {
+ beforeEach(() => {
+ capturedBodies.length = 0;
+ });
+
+ it("omits response_format for schema-less text inference in json mode", async () => {
+ const client = new OpenAIInferenceClient(makeConfig("json"));
+
+ await client.inferFromText("summarize this text", { schema: null });
+
+ expect(capturedBodies).toHaveLength(1);
+ expect(capturedBodies[0].response_format).toBeUndefined();
+ });
+
+ it("keeps json_object for schema-backed text inference in json mode", async () => {
+ const client = new OpenAIInferenceClient(makeConfig("json"));
+
+ await client.inferFromText("infer tags as json", { schema: tagSchema });
+
+ expect(capturedBodies).toHaveLength(1);
+ expect(capturedBodies[0].response_format).toEqual({ type: "json_object" });
+ });
+
+ it("omits response_format for schema-less image inference in json mode", async () => {
+ const client = new OpenAIInferenceClient(makeConfig("json"));
+
+ await client.inferFromImage("describe this image", "image/png", "BASE64", {
+ schema: null,
+ });
+
+ expect(capturedBodies).toHaveLength(1);
+ expect(capturedBodies[0].response_format).toBeUndefined();
+ });
+
+ it("keeps structured response_format for schema-backed text inference in structured mode", async () => {
+ const client = new OpenAIInferenceClient(makeConfig("structured"));
+
+ await client.inferFromText("infer tags", { schema: tagSchema });
+
+ expect(capturedBodies).toHaveLength(1);
+ expect(capturedBodies[0].response_format).toMatchObject({
+ type: "json_schema",
+ json_schema: { name: "schema" },
+ });
+ });
+});
diff --git a/packages/shared/inference.ts b/packages/shared/inference.ts
index a7c7e9c8..10ef73fd 100644
--- a/packages/shared/inference.ts
+++ b/packages/shared/inference.ts
@@ -136,6 +136,24 @@ const mapInferenceOutputSchema = <
return opts[type];
};
+const mapOpenAIResponseFormat = (
+ schema: z.ZodSchema | null,
+ outputSchema: typeof serverConfig.inference.outputSchema,
+) => {
+ if (schema === null) {
+ return undefined;
+ }
+
+ return mapInferenceOutputSchema(
+ {
+ structured: zodResponseFormat(schema, "schema"),
+ json: { type: "json_object" as const },
+ plain: undefined,
+ },
+ outputSchema,
+ );
+};
+
export interface OpenAIInferenceConfig {
apiKey: string;
baseURL?: string;
@@ -221,14 +239,8 @@ export class OpenAIInferenceClient implements InferenceClient {
...(this.config.useMaxCompletionTokens
? { max_completion_tokens: this.config.maxOutputTokens }
: { max_tokens: this.config.maxOutputTokens }),
- response_format: mapInferenceOutputSchema(
- {
- structured: optsWithDefaults.schema
- ? zodResponseFormat(optsWithDefaults.schema, "schema")
- : undefined,
- json: { type: "json_object" },
- plain: undefined,
- },
+ response_format: mapOpenAIResponseFormat(
+ optsWithDefaults.schema,
this.config.outputSchema,
),
reasoning_effort: this.config.reasoningEffort,
@@ -264,14 +276,8 @@ export class OpenAIInferenceClient implements InferenceClient {
...(this.config.useMaxCompletionTokens
? { max_completion_tokens: this.config.maxOutputTokens }
: { max_tokens: this.config.maxOutputTokens }),
- response_format: mapInferenceOutputSchema(
- {
- structured: optsWithDefaults.schema
- ? zodResponseFormat(optsWithDefaults.schema, "schema")
- : undefined,
- json: { type: "json_object" },
- plain: undefined,
- },
+ response_format: mapOpenAIResponseFormat(
+ optsWithDefaults.schema,
this.config.outputSchema,
),
messages: [