The minimalistic document archiving platform.
fix(ai): openai doesn't respect all json schemas (#1320)
Corentin Thomasset2ef314835ee3
diff --git a/.changeset/forty-readers-feel.md b/.changeset/forty-readers-feel.mdnew file mode 100644index 00000000..68643ebd--- /dev/null+++ b/.changeset/forty-readers-feel.md@@ -0,0 +1,5 @@+---+'@papra/app': patch+---++Fix auto tagging schema definition for some ai providers (notably OpenAI and Anthropic), as they do not support all JSON schema features. It's really nice to have a common spec, if only all providers respected it...diff --git a/apps/papra-server/package.json b/apps/papra-server/package.jsonindex 0041b60b..380b6bf6 100644--- a/apps/papra-server/package.json+++ b/apps/papra-server/package.json@@ -64,9 +64,9 @@"@papra/webhooks": "workspace:*","@paralleldrive/cuid2": "catalog:","@sindresorhus/slugify": "^3.0.0",- "@tanstack/ai": "^0.28.0",- "@tanstack/ai-anthropic": "^0.15.11",- "@tanstack/ai-openai": "^0.14.1",+ "@tanstack/ai": "^0.39.1",+ "@tanstack/ai-anthropic": "^0.16.0",+ "@tanstack/ai-openai": "^0.15.10","@valibot/to-json-schema": "^1.6.0","better-auth": "catalog:","busboy": "^1.6.0",diff --git a/apps/papra-server/src/modules/auto-tagging/auto-tagging.models.test.ts b/apps/papra-server/src/modules/auto-tagging/auto-tagging.models.test.tsindex bdc135aa..9d0ec974 100644--- a/apps/papra-server/src/modules/auto-tagging/auto-tagging.models.test.ts+++ b/apps/papra-server/src/modules/auto-tagging/auto-tagging.models.test.ts@@ -106,13 +106,16 @@ describe('auto-tagging.models', () => {const existingTags = [{ id: 'tag_1', name: 'Finance' },{ id: 'tag_2', name: 'Household' },+ { id: 'tag_3', name: 'Gardening' },+ { id: 'tag_4', name: 'Travel' },];test('when the creation of new tags is not allowed and there are existing tags, the prediction returns an array of existing tags name', () => {expect(getTagsActions({- requestedTags: ['Finance', 'Household'],+ autoTaggingResponse: { existingTags: ['Finance', 'Household'] },existingTags,+ maxTags: 5,}),).to.eql({tagIdsToAdd: ['tag_1', 'tag_2'],@@ -123,11 +126,12 @@ describe('auto-tagging.models', () => {test('when the creation of new tags is allowed and there are existing tags, the prediction returns an array of existing tags name and new tags', () => {expect(getTagsActions({- requestedTags: [- { type: 'existing', name: 'Finance' },- { type: 'new', name: 'New Tag', description: 'A new tag description' },- ],+ autoTaggingResponse: {+ existingTags: ['Finance'],+ newTags: [{ name: 'New Tag', description: 'A new tag description' }],+ },existingTags,+ maxTags: 5,}),).to.eql({tagIdsToAdd: ['tag_1'],@@ -138,20 +142,123 @@ describe('auto-tagging.models', () => {test('when the creation of new tags is allowed and there are no existing tags, the prediction returns an array of new tags', () => {expect(getTagsActions({- requestedTags: [- { name: 'New Tag 1', description: 'Description 1' },- { name: 'New Tag 2' },- ],+ autoTaggingResponse: {+ newTags: [{ name: 'New Tag 1', description: 'Description 1' }, { name: 'New Tag 2' }],+ },+ maxTags: 5,existingTags: [],}),).to.eql({tagIdsToAdd: [],tagsToCreate: [{ name: 'New Tag 1', description: 'Description 1' },- { name: 'New Tag 2', description: undefined },+ { name: 'New Tag 2' },],});});++ test('should not happen because of guards, but when the creation of new tags is not allowed and there are no existing tags, the prediction returns an empty array', () => {+ expect(+ getTagsActions({+ autoTaggingResponse: {},+ maxTags: 5,+ existingTags: [],+ }),+ ).to.eql({+ tagIdsToAdd: [],+ tagsToCreate: [],+ });+ });++ test('duplicated existing are deduplicated and only one tag is returned', () => {+ expect(+ getTagsActions({+ autoTaggingResponse: { existingTags: ['Finance', 'Finance'] },+ existingTags,+ maxTags: 5,+ }),+ ).to.eql({+ tagIdsToAdd: ['tag_1'],+ tagsToCreate: [],+ });+ });++ test('the number of tags returned should not exceed the maxTags limit, the extra tags should be ignored, with the priority given to existing tags over new tags', () => {+ expect(+ getTagsActions({+ autoTaggingResponse: {+ existingTags: ['Finance', 'Household', 'Gardening', 'Travel'],+ },+ maxTags: 3,+ existingTags,+ }),+ ).to.eql({+ tagIdsToAdd: ['tag_1', 'tag_2', 'tag_3'],+ tagsToCreate: [],+ });++ expect(+ getTagsActions({+ autoTaggingResponse: {+ newTags: [+ { name: 'New Tag 1' },+ { name: 'New Tag 2' },+ { name: 'New Tag 3' },+ { name: 'New Tag 4' },+ ],+ },+ maxTags: 3,+ existingTags,+ }),+ ).to.eql({+ tagIdsToAdd: [],+ tagsToCreate: [{ name: 'New Tag 1' }, { name: 'New Tag 2' }, { name: 'New Tag 3' }],+ });++ expect(+ getTagsActions({+ autoTaggingResponse: {+ existingTags: ['Finance', 'Household'],+ newTags: [{ name: 'New Tag 1' }, { name: 'New Tag 2' }],+ },+ maxTags: 3,+ existingTags,+ }),+ ).to.eql({+ tagIdsToAdd: ['tag_1', 'tag_2'],+ tagsToCreate: [{ name: 'New Tag 1' }],+ });++ expect(+ getTagsActions({+ autoTaggingResponse: {+ existingTags: ['Finance', 'Household', 'Gardening', 'Travel'],+ newTags: [{ name: 'New Tag 1' }, { name: 'New Tag 2' }],+ },+ maxTags: 3,+ existingTags,+ }),+ ).to.eql({+ tagIdsToAdd: ['tag_1', 'tag_2', 'tag_3'],+ tagsToCreate: [],+ });+ });++ test('duplicated tags do not count towards the maxTags limit', () => {+ expect(+ getTagsActions({+ autoTaggingResponse: {+ existingTags: ['Finance', 'Finance', 'Household'],+ newTags: [{ name: 'New Tag 1' }, { name: 'New Tag 2' }],+ },+ maxTags: 3,+ existingTags,+ }),+ ).to.eql({+ tagIdsToAdd: ['tag_1', 'tag_2'],+ tagsToCreate: [{ name: 'New Tag 1' }],+ });+ });});});});diff --git a/apps/papra-server/src/modules/auto-tagging/auto-tagging.models.ts b/apps/papra-server/src/modules/auto-tagging/auto-tagging.models.tsindex 6a691d7e..c4dc5700 100644--- a/apps/papra-server/src/modules/auto-tagging/auto-tagging.models.ts+++ b/apps/papra-server/src/modules/auto-tagging/auto-tagging.models.ts@@ -1,4 +1,5 @@import * as v from 'valibot';+import { uniq } from '../shared/utils';function buildTagLine({ name, description }: { name: string; description?: string | null }) {return description ? `- ${name}: ${description}` : `- ${name}`;@@ -50,92 +51,59 @@ export function buildAutoTaggingUserPrompt({return [`Document name: ${document.name}`, 'Document content:', document.content].join('\n');}-type AutoTaggingResponse = v.InferOutput<ReturnType<typeof buildAutoTaggingSchema>>;+export type AutoTaggingResponse = {+ existingTags?: string[];+ newTags?: { name: string; description?: string }[];+};export function buildAutoTaggingSchema({existingTags,- maxTags,canCreateNewTags,}: {existingTags: { name: string; description?: string | null }[];- maxTags: number;canCreateNewTags: boolean;-}) {- if (existingTags.length === 0) {- return v.pipe(- v.array(- v.object({- name: v.string(),- description: v.optional(v.string()),- }),- ),- v.maxLength(maxTags),- );+}): v.GenericSchema<AutoTaggingResponse> {+ const hasExistingTags = existingTags.length > 0;++ if (canCreateNewTags && hasExistingTags) {+ return v.object({+ existingTags: v.array(v.picklist(existingTags.map((tag) => tag.name))),+ newTags: v.array(v.object({ name: v.string(), description: v.optional(v.string()) })),+ });}- if (!canCreateNewTags) {- return v.pipe(v.array(v.picklist(existingTags.map((tag) => tag.name))), v.maxLength(maxTags));+ if (canCreateNewTags && !hasExistingTags) {+ return v.object({+ newTags: v.array(v.object({ name: v.string(), description: v.optional(v.string()) })),+ });}- return v.pipe(- v.array(- v.union([- v.object({- type: v.literal('existing'),- name: v.picklist(existingTags.map((tag) => tag.name)),- }),- v.object({- type: v.literal('new'),- name: v.string(),- description: v.optional(v.string()),- }),- ]),- ),- v.maxLength(maxTags),- );+ return v.object({+ existingTags: v.array(v.picklist(existingTags.map((tag) => tag.name))),+ });}export function getTagsActions({- requestedTags,+ autoTaggingResponse,existingTags,+ maxTags,}: {- requestedTags: AutoTaggingResponse;+ autoTaggingResponse: AutoTaggingResponse;existingTags: { name: string; id: string }[];-}) {- const tagIdsToAdd: string[] = [];- const tagsToCreate: { name: string; description?: string }[] = [];+ maxTags: number;+}): {+ tagIdsToAdd: string[];+ tagsToCreate: { name: string; description?: string }[];+} {+ const { existingTags: requestedExistingTagNames = [], newTags = [] } = autoTaggingResponse;const existingTagNamesMap = new Map(existingTags.map((tag) => [tag.name, tag.id]));- const addExistingTagByName = (tagName: string) => {- const existingTag = existingTagNamesMap.get(tagName);- if (existingTag) {- tagIdsToAdd.push(existingTag);- }- };-- for (const tag of requestedTags) {- // Prediction returns a string array when they are only existing tags, none to create- if (typeof tag === 'string') {- addExistingTagByName(tag);- continue;- }-- if ('type' in tag) {- if (tag.type === 'existing') {- addExistingTagByName(tag.name);- continue;- }+ const tagIdsToAdd = uniq(+ requestedExistingTagNames.map((tagName) => existingTagNamesMap.get(tagName)).filter(Boolean),+ ).slice(0, maxTags);- if (tag.type === 'new') {- tagsToCreate.push({ name: tag.name, description: tag.description });- continue;- }- }-- tagsToCreate.push({ name: tag.name, description: tag.description });- continue;- }+ const tagsToCreate = newTags.slice(0, maxTags - tagIdsToAdd.length);return { tagIdsToAdd, tagsToCreate };}diff --git a/apps/papra-server/src/modules/auto-tagging/auto-tagging.usecases.test.ts b/apps/papra-server/src/modules/auto-tagging/auto-tagging.usecases.test.tsindex e1b11035..8a2999a7 100644--- a/apps/papra-server/src/modules/auto-tagging/auto-tagging.usecases.test.ts+++ b/apps/papra-server/src/modules/auto-tagging/auto-tagging.usecases.test.ts@@ -9,6 +9,7 @@ import { createTestLogger } from '../shared/logger/logger.test-utils';import { createTagsRepository } from '../tags/tags.repository';import { documentsTagsTable, tagsTable } from '../tags/tags.table';import { autoTagDocument } from './auto-tagging.usecases';+import type { AutoTaggingResponse } from './auto-tagging.models';function createTestResolveOrganizationSettings(autoTagging: Partial<{ isEnabled: boolean; canCreateNewTags: boolean; maxTags: number }> = {},@@ -28,7 +29,7 @@ function createTestResolveOrganizationSettings(});}-function createTestAiServices({ response }: { response: unknown }) {+function createTestAiServices({ response }: { response: AutoTaggingResponse }) {const generateStructuredData = vi.fn().mockResolvedValue(response);return {@@ -78,7 +79,7 @@ describe('auto-tagging usecases', () => {],});const { aiServices, generateStructuredData } = createTestAiServices({- response: ['Finance'],+ response: { existingTags: ['Finance'] },});await autoTagDocument({@@ -100,7 +101,9 @@ describe('auto-tagging usecases', () => {organizations: [{ id: 'org_1', name: 'Org 1' }],documents: [baseDocument],});- const { aiServices, generateStructuredData } = createTestAiServices({ response: [] });+ const { aiServices, generateStructuredData } = createTestAiServices({+ response: { existingTags: [] },+ });await autoTagDocument({...deps,@@ -140,7 +143,7 @@ describe('auto-tagging usecases', () => {},],});- const { aiServices } = createTestAiServices({ response: ['Finance'] });+ const { aiServices } = createTestAiServices({ response: { existingTags: ['Finance'] } });await autoTagDocument({...deps,@@ -166,7 +169,7 @@ describe('auto-tagging usecases', () => {documents: [baseDocument],});const { aiServices } = createTestAiServices({- response: [{ name: 'Invoice', description: 'Invoices and receipts' }],+ response: { newTags: [{ name: 'Invoice', description: 'Invoices and receipts' }] },});await autoTagDocument({diff --git a/apps/papra-server/src/modules/auto-tagging/auto-tagging.usecases.ts b/apps/papra-server/src/modules/auto-tagging/auto-tagging.usecases.tsindex 32c54b38..2bb03763 100644--- a/apps/papra-server/src/modules/auto-tagging/auto-tagging.usecases.ts+++ b/apps/papra-server/src/modules/auto-tagging/auto-tagging.usecases.ts@@ -33,16 +33,17 @@ export async function promptForAutoTagging({maxTags: number;modelId: string;}) {- const response = await aiServices.generateStructuredData({+ const autoTaggingResponse = await aiServices.generateStructuredData({modelId,- schema: buildAutoTaggingSchema({ existingTags, canCreateNewTags, maxTags }),+ schema: buildAutoTaggingSchema({ existingTags, canCreateNewTags }),systemPrompt: buildAutoTaggingSystemPrompt({ existingTags, canCreateNewTags, maxTags }),userPrompt: buildAutoTaggingUserPrompt({ document }),});const { tagIdsToAdd, tagsToCreate } = getTagsActions({- requestedTags: response,+ autoTaggingResponse,existingTags,+ maxTags,});return { tagIdsToAdd, tagsToCreate };diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 3db9e18a..88cac2be 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -480,14 +480,14 @@ importers:specifier: ^3.0.0version: 3.0.0'@tanstack/ai':- specifier: ^0.28.0- version: 0.28.0+ specifier: ^0.39.1+ version: 0.39.1'@tanstack/ai-anthropic':- specifier: ^0.15.11- version: 0.15.11(@tanstack/[email protected])([email protected])+ specifier: ^0.16.0+ version: 0.16.0(@tanstack/[email protected])([email protected])'@tanstack/ai-openai':- specifier: ^0.14.1- version: 0.14.4(@tanstack/[email protected])(@tanstack/[email protected])([email protected])([email protected])+ specifier: ^0.15.10'@valibot/to-json-schema':specifier: ^1.6.0version: 1.6.0([email protected]([email protected]))@@ -4571,49 +4571,26 @@ packages:'@swc/[email protected]':resolution: {integrity: sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==}- '@tanstack/[email protected]':- resolution: {integrity: sha512-4H/5sSNYeR5ToIrc2TtNUHlDfYjgYqbTv8hY5LGfLgJq/cKvU+p/WT5u+MHBZlMnOXk44X23QryDOI78b94WqA==}+ '@tanstack/[email protected]':+ resolution: {integrity: sha512-O7p9G9Ms6AyrusLn0a9yOHFMeFxhrQldNRndX+KXmBhozYm7D0KQgFxGNKbOLeNsummdpRfOQpjRIo073cS1RQ==}peerDependencies:- '@tanstack/ai': ^0.38.0+ '@tanstack/ai': ^0.39.1zod: ^4.0.0- '@tanstack/[email protected]':- resolution: {integrity: sha512-Bhla99u9zBDio1Vm2zfFn75wYCaUi2+ijsL6MEMYPfqPK9BhXSTFHFGlo8EDrw0P0LaDI56g05gncVsgxNjpgw==}+ '@tanstack/[email protected]':+ resolution: {integrity: sha512-h7/HLz9u2LF9ba6uKFMnSZkFlkzQONJ5u3Hi+4qGxpsHyvcGRtW6v186jaFLoklndhEwyC49ytkC4eafi7Qq8w==}- '@tanstack/[email protected]':- resolution: {integrity: sha512-DoF6LKkGCVeF0N6Vky3DdhYxseTZfvTpOkhEQud/Za/GLQ+5p1xH4egQX8oZJkx1S6HZEZTQA1hKSGFYGULtFw==}+ '@tanstack/[email protected]':+ resolution: {integrity: sha512-S/mbplx7sf3lhJrMZBTIBteaaWID6LuNBW8MkOk1I12x0fCmHxZaisf4fKILekcrZ6WniqVNyOLKrJnDD+T2PQ==}peerDependencies:- '@tanstack/ai': 0.28.0-- '@tanstack/[email protected]':- resolution: {integrity: sha512-407B7PZqpTu5aaafi5HnRmK2gTNoa3mIhCF5lL1tgbEJsiomS0+mZuUa9i+BVUVX3+EnlwurQdSVRVgTy6NpVg==}- peerDependencies:- '@tanstack/ai': 0.31.0-- '@tanstack/[email protected]':- resolution: {integrity: sha512-jcKPuI+6GeqQQsBCJj01TIX6PhCeqAn0BAd52Fshoeo+KvczzLzY5yhu+ntds52dtZu2+QMfeIFhM6KLlnwEFg==}- peerDependencies:- '@tanstack/ai': ^0.31.0- '@tanstack/ai-client': ^0.17.3+ '@tanstack/ai': ^0.39.0zod: ^4.0.0- '@tanstack/[email protected]':- resolution: {integrity: sha512-QZNgbhoPhh5snlMf21SuF4QNTJme4aldvBx2ppMefqfMQHBQPFDJl6TxHDNW8voIzQe42rE5U9XfoWrS0K/xFQ==}-'@tanstack/[email protected]':resolution: {integrity: sha512-fgbjd5DohL3k5rTWr/KInauLVYMiHVm0cnmTsNrzL3cr4wWhEld5vmFSrjiwUWACAuONjZa+uBXuS+ZSO8fwDQ==}- '@tanstack/[email protected]':- resolution: {integrity: sha512-DT02H5CeqtsdH90bb1hx3zIxboiyMjuQPzbj7ov3uVkMWIFjuCjhgJbc7w27AY38ewd4d8G63QHYj4AnMSEB9Q==}- engines: {node: '>=18'}- peerDependencies:- '@opentelemetry/api': '>=1.9.0'- peerDependenciesMeta:- '@opentelemetry/api':- optional: true-- '@tanstack/[email protected]':- resolution: {integrity: sha512-DsOaeDCbyF8J8i6e8ldvM+iqDjQa0xMG8cvlfzAghpOV+lxMRdkdX16bopxN2gn9C9LQ6hnXtqajRzRPB34dfA==}+ '@tanstack/[email protected]':+ resolution: {integrity: sha512-lTmwGJ5/hzzr/qGuh56zuD7SjWiZu3jpOQMEEtgw6ImuWjQlsBdw5dKV+oykw3y9FzLLP1LlKb9gAHZ0OR2Fyg==}engines: {node: '>=18'}peerDependencies:'@opentelemetry/api': '>=1.9.0'@@ -4633,10 +4610,10 @@ packages:'@tanstack/[email protected]':resolution: {integrity: sha512-OEWW2uTOFMyRmHrVEiPOn+J27ekQ/vXwRAJt9kD8U8vCt8CmpClj989OOGGSBSVJtDNxGUcWyKF8gYznnqIyaw==}- '@tanstack/[email protected]':- resolution: {integrity: sha512-3z+5MkGWMT/irCA7eE0qmMUpusm9Kk+Rb1oyCp1LZPRXIOrmz6ozcIlrW+CN5ABix8diH5KbUDRZoEywyf1BQQ==}+ '@tanstack/[email protected]':+ resolution: {integrity: sha512-vHF5VTKrLrb0c+fLm4hdd5xOKsYUI1/Qt3bLaFYMeUogfYq1LIZlEGUb4EJaFaMYLrjIuep+exjpaHx0TY9kMQ==}peerDependencies:- '@tanstack/ai': ^0.31.0+ '@tanstack/ai': ^0.39.0'@tanstack/[email protected]':resolution: {integrity: sha512-vGY+CWsFZeac3dELgB6UZ4c7OacwsLb8hvL2gLS6hTgy8Fl0Bm/aLokHaeDIP+q9F9HUZTnp360z9uv78eg8pg==}@@ -14746,57 +14723,35 @@ snapshots:dependencies:tslib: 2.8.1dependencies:'@anthropic-ai/sdk': 0.97.1([email protected])- '@tanstack/ai': 0.28.0+ '@tanstack/ai': 0.39.1'@tanstack/ai-utils': 0.3.1zod: 4.4.3- '@tanstack/[email protected]':- dependencies:- '@tanstack/ai': 0.31.0- '@tanstack/ai-event-client': 0.6.2(@tanstack/[email protected])- transitivePeerDependencies:- - '@opentelemetry/api'-- '@tanstack/[email protected](@tanstack/[email protected])':+ '@tanstack/[email protected]':dependencies:- '@tanstack/ai': 0.28.0'@tanstack/devtools-event-client': 0.4.4- '@tanstack/[email protected](@tanstack/[email protected])':dependencies:- '@tanstack/ai': 0.31.0- '@tanstack/devtools-event-client': 0.4.4-- '@tanstack/[email protected](@tanstack/[email protected])(@tanstack/[email protected])([email protected])([email protected])':- dependencies:- '@tanstack/ai': 0.28.0- '@tanstack/ai-client': 0.17.3- '@tanstack/ai-utils': 0.2.2+ '@tanstack/ai': 0.39.1+ '@tanstack/ai-utils': 0.3.1openai: 6.44.0([email protected])([email protected])zod: 4.4.3transitivePeerDependencies:- ws- '@tanstack/[email protected]': {}-'@tanstack/[email protected]': {}- '@tanstack/[email protected]':+ '@tanstack/[email protected]':dependencies:'@ag-ui/core': 0.0.52'@standard-schema/spec': 1.1.0- '@tanstack/ai-event-client': 0.5.4(@tanstack/[email protected])- partial-json: 0.1.7-- '@tanstack/[email protected]':- dependencies:- '@ag-ui/core': 0.0.52- '@standard-schema/spec': 1.1.0- '@tanstack/ai-event-client': 0.6.2(@tanstack/[email protected])+ '@tanstack/ai-event-client': 0.6.8+ '@tanstack/ai-utils': 0.3.1partial-json: 0.1.7'@tanstack/[email protected]': {}@@ -14809,10 +14764,10 @@ snapshots:'@tanstack/pacer': 0.15.4'@tanstack/store': 0.7.7dependencies:- '@tanstack/ai': 0.28.0- '@tanstack/ai-utils': 0.2.2+ '@tanstack/ai': 0.39.1+ '@tanstack/ai-utils': 0.3.1openai: 6.44.0([email protected])([email protected])transitivePeerDependencies:- ws