Skip to content

The open alternative to Salesforce, designed for AI.

Build a strict schema for agent structured output (#26281)

nitinfe9819f2803b
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
index 908cea19000..0f24579585d 100644
--- a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/services/agent-async-executor.service.ts
@@ -48,7 +48,9 @@ import { WORKFLOW_AGENT_REGISTRY_TOOL_CATEGORIES } from 'src/engine/metadata-mod
import { RunAgentAttachmentService } from 'src/engine/metadata-modules/ai/ai-agent-execution/services/run-agent-attachment.service';
import { type AgentExecutionResult } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-execution-result.type';
import { type AgentToolLoadingStrategy } from 'src/engine/metadata-modules/ai/ai-agent-execution/types/agent-tool-loading-strategy.type';
+import { assertAgentResponseFormatHasOutputFieldsOrThrow } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/assert-agent-response-format-has-output-fields-or-throw.util';
import { buildAgentRolePermissionConfig } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-agent-role-permission-config.util';
+import { buildStrictAgentResponseSchema } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-strict-agent-response-schema.util';
import { AGENT_CONFIG } from 'src/engine/metadata-modules/ai/ai-agent/constants/agent-config.const';
import { STRUCTURED_OUTPUT_SYSTEM_PROMPT } from 'src/engine/metadata-modules/ai/ai-agent/constants/structured-output-system-prompt.const';
import { type AgentEntity } from 'src/engine/metadata-modules/ai/ai-agent/entities/agent.entity';
@@ -288,6 +290,8 @@ export class AgentAsyncExecutorService {
);
}
+ assertAgentResponseFormatHasOutputFieldsOrThrow(agent?.responseFormat);
+
await this.aiBillingService.assertAiExecutionAllowed({
workspaceId,
operationType,
@@ -511,7 +515,9 @@ export class AgentAsyncExecutorService {
Execution Results: ${textResponse.text}
Please generate the structured output based on the execution results and context above.`,
- output: Output.object({ schema: jsonSchema(agentSchema) }),
+ output: Output.object({
+ schema: jsonSchema(buildStrictAgentResponseSchema(agentSchema)),
+ }),
providerOptions: getCallLevelProviderOptions({
sdkPackage: registeredModel.sdkPackage,
providerOptions: undefined,
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/assert-agent-response-format-has-output-fields-or-throw.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/assert-agent-response-format-has-output-fields-or-throw.util.spec.ts
new file mode 100644
index 00000000000..d8874a9fa7b
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/assert-agent-response-format-has-output-fields-or-throw.util.spec.ts
@@ -0,0 +1,51 @@
+import { type AgentResponseFormat } from 'twenty-shared/ai';
+
+import { assertAgentResponseFormatHasOutputFieldsOrThrow } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/assert-agent-response-format-has-output-fields-or-throw.util';
+import {
+ AiException,
+ AiExceptionCode,
+} from 'src/engine/metadata-modules/ai/ai.exception';
+
+describe('assertAgentResponseFormatHasOutputFieldsOrThrow', () => {
+ it('does not throw without a response format', () => {
+ expect(() =>
+ assertAgentResponseFormatHasOutputFieldsOrThrow(undefined),
+ ).not.toThrow();
+ });
+
+ it('does not throw for a text response format', () => {
+ expect(() =>
+ assertAgentResponseFormatHasOutputFieldsOrThrow({ type: 'text' }),
+ ).not.toThrow();
+ });
+
+ it('does not throw for a json response format with output fields', () => {
+ expect(() =>
+ assertAgentResponseFormatHasOutputFieldsOrThrow({
+ type: 'json',
+ schema: {
+ type: 'object',
+ properties: { summary: { type: 'string' } },
+ },
+ }),
+ ).not.toThrow();
+ });
+
+ it('throws an invalid agent input error when a json schema has no properties', () => {
+ const responseFormatSavedWithoutProperties = {
+ type: 'json',
+ schema: { type: 'object' },
+ } as AgentResponseFormat;
+
+ expect(() =>
+ assertAgentResponseFormatHasOutputFieldsOrThrow(
+ responseFormatSavedWithoutProperties,
+ ),
+ ).toThrow(
+ new AiException(
+ 'This agent has no output fields. Add at least one output field to its response format.',
+ AiExceptionCode.INVALID_AGENT_INPUT,
+ ),
+ );
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-strict-agent-response-schema.util.spec.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-strict-agent-response-schema.util.spec.ts
new file mode 100644
index 00000000000..d2f668f76f8
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/__tests__/build-strict-agent-response-schema.util.spec.ts
@@ -0,0 +1,37 @@
+import { buildStrictAgentResponseSchema } from 'src/engine/metadata-modules/ai/ai-agent-execution/utils/build-strict-agent-response-schema.util';
+
+describe('buildStrictAgentResponseSchema', () => {
+ it('adds additionalProperties false and requires every property when both are missing', () => {
+ expect(
+ buildStrictAgentResponseSchema({
+ type: 'object',
+ properties: {
+ summary: { type: 'string', description: 'Summary' },
+ score: { type: 'number' },
+ },
+ }),
+ ).toEqual({
+ type: 'object',
+ properties: {
+ summary: { type: 'string', description: 'Summary' },
+ score: { type: 'number' },
+ },
+ required: ['summary', 'score'],
+ additionalProperties: false,
+ });
+ });
+
+ it('requires every property when only some are listed as required', () => {
+ expect(
+ buildStrictAgentResponseSchema({
+ type: 'object',
+ properties: {
+ summary: { type: 'string' },
+ isQualified: { type: 'boolean' },
+ },
+ required: ['summary'],
+ additionalProperties: false,
+ }).required,
+ ).toEqual(['summary', 'isQualified']);
+ });
+});
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/assert-agent-response-format-has-output-fields-or-throw.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/assert-agent-response-format-has-output-fields-or-throw.util.ts
new file mode 100644
index 00000000000..61c9462cd1f
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/assert-agent-response-format-has-output-fields-or-throw.util.ts
@@ -0,0 +1,25 @@
+import { msg } from '@lingui/core/macro';
+import { type AgentResponseFormat } from 'twenty-shared/ai';
+import { isDefined } from 'twenty-shared/utils';
+
+import {
+ AiException,
+ AiExceptionCode,
+} from 'src/engine/metadata-modules/ai/ai.exception';
+
+export const assertAgentResponseFormatHasOutputFieldsOrThrow = (
+ responseFormat: AgentResponseFormat | undefined,
+): void => {
+ if (
+ responseFormat?.type === 'json' &&
+ !isDefined(responseFormat.schema?.properties)
+ ) {
+ throw new AiException(
+ 'This agent has no output fields. Add at least one output field to its response format.',
+ AiExceptionCode.INVALID_AGENT_INPUT,
+ {
+ userFriendlyMessage: msg`This agent has no output fields. Add at least one output field to its response format.`,
+ },
+ );
+ }
+};
diff --git a/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-strict-agent-response-schema.util.ts b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-strict-agent-response-schema.util.ts
new file mode 100644
index 00000000000..515e6b881f6
--- /dev/null
+++ b/packages/twenty-server/src/engine/metadata-modules/ai/ai-agent-execution/utils/build-strict-agent-response-schema.util.ts
@@ -0,0 +1,9 @@
+import { type AgentResponseSchema } from 'twenty-shared/ai';
+
+export const buildStrictAgentResponseSchema = (
+ schema: AgentResponseSchema,
+): AgentResponseSchema => ({
+ ...schema,
+ required: Object.keys(schema.properties),
+ additionalProperties: false,
+});
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/__tests__/validate-agent-response-format.util.spec.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/__tests__/validate-agent-response-format.util.spec.ts
index e41bfc7b42b..32ba640460f 100644
--- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/__tests__/validate-agent-response-format.util.spec.ts
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/__tests__/validate-agent-response-format.util.spec.ts
@@ -59,7 +59,7 @@ describe('validateAgentResponseFormat', () => {
expect(errors[0].code).toBe(AiExceptionCode.INVALID_AGENT_INPUT);
});
- it('should not throw when a json schema is missing its properties', () => {
+ it('should return an error without throwing when a json schema is missing its properties', () => {
// Legacy or API-provided data can omit properties despite the type
const malformedFormat = {
type: 'json',
@@ -69,9 +69,13 @@ describe('validateAgentResponseFormat', () => {
expect(() =>
validateAgentResponseFormat({ responseFormat: malformedFormat }),
).not.toThrow();
- expect(
- validateAgentResponseFormat({ responseFormat: malformedFormat }),
- ).toEqual([]);
+
+ const errors = validateAgentResponseFormat({
+ responseFormat: malformedFormat,
+ });
+
+ expect(errors).toHaveLength(1);
+ expect(errors[0].code).toBe(AiExceptionCode.INVALID_AGENT_INPUT);
});
it('should report every invalid property name at once', () => {
diff --git a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util.ts b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util.ts
index b9235bdb4b4..c40d8248a1d 100644
--- a/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util.ts
+++ b/packages/twenty-server/src/engine/workspace-manager/workspace-migration/workspace-migration-builder/validators/utils/validate-agent-response-format.util.ts
@@ -33,6 +33,18 @@ export const validateAgentResponseFormat = ({
});
}
+ if (
+ type === 'json' &&
+ isDefined(responseFormat.schema) &&
+ !isDefined(responseFormat.schema.properties)
+ ) {
+ errors.push({
+ code: AiExceptionCode.INVALID_AGENT_INPUT,
+ message: t`Response format with type "json" must include output fields`,
+ userFriendlyMessage: msg`Add at least one output field to the JSON response format`,
+ });
+ }
+
if (type === 'json' && isDefined(responseFormat.schema)) {
const invalidPropertyNames = Object.keys(
responseFormat.schema.properties ?? {},