Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

feat(ai): the five simple AI steps run on the worker, not in the sandbox (#15493)

Abdulcc09b97f8679
diff --git a/packages/core/execution/src/lib/workers/job-data.ts b/packages/core/execution/src/lib/workers/job-data.ts
index 57f97da272a..f8fa01b0a25 100644
--- a/packages/core/execution/src/lib/workers/job-data.ts
+++ b/packages/core/execution/src/lib/workers/job-data.ts
@@ -418,7 +418,8 @@ const AiStepJobBase = z.object({
platformId: z.string(),
flowId: z.string(),
flowRunId: z.string(),
- waitpointId: z.string(),
+ waitpointId: z.string().optional(),
+ webserverId: z.string().optional(),
provider: z.enum(AIProviderName),
providerConfigId: z.string().optional(),
modelId: z.string(),
diff --git a/packages/pieces/community/ai/package.json b/packages/pieces/community/ai/package.json
index dcf8940898c..03281a67820 100644
--- a/packages/pieces/community/ai/package.json
+++ b/packages/pieces/community/ai/package.json
@@ -1,6 +1,6 @@
{
"name": "@activepieces/piece-ai",
- "version": "0.10.3",
+ "version": "0.11.0",
"type": "commonjs",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
diff --git a/packages/pieces/community/ai/src/lib/actions/image/generate-image.ts b/packages/pieces/community/ai/src/lib/actions/image/generate-image.ts
index 4a194288f9a..1fdb5f3e25c 100644
--- a/packages/pieces/community/ai/src/lib/actions/image/generate-image.ts
+++ b/packages/pieces/community/ai/src/lib/actions/image/generate-image.ts
@@ -1,25 +1,17 @@
import {
+ AIProviderName,
ApFile,
createAction,
DynamicPropsValue,
+ getEffectiveProviderAndModel,
InputPropertyMap,
+ isNil,
PieceAuth,
Property,
+ spreadIfDefined,
} from '@activepieces/pieces-framework';
-import {
- GeneratedFile,
- generateText,
- GenerateTextResult,
- ImagePart,
- LanguageModel,
- ToolSet,
-} from 'ai';
-import { generateImage } from 'ai';
import mime from 'mime-types';
-import { isNil } from '@activepieces/pieces-framework';
-import { getEffectiveProviderAndModel, spreadIfDefined } from '@activepieces/pieces-framework';
-import { createAIModel } from '../../common/ai-sdk';
-import { AIProviderName } from '@activepieces/pieces-framework';
+import { runOnWorker, uploadAiFiles } from '../../common/ai-step';
import { aiProps, aiProviderSelection } from '../../common/props';
export const generateImageAction = createAction({
@@ -142,39 +134,46 @@ export const generateImageAction = createAction({
},
async run(context) {
const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);
- const modelId = context.propsValue.model;
const inputImages = collectInputImages({
inputImages: context.propsValue.inputImages,
advancedOptions: context.propsValue.advancedOptions,
});
- const image = await getGeneratedImage({
- provider,
- ...spreadIfDefined('configId', configId),
- modelId,
- engineToken: context.server.token,
- apiUrl: context.server.apiUrl,
- prompt: context.propsValue.prompt,
- inputImages,
- projectId: context.project.id,
- flowId: context.flows.current.id,
- runId: context.run.id,
- advancedOptions: context.propsValue.advancedOptions,
+ const result = await runOnWorker({
+ context,
+ buildRequest: async () => ({
+ action: 'GENERATE_IMAGE',
+ provider,
+ ...spreadIfDefined('providerConfigId', configId),
+ modelId: context.propsValue.model,
+ prompt: context.propsValue.prompt,
+ files: await uploadAiFiles({ context, files: inputImages, mimeTypeOf: inputImageMimeType }),
+ ...spreadIfDefined('advancedOptions', withoutInputImages(context.propsValue.advancedOptions)),
+ }),
});
- const imageData =
- image.base64 && image.base64.length > 0
- ? Buffer.from(image.base64, 'base64')
- : Buffer.from(image.uint8Array);
+ if (result.status === 'paused') {
+ return {};
+ }
- return context.files.write({
- data: imageData,
- fileName: 'image.png',
- });
+ return result.output.answer;
},
});
+const withoutInputImages = (advancedOptions: DynamicPropsValue | undefined): Record<string, unknown> | undefined => {
+ if (isNil(advancedOptions)) {
+ return undefined;
+ }
+ const { image: _inputImages, ...rest } = advancedOptions as Record<string, unknown>;
+ return rest;
+};
+
+const inputImageMimeType = (file: ApFile): string => {
+ const detected = file.extension ? mime.lookup(file.extension) : false;
+ return detected && ALLOWED_IMAGE_MIME_TYPES.has(detected) ? detected : 'image/jpeg';
+};
+
const collectInputImages = ({
inputImages,
advancedOptions,
@@ -206,150 +205,6 @@ const extractImageFiles = (value: unknown): ApFile[] => {
});
};
-const getGeneratedImage = async ({
- provider,
- configId,
- modelId,
- engineToken,
- apiUrl,
- prompt,
- inputImages,
- projectId,
- flowId,
- runId,
- advancedOptions,
-}: {
- provider: AIProviderName;
- configId?: string;
- modelId: string;
- engineToken: string;
- apiUrl: string;
- prompt: string;
- inputImages: ApFile[];
- projectId: string;
- flowId: string;
- runId: string;
- advancedOptions?: DynamicPropsValue;
-}): Promise<GeneratedFile> => {
- const model = await createAIModel({
- provider,
- ...spreadIfDefined('configId', configId),
- modelId,
- engineToken,
- apiUrl,
- projectId,
- flowId,
- runId,
- isImage: true,
- });
-
- const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider, model: modelId });
- const resolvedProvider = (effectiveProvider ?? provider) as AIProviderName;
-
- const hasInputImages = inputImages.length > 0;
-
- return withImageInputErrorContext({ modelId, hasInputImages }, async () => {
- switch (resolvedProvider) {
- case AIProviderName.GOOGLE:
- case AIProviderName.ACTIVEPIECES:
- case AIProviderName.OPENROUTER:
- case AIProviderName.CLOUDFLARE_GATEWAY:
- return generateImageUsingGenerateText({
- model: model as unknown as LanguageModel,
- prompt,
- inputImages,
- });
- default: {
- const sanitizedAdvancedOptions = stripLegacyImageField(advancedOptions);
- const sdkImages = inputImages.map((file) =>
- Buffer.from(file.base64, 'base64'),
- );
- const { image } = await generateImage({
- model,
- prompt: hasInputImages
- ? { text: prompt, images: sdkImages }
- : prompt,
- providerOptions: {
- [resolvedProvider]: { ...sanitizedAdvancedOptions },
- },
- });
- return image;
- }
- }
- });
-};
-
-const withImageInputErrorContext = async <T>(
- { modelId, hasInputImages }: { modelId: string; hasInputImages: boolean },
- run: () => Promise<T>,
-): Promise<T> => {
- try {
- return await run();
- } catch (error) {
- if (!hasInputImages) {
- throw error;
- }
- const original = error instanceof Error ? error.message : String(error);
- throw new Error(
- `Image generation failed for model "${modelId}". ` +
- `This model may not support input images. Try a model that supports image editing — ` +
- `for example gpt-image-1, dall-e-2, or a Gemini Nano Banana model — or remove the input images. ` +
- `Original error: ${original}`,
- );
- }
-};
-
-const generateImageUsingGenerateText = async ({
- model,
- prompt,
- inputImages,
-}: {
- model: LanguageModel;
- prompt: string;
- inputImages: ApFile[];
-}): Promise<GeneratedFile> => {
- const imageFiles = inputImages.map<ImagePart>((file) => {
- const detected = file.extension ? mime.lookup(file.extension) : false;
- const fileType =
- detected && ALLOWED_IMAGE_MIME_TYPES.has(detected)
- ? detected
- : 'image/jpeg';
-
- return {
- type: 'image',
- image: `data:${fileType};base64,${file.base64}`,
- };
- });
-
- const result = await generateText({
- model,
- providerOptions: {
- google: { responseModalities: ['TEXT', 'IMAGE'] },
- openrouter: { modalities: ['image', 'text'] },
- },
- messages: [
- {
- role: 'user',
- content: [{ type: 'text', text: prompt }, ...imageFiles],
- },
- ],
- });
-
- assertImageGenerationSuccess(result);
-
- return result.files[0];
-};
-
-const stripLegacyImageField = (
- advancedOptions: DynamicPropsValue | undefined,
-): DynamicPropsValue | undefined => {
- if (isNil(advancedOptions)) {
- return advancedOptions;
- }
- const { image: _legacy, ...rest } = advancedOptions as Record<string, unknown>;
- return rest as DynamicPropsValue;
-};
-
const ALLOWED_IMAGE_MIME_TYPES: ReadonlySet<string> = new Set([
'image/jpeg',
'image/png',
@@ -358,34 +213,6 @@ const ALLOWED_IMAGE_MIME_TYPES: ReadonlySet<string> = new Set([
'image/avif',
]);
-const assertImageGenerationSuccess = (
- result: GenerateTextResult<ToolSet, never>
-): void => {
- const responseBody =
- result.response.body &&
- typeof result.response.body === 'object' &&
- 'candidates' in result.response.body
- ? result.response.body
- : { candidates: [] };
-
- const responseCandidates = Array.isArray(responseBody?.candidates)
- ? responseBody?.candidates
- : [];
-
- responseCandidates.forEach((candidate: { finishReason: string }) => {
- if (candidate.finishReason !== 'STOP') {
- throw new Error(
- 'Image generation failed Reason:\n ' +
- JSON.stringify(responseCandidates, null, 2)
- );
- }
- });
-
- if (isNil(result.files) || result.files.length === 0) {
- throw new Error('No image generated');
- }
-};
-
function isGptImageModel({ modelId }: { modelId: string }): boolean {
return modelId.startsWith('gpt-image');
}
diff --git a/packages/pieces/community/ai/src/lib/actions/output-shape.test.ts b/packages/pieces/community/ai/src/lib/actions/output-shape.test.ts
new file mode 100644
index 00000000000..ea8e9d09126
--- /dev/null
+++ b/packages/pieces/community/ai/src/lib/actions/output-shape.test.ts
@@ -0,0 +1,326 @@
+import { httpClient } from '@activepieces/pieces-common';
+import { createMockActionContext } from '@activepieces/pieces-framework';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { generateImageAction } from './image/generate-image';
+import { askAI } from './text/ask-ai';
+import { summarizeText } from './text/summarize-text';
+import { classifyText } from './utility/classify-text';
+import { extractStructuredData } from './utility/extract-structured-data';
+
+vi.mock('@activepieces/pieces-common', async (importOriginal) => {
+ const original = await importOriginal<Record<string, unknown>>();
+ return {
+ ...original,
+ httpClient: { sendRequest: vi.fn(async () => ({ body: { requestId: 'request-1' } })) },
+ };
+});
+
+const sendRequest = vi.mocked(httpClient.sendRequest);
+
+const provider = { provider: 'openai', configId: 'config-1' };
+
+function answered(answer: unknown, conversation?: unknown) {
+ return { body: { output: { answer, conversation } } };
+}
+
+function requestBody(callIndex = 0): Record<string, unknown> {
+ return sendRequest.mock.calls[callIndex][0].body as Record<string, unknown>;
+}
+
+beforeEach(() => {
+ sendRequest.mockClear();
+ sendRequest.mockResolvedValue({ body: { requestId: 'request-1' } } as never);
+});
+
+describe('the shape every AI action returns once the worker answers', () => {
+ it('Ask AI returns the answer as a bare string', async () => {
+ const output = await askAI.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },
+ resumePayload: answered('sunny'),
+ }));
+
+ expect(output).toBe('sunny');
+ });
+
+ it('Ask AI returns text and sources together when web search asks for sources', async () => {
+ const output = await askAI.run(createMockActionContext({
+ propsValue: {
+ provider,
+ model: 'gpt-test',
+ prompt: 'weather?',
+ webSearch: true,
+ webSearchOptions: { includeSources: true },
+ maxOutputTokens: 2000,
+ },
+ resumePayload: answered({ text: 'sunny', sources: [{ id: 'source-1' }] }),
+ }));
+
+ expect(output).toEqual({ text: 'sunny', sources: [{ id: 'source-1' }] });
+ });
+
+ it('Summarize Text returns the summary as a bare string', async () => {
+ const output = await summarizeText.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', text: 'a long article', prompt: 'Summarize', maxOutputTokens: 2000 },
+ resumePayload: answered('the short version'),
+ }));
+
+ expect(output).toBe('the short version');
+ });
+
+ it('Classify Text returns the chosen label as a bare string', async () => {
+ const output = await classifyText.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', text: 'it is warm today', categories: ['sunny', 'rainy'] },
+ resumePayload: answered('sunny'),
+ }));
+
+ expect(output).toBe('sunny');
+ });
+
+ it('Extract Structured Data returns the extracted object itself', async () => {
+ const output = await extractStructuredData.run(createMockActionContext({
+ propsValue: {
+ provider,
+ model: 'gpt-test',
+ text: 'an invoice totalling 42',
+ mode: 'simple',
+ schema: { fields: [{ name: 'total', type: 'number', isRequired: true }] },
+ maxOutputTokens: 2000,
+ },
+ resumePayload: answered({ total: 42 }),
+ }));
+
+ expect(output).toEqual({ total: 42 });
+ });
+
+ it('Generate Image returns the stored file url as a bare string', async () => {
+ const output = await generateImageAction.run(createMockActionContext({
+ propsValue: { provider, model: 'dall-e-3', prompt: 'a cat', advancedOptions: {} },
+ resumePayload: answered('test-file-url'),
+ }));
+
+ expect(output).toBe('test-file-url');
+ });
+});
+
+describe('handing the step to the worker', () => {
+ it('pauses the flow on the waitpoint it just created', async () => {
+ const waitedOn: string[] = [];
+
+ const output = await askAI.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },
+ onWaitForWaitpoint: (waitpointId) => waitedOn.push(waitpointId),
+ }));
+
+ expect(output).toEqual({});
+ expect(waitedOn).toEqual(['test-waitpoint-id']);
+ expect(requestBody()['waitpointId']).toBe('test-waitpoint-id');
+ });
+
+ it('gives the waitpoint a backstop so a dead worker cannot hang the run forever', async () => {
+ const created: { resumeDateTime?: string }[] = [];
+
+ await askAI.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },
+ onCreateWaitpoint: (waitpoint) => created.push(waitpoint),
+ }));
+
+ expect(created).toHaveLength(1);
+ expect(new Date(created[0].resumeDateTime as string).getTime()).toBeGreaterThan(Date.now());
+ });
+
+ it('asks for the answer in the same request when the step cannot pause', async () => {
+ sendRequest.mockResolvedValue(answered('sunny') as never);
+
+ const output = await askAI.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },
+ canPause: false,
+ }));
+
+ expect(output).toBe('sunny');
+ expect(requestBody()['waitpointId']).toBeUndefined();
+ });
+
+ it('raises the failure the worker reported', async () => {
+ await expect(askAI.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },
+ resumePayload: { body: { failure: 'the model refused' } },
+ }))).rejects.toThrow('the model refused');
+ });
+
+ it('raises a clear error when the backstop fires with nothing to report', async () => {
+ await expect(askAI.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },
+ resumePayload: { body: undefined },
+ }))).rejects.toThrow('did not report a result');
+ });
+});
+
+describe('what a saved flow sends over the wire', () => {
+ it('turns categories saved as numbers into strings', async () => {
+ await classifyText.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', text: 'two', categories: [1, 2] },
+ }));
+
+ expect(requestBody()['categories']).toEqual(['1', '2']);
+ });
+
+ it('skips an empty file row instead of sending it', async () => {
+ await extractStructuredData.run(createMockActionContext({
+ propsValue: {
+ provider,
+ model: 'gpt-test',
+ text: 'an invoice',
+ files: [{ file: undefined }],
+ mode: 'simple',
+ schema: { fields: [] },
+ },
+ }));
+
+ expect(requestBody()['files']).toEqual([]);
+ });
+
+ it('reads an Ask AI history stored in the legacy messages wrapper', async () => {
+ const context = createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', prompt: 'and then?', conversationKey: 'thread-1', webSearch: false },
+ });
+ context.store.get = (async () => ({ messages: [{ role: 'user', content: 'hello' }] })) as typeof context.store.get;
+
+ await askAI.run(context);
+
+ expect(requestBody()['conversation']).toEqual([{ role: 'user', content: 'hello' }]);
+ });
+
+ it('still sends an input image pinned under the legacy advanced option', async () => {
+ await generateImageAction.run(createMockActionContext({
+ propsValue: {
+ provider,
+ model: 'gemini-2.5-flash-image',
+ prompt: 'a cat',
+ advancedOptions: { image: [{ file: { filename: 'cat.png', data: Buffer.from('png'), extension: 'png' } }] },
+ },
+ }));
+
+ expect(requestBody()['files']).toEqual([{ fileId: 'test-file-id', mimeType: 'image/png', filename: 'cat.png' }]);
+ expect(requestBody()['advancedOptions']).toEqual({});
+ });
+
+ it('turns a web search limit saved as text into the number the schema accepts', async () => {
+ await askAI.run(createMockActionContext({
+ propsValue: {
+ provider,
+ model: 'gpt-test',
+ prompt: 'weather?',
+ webSearch: true,
+ webSearchOptions: { maxUses: '7', allowedDomains: ['example.com'] },
+ },
+ }));
+
+ const webSearch = requestBody()['webSearch'] as { options: Record<string, unknown> };
+ expect(webSearch.options['maxUses']).toBe(7);
+ expect(webSearch.options['allowedDomains']).toEqual([{ domain: 'example.com' }]);
+ });
+
+ it('drops a stale Include Sources left behind by a provider that has no native search', async () => {
+ await askAI.run(createMockActionContext({
+ propsValue: {
+ provider: { provider: 'openrouter', configId: 'config-1' },
+ model: 'some/model',
+ prompt: 'weather?',
+ webSearch: true,
+ webSearchOptions: { includeSources: true },
+ },
+ }));
+
+ const webSearch = requestBody()['webSearch'] as { options: Record<string, unknown> };
+ expect(webSearch.options).not.toHaveProperty('includeSources');
+ });
+
+ it('still runs an Extract whose only file row is empty, as it did before', async () => {
+ await extractStructuredData.run(createMockActionContext({
+ propsValue: {
+ provider,
+ model: 'gpt-test',
+ files: [{ file: undefined }],
+ mode: 'simple',
+ schema: { fields: [] },
+ },
+ }));
+
+ expect(requestBody()['files']).toEqual([]);
+ });
+
+ it('sends the creativity as a temperature the model understands', async () => {
+ await askAI.run(createMockActionContext({
+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', creativity: 50, webSearch: false },
+ }));
+
+ expect(requestBody()['temperature']).toBe(0.5);
+ });
+});
+
+describe('what a resumed step does before it reads its answer', () => {
+ function fileProp() {
+ return { file: { filename: 'invoice.pdf', extension: 'pdf', base64: 'eA==', data: Buffer.from('x') } };
+ }
+
+ function countingUploads<T extends { files: { upload: unknown } }>(context: T): { context: T, uploads: () => number } {
+ let uploads = 0;
+ context.files.upload = async () => {
+ uploads += 1;
+ return { id: 'test-file-id', url: 'test-file-url' };
+ };
+ return { context, uploads: () => uploads };
+ }
+
+ it('Extract Structured Data does not upload its attachments a second time', async () => {
+ const { context, uploads } = countingUploads(createMockActionContext({
+ propsValue: {
+ provider,
+ model: 'gpt-test',
+ text: 'an invoice totalling 42',
+ files: [fileProp()],
+ mode: 'simple',
+ schema: { fields: [{ name: 'total', type: 'number', isRequired: true }] },
+ maxOutputTokens: 2000,
+ },
+ resumePayload: answered({ total: 42 }),
+ }));
+
+ const output = await extractStructuredData.run(context);
+
+ expect(output).toEqual({ total: 42 });
+ expect(uploads()).toBe(0);
+ expect(sendRequest).not.toHaveBeenCalled();
+ });
+
+ it('Generate Image does not upload its input images a second time', async () => {
+ const { context, uploads } = countingUploads(createMockActionContext({
+ propsValue: { provider, model: 'dall-e-3', prompt: 'a cat', inputImages: [fileProp()], advancedOptions: {} },
+ resumePayload: answered('test-file-url'),
+ }));
+
+ const output = await generateImageAction.run(context);
+
+ expect(output).toBe('test-file-url');
+ expect(uploads()).toBe(0);
+ expect(sendRequest).not.toHaveBeenCalled();
+ });
+
+ it('still uploads on the first run, so the worker has something to read', async () => {
+ const { context, uploads } = countingUploads(createMockActionContext({
+ propsValue: {
+ provider,
+ model: 'gpt-test',
+ text: 'an invoice totalling 42',
+ files: [fileProp()],
+ mode: 'simple',
+ schema: { fields: [{ name: 'total', type: 'number', isRequired: true }] },
+ maxOutputTokens: 2000,
+ },
+ }));
+
+ await extractStructuredData.run(context);
+
+ expect(uploads()).toBe(1);
+ });
+});
diff --git a/packages/pieces/community/ai/src/lib/actions/text/ask-ai.ts b/packages/pieces/community/ai/src/lib/actions/text/ask-ai.ts
index 03de83ac509..773ef92e11e 100644
--- a/packages/pieces/community/ai/src/lib/actions/text/ask-ai.ts
+++ b/packages/pieces/community/ai/src/lib/actions/text/ask-ai.ts
@@ -1,8 +1,7 @@
-import { ModelMessage, generateText, stepCountIs } from 'ai';
-import { AIProviderName, createAction, getEffectiveProviderAndModel, isNil, Property, spreadIfDefined } from '@activepieces/pieces-framework';
+import { createAction, isNil, Property, spreadIfDefined } from '@activepieces/pieces-framework';
+import { runOnWorker } from '../../common/ai-step';
import { aiProps, aiProviderSelection } from '../../common/props';
-import { createAIModel } from '../../common/ai-sdk';
-import { buildWebSearchOptionsProperty, buildWebSearchConfig, WebSearchOptions } from '../../common/web-search';
+import { buildWebSearchOptionsProperty, sanitizeWebSearchOptions, usesNativeWebSearchTools } from '../../common/web-search';
export const askAI = createAction({
audience: 'both',
@@ -50,84 +49,67 @@ export const askAI = createAction({
},
async run(context) {
const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);
- const modelId = context.propsValue.model;
- const storage = context.store;
const webSearchEnabled = !!context.propsValue.webSearch;
- const webSearchOptions = (context.propsValue.webSearchOptions ?? {}) as WebSearchOptions;
-
- const { tools: webSearchTools, providerOptions } = buildWebSearchConfig({
+ const webSearchOptions = webSearchOptionsToSend({
provider,
- model: modelId,
- webSearchEnabled,
- webSearchOptions,
+ model: context.propsValue.model,
+ saved: context.propsValue.webSearchOptions,
});
+ const storageKey = conversationStorageKey(context.propsValue.conversationKey);
- const { provider: effectiveProvider } = getEffectiveProviderAndModel({
- provider,
- model: modelId,
- });
- const model = await createAIModel({
- provider,
- ...spreadIfDefined('configId', configId),
- modelId,
- engineToken: context.server.token,
- apiUrl: context.server.apiUrl,
- projectId: context.project.id,
- flowId: context.flows.current.id,
- runId: context.run.id,
- ...spreadIfDefined('openaiResponsesModel', webSearchEnabled && effectiveProvider === AIProviderName.OPENAI ? true : undefined),
+ const result = await runOnWorker({
+ context,
+ buildRequest: async () => ({
+ action: 'ASK_AI',
+ provider,
+ ...spreadIfDefined('providerConfigId', configId),
+ modelId: context.propsValue.model,
+ prompt: context.propsValue.prompt,
+ ...spreadIfDefined('maxOutputTokens', context.propsValue.maxOutputTokens),
+ ...spreadIfDefined('temperature', isNil(context.propsValue.creativity) ? undefined : context.propsValue.creativity / 100),
+ ...spreadIfDefined('conversation', isNil(storageKey) ? undefined : await readConversation(context.store, storageKey)),
+ webSearch: { enabled: webSearchEnabled, options: webSearchOptions },
+ }),
});
- const conversationKey = context.propsValue.conversationKey
- ? `ask-ai-conversation:${context.propsValue.conversationKey}`
- : null;
+ if (result.status === 'paused') {
+ return {};
+ }
- let conversation = null;
- if (conversationKey) {
- conversation = (await storage.get<ModelMessage[]>(conversationKey)) ?? [];
- if (!conversation) {
- await storage.put(conversationKey, { messages: [] });
- }
+ if (!isNil(storageKey) && !isNil(result.output.conversation)) {
+ await context.store.put(storageKey, result.output.conversation);
}
- const stopWhen = webSearchTools
- ? stepCountIs(webSearchOptions?.maxUses ?? 5)
- : undefined;
+ return result.output.answer;
+ },
+});
- const response = await generateText({
- model,
- messages: [
- ...(conversation ?? []),
- {
- role: 'user',
- content: context.propsValue.prompt,
- },
- ],
- maxOutputTokens: context.propsValue.maxOutputTokens,
- ...spreadIfDefined('temperature', isNil(context.propsValue.creativity) ? undefined : context.propsValue.creativity / 100),
- tools: webSearchTools,
- stopWhen,
- providerOptions,
- });
+function webSearchOptionsToSend({ provider, model, saved }: { provider: string; model: string; saved: unknown }): Record<string, unknown> {
+ const options = sanitizeWebSearchOptions(saved);
+ if (usesNativeWebSearchTools({ provider, model })) {
+ return options;
+ }
+ const { includeSources: _onlyWithNativeTools, ...rest } = options;
+ return rest;
+}
- conversation?.push({
- role: 'user',
- content: context.propsValue.prompt,
- });
+function conversationStorageKey(conversationKey: string | undefined): string | null {
+ return conversationKey ? `ask-ai-conversation:${conversationKey}` : null;
+}
- conversation?.push({
- role: 'assistant',
- content: response.text ?? '',
- });
+async function readConversation(store: ConversationStore, storageKey: string): Promise<Record<string, unknown>[]> {
+ const stored = await store.get<StoredConversation>(storageKey);
+ if (Array.isArray(stored)) {
+ return stored;
+ }
+ if (!isNil(stored) && Array.isArray(stored.messages)) {
+ return stored.messages;
+ }
+ return [];
+}
- if (conversationKey) {
- await storage.put(conversationKey, conversation);
- }
+type StoredConversation = Record<string, unknown>[] | { messages?: Record<string, unknown>[] };
- const includeSources = webSearchTools && webSearchOptions.includeSources;
- if (includeSources) {
- return { text: response.text, sources: response.sources };
- }
- return response.text;
- },
-});
+type ConversationStore = {
+ get<T>(key: string): Promise<T | null>;
+};
diff --git a/packages/pieces/community/ai/src/lib/actions/text/summarize-text.ts b/packages/pieces/community/ai/src/lib/actions/text/summarize-text.ts
index e60eb26c65e..b85d4b48f89 100644
--- a/packages/pieces/community/ai/src/lib/actions/text/summarize-text.ts
+++ b/packages/pieces/community/ai/src/lib/actions/text/summarize-text.ts
@@ -1,7 +1,5 @@
-import { AIProviderName, spreadIfDefined } from '@activepieces/pieces-framework';
-import { createAIModel } from '../../common/ai-sdk';
-import { createAction, Property } from '@activepieces/pieces-framework';
-import { generateText } from 'ai';
+import { createAction, Property, spreadIfDefined } from '@activepieces/pieces-framework';
+import { runOnWorker } from '../../common/ai-step';
import { aiProps, aiProviderSelection } from '../../common/props';
export const summarizeText = createAction({
@@ -32,35 +30,24 @@ export const summarizeText = createAction({
},
async run(context) {
const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);
- const modelId = context.propsValue.model;
- const model = await createAIModel({
- provider,
- ...spreadIfDefined('configId', configId),
- modelId,
- engineToken: context.server.token,
- apiUrl: context.server.apiUrl,
- projectId: context.project.id,
- flowId: context.flows.current.id,
- runId: context.run.id,
+ const result = await runOnWorker({
+ context,
+ buildRequest: async () => ({
+ action: 'SUMMARIZE_TEXT',
+ provider,
+ ...spreadIfDefined('providerConfigId', configId),
+ modelId: context.propsValue.model,
+ prompt: context.propsValue.prompt,
+ text: context.propsValue.text,
+ ...spreadIfDefined('maxOutputTokens', context.propsValue.maxOutputTokens),
+ }),
});
- const response = await generateText({
- model,
- messages: [
- {
- role: 'user',
- content: `${context.propsValue.prompt} Summarize the following text : ${context.propsValue.text}`
- },
- ],
- maxOutputTokens: context.propsValue.maxOutputTokens,
- providerOptions: {
- [provider]: {
- ...(provider === AIProviderName.OPENAI ? { reasoning_effort: 'minimal' } : {}),
- }
- }
- });
+ if (result.status === 'paused') {
+ return {};
+ }
- return response.text ?? '';
+ return result.output.answer;
},
});
diff --git a/packages/pieces/community/ai/src/lib/actions/text/temperature.test.ts b/packages/pieces/community/ai/src/lib/actions/text/temperature.test.ts
index 7bd72926d73..946480e50b4 100644
--- a/packages/pieces/community/ai/src/lib/actions/text/temperature.test.ts
+++ b/packages/pieces/community/ai/src/lib/actions/text/temperature.test.ts
@@ -1,19 +1,18 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { httpClient } from '@activepieces/pieces-common';
import { createMockActionContext } from '@activepieces/pieces-framework';
-import { generateText } from 'ai';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
import { askAI } from './ask-ai';
import { summarizeText } from './summarize-text';
-vi.mock('ai', () => ({
- generateText: vi.fn(async () => ({ text: 'ok', sources: [] })),
- stepCountIs: vi.fn(),
-}));
-
-vi.mock('../../common/ai-sdk', () => ({
- createAIModel: vi.fn(async () => ({})),
-}));
+vi.mock('@activepieces/pieces-common', async (importOriginal) => {
+ const original = await importOriginal<Record<string, unknown>>();
+ return {
+ ...original,
+ httpClient: { sendRequest: vi.fn(async () => ({ body: { requestId: 'request-1' } })) },
+ };
+});
-const generateTextMock = vi.mocked(generateText);
+const sendRequest = vi.mocked(httpClient.sendRequest);
const baseProps = {
provider: { provider: 'openai', configId: 'config1' },
@@ -21,41 +20,41 @@ const baseProps = {
maxOutputTokens: 2000,
};
-async function askAiGenerateTextArgs({ creativity }: { creativity: number | null | undefined }) {
+async function askAiRequestBody({ creativity }: { creativity: number | null | undefined }) {
await askAI.run(createMockActionContext({
propsValue: { ...baseProps, prompt: 'hello', webSearch: false, creativity },
}));
- return generateTextMock.mock.calls[0][0];
+ return sendRequest.mock.calls[0][0].body as Record<string, unknown>;
}
beforeEach(() => {
- generateTextMock.mockClear();
+ sendRequest.mockClear();
});
describe('askAI temperature', () => {
it('omits temperature when creativity is not set', async () => {
- const args = await askAiGenerateTextArgs({ creativity: undefined });
- expect(args).not.toHaveProperty('temperature');
+ const body = await askAiRequestBody({ creativity: undefined });
+ expect(body).not.toHaveProperty('temperature');
});
it('omits temperature when creativity is null', async () => {
- const args = await askAiGenerateTextArgs({ creativity: null });
- expect(args).not.toHaveProperty('temperature');
+ const body = await askAiRequestBody({ creativity: null });
+ expect(body).not.toHaveProperty('temperature');
});
it('sends temperature scaled from an explicit creativity', async () => {
- const args = await askAiGenerateTextArgs({ creativity: 50 });
- expect(args.temperature).toBe(0.5);
+ const body = await askAiRequestBody({ creativity: 50 });
+ expect(body['temperature']).toBe(0.5);
});
it('sends temperature 1 for the previously seeded default of 100', async () => {
- const args = await askAiGenerateTextArgs({ creativity: 100 });
- expect(args.temperature).toBe(1);
+ const body = await askAiRequestBody({ creativity: 100 });
+ expect(body['temperature']).toBe(1);
});
it('sends temperature 0 when creativity is 0', async () => {
- const args = await askAiGenerateTextArgs({ creativity: 0 });
- expect(args.temperature).toBe(0);
+ const body = await askAiRequestBody({ creativity: 0 });
+ expect(body['temperature']).toBe(0);
});
});
@@ -64,6 +63,6 @@ describe('summarizeText temperature', () => {
await summarizeText.run(createMockActionContext({
propsValue: { ...baseProps, text: 'long text', prompt: 'Summarize' },
}));
- expect(generateTextMock.mock.calls[0][0]).not.toHaveProperty('temperature');
+ expect(sendRequest.mock.calls[0][0].body).not.toHaveProperty('temperature');
});
});
diff --git a/packages/pieces/community/ai/src/lib/actions/utility/classify-text.ts b/packages/pieces/community/ai/src/lib/actions/utility/classify-text.ts
index 7bfd502cdab..714559bb1e5 100644
--- a/packages/pieces/community/ai/src/lib/actions/utility/classify-text.ts
+++ b/packages/pieces/community/ai/src/lib/actions/utility/classify-text.ts
@@ -1,8 +1,6 @@
-import { createAction, Property } from '@activepieces/pieces-framework';
-import { generateText } from 'ai';
-import { createAIModel } from '../../common/ai-sdk';
+import { createAction, Property, spreadIfDefined } from '@activepieces/pieces-framework';
+import { runOnWorker } from '../../common/ai-step';
import { aiProps, aiProviderSelection } from '../../common/props';
-import { spreadIfDefined } from '@activepieces/pieces-framework';
export const classifyText = createAction({
audience: 'both',
@@ -25,37 +23,25 @@ export const classifyText = createAction({
}),
},
async run(context) {
- const categories = (context.propsValue.categories as string[]) ?? [];
-
const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);
- const modelId = context.propsValue.model;
-
- const model = await createAIModel({
- provider,
- ...spreadIfDefined('configId', configId),
- modelId,
- engineToken: context.server.token,
- apiUrl: context.server.apiUrl,
- projectId: context.project.id,
- flowId: context.flows.current.id,
- runId: context.run.id,
- });
+ const categories = ((context.propsValue.categories as unknown[]) ?? []).map((category) => String(category));
- const response = await generateText({
- model,
- prompt: `As a text classifier, your task is to assign one of the following categories to the provided text: ${categories.join(
- ', '
- )}. Please respond with only the selected category as a single word, and nothing else.
- Text to classify: "${context.propsValue.text}"`,
+ const result = await runOnWorker({
+ context,
+ buildRequest: async () => ({
+ action: 'CLASSIFY_TEXT',
+ provider,
+ ...spreadIfDefined('providerConfigId', configId),
+ modelId: context.propsValue.model,
+ text: context.propsValue.text,
+ categories,
+ }),
});
- const result = response.text.trim();
- if (!categories.includes(result)) {
- throw new Error(
- 'Unable to classify the text into the provided categories.'
- );
+ if (result.status === 'paused') {
+ return {};
}
- return result;
+ return result.output.answer;
},
});
diff --git a/packages/pieces/community/ai/src/lib/actions/utility/extract-structured-data.ts b/packages/pieces/community/ai/src/lib/actions/utility/extract-structured-data.ts
index 36dbeff5c14..cfef9bb5269 100644
--- a/packages/pieces/community/ai/src/lib/actions/utility/extract-structured-data.ts
+++ b/packages/pieces/community/ai/src/lib/actions/utility/extract-structured-data.ts
@@ -1,10 +1,8 @@
-import { ApFile, createAction, PieceAuth, Property } from '@activepieces/pieces-framework';
-import { createAIModel } from '../../common/ai-sdk';
-import { generateText, tool, jsonSchema, ModelMessage, UserModelMessage } from 'ai';
-import mime from 'mime-types';
+import { ApFile, createAction, PieceAuth, Property, spreadIfDefined } from '@activepieces/pieces-framework';
import Ajv from 'ajv';
+import mime from 'mime-types';
+import { runOnWorker, uploadAiFiles } from '../../common/ai-step';
import { aiProps, aiProviderSelection } from '../../common/props';
-import { spreadIfDefined } from '@activepieces/pieces-framework';
export const extractStructuredData = createAction({
audience: 'both',
@@ -129,164 +127,55 @@ export const extractStructuredData = createAction({
},
async run(context) {
const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);
- const modelId = context.propsValue.model;
- const text = context.propsValue.text;
- const files = (context.propsValue.files as Array<{ file: ApFile }>) ?? [];
- const prompt = context.propsValue.prompt;
- const schema = context.propsValue.schema;
- const maxOutputTokens = context.propsValue.maxOutputTokens;
+ const attachments = ((context.propsValue.files as { file?: ApFile }[] | undefined) ?? []).map((row) => row?.file);
- if (!text && !files.length) {
+ if (!context.propsValue.text && attachments.length === 0) {
throw new Error('Please provide text or image/PDF to extract data from.');
}
- const model = await createAIModel({
- provider,
- ...spreadIfDefined('configId', configId),
- modelId,
- engineToken: context.server.token,
- apiUrl: context.server.apiUrl,
- projectId: context.project.id,
- flowId: context.flows.current.id,
- runId: context.run.id,
- });
-
- let schemaDefinition: any;
- // Track sanitized-to-original name mapping to restore output keys.
- const sanitizedNameMap: Record<string, string> = {};
-
if (context.propsValue.mode === 'advanced') {
- const ajv = new Ajv();
- const isValidSchema = ajv.validateSchema(schema['fields']);
-
- if (!isValidSchema) {
- throw new Error(
- JSON.stringify({
- message: 'Invalid JSON schema',
- errors: ajv.errors,
- }),
- );
- }
-
- schemaDefinition = jsonSchema(schema['fields'] as any);
- } else {
- const fields = schema['fields'] as Array<{
- name: string;
- description?: string;
- type: string;
- isRequired: boolean;
- }>;
-
- const properties: Record<string, any> = {};
- const required: string[] = [];
-
- fields.forEach((field) => {
- const sanitizedFieldName = field.name.replace(/[^a-zA-Z0-9_.-]/g, '_');
- sanitizedNameMap[sanitizedFieldName] = field.name;
-
- properties[sanitizedFieldName] = {
- type: field.type,
- description: field.description,
- };
-
- if (field.isRequired) {
- required.push(sanitizedFieldName);
- }
- });
-
- const jsonSchemaObject = {
- type: 'object' as const,
- properties,
- required,
- };
-
- schemaDefinition = jsonSchema(jsonSchemaObject);
- }
-
- const extractionTool = tool({
- description: 'Extract structured data from the provided content',
- inputSchema: schemaDefinition,
- execute: async (data) => {
- return data;
- },
- });
-
- const messages: Array<ModelMessage> = [];
-
- const contentParts: UserModelMessage['content'] = [];
-
- let textContent = prompt || 'Extract the following data from the provided data.';
- if (text) {
- textContent += `\n\nText to analyze:\n${text}`;
+ assertValidJsonSchema(context.propsValue.schema['fields']);
}
- contentParts.push({
- type: 'text',
- text: textContent,
+ const result = await runOnWorker({
+ context,
+ buildRequest: async () => ({
+ action: 'EXTRACT_STRUCTURED_DATA',
+ provider,
+ ...spreadIfDefined('providerConfigId', configId),
+ modelId: context.propsValue.model,
+ ...spreadIfDefined('text', context.propsValue.text),
+ ...spreadIfDefined('prompt', context.propsValue.prompt),
+ files: await uploadAiFiles({ context, files: attachments, mimeTypeOf: documentMimeType }),
+ schema: { mode: context.propsValue.mode ?? 'simple', fields: context.propsValue.schema['fields'] },
+ ...spreadIfDefined('maxOutputTokens', context.propsValue.maxOutputTokens),
+ }),
});
- if (files.length > 0) {
- for (const fileWrapper of files) {
- const file = fileWrapper.file;
- if (!file) {
- continue;
- }
- const fileType = file.extension ? mime.lookup(file.extension) : 'image/jpeg';
-
- if (fileType && fileType.startsWith('image') && file.base64) {
- contentParts.push({
- type: 'image',
- image: `data:${fileType};base64,${file.base64}`,
- });
- } else if (fileType && fileType.startsWith('application/pdf') && file.base64) {
- contentParts.push({
- type: 'file',
- data: `data:${fileType};base64,${file.base64}`,
- mediaType: fileType,
- filename: file.filename,
- });
- }
- }
+ if (result.status === 'paused') {
+ return {};
}
- messages.push({
- role: 'user',
- content: contentParts,
- });
-
- try {
- const result = await generateText({
- model,
- maxOutputTokens,
- tools: {
- extractData: extractionTool,
- },
- toolChoice: 'required',
- messages,
- });
-
- const toolCalls = result.toolCalls;
- if (!toolCalls || toolCalls.length === 0) {
- throw new Error('No structured data could be extracted from the input.');
- }
-
- const extractedData = toolCalls[0].input;
-
- if (Object.keys(sanitizedNameMap).length > 0 && extractedData && typeof extractedData === 'object') {
- const restoredData: Record<string, unknown> = {};
- for (const [key, value] of Object.entries(extractedData)) {
- const originalName = sanitizedNameMap[key] ?? key;
- restoredData[originalName] = value;
- }
- return restoredData;
- }
-
- return extractedData;
-
- } catch (error) {
- throw new Error(`Failed to extract structured data: ${error instanceof Error ? error.message : 'Unknown error'}`);
- }
+ return result.output.answer;
},
});
-
+function assertValidJsonSchema(fields: unknown): void {
+ const ajv = new Ajv();
+ if (!ajv.validateSchema(fields as Parameters<Ajv['validateSchema']>[0])) {
+ throw new Error(
+ JSON.stringify({
+ message: 'Invalid JSON schema',
+ errors: ajv.errors,
+ }),
+ );
+ }
+}
+
+function documentMimeType(file: ApFile): string | undefined {
+ if (!file.extension) {
+ return 'image/jpeg';
+ }
+ const detected = mime.lookup(file.extension);
+ return detected === false ? undefined : detected;
+}
diff --git a/packages/pieces/community/ai/src/lib/common/ai-step.ts b/packages/pieces/community/ai/src/lib/common/ai-step.ts
new file mode 100644
index 00000000000..10eaba85803
--- /dev/null
+++ b/packages/pieces/community/ai/src/lib/common/ai-step.ts
@@ -0,0 +1,131 @@
+import { AuthenticationType, httpClient, HttpMethod } from '@activepieces/pieces-common';
+import { ActionContext, ApFile, ExecutionType, InputPropertyMap, isNil, spreadIfDefined } from '@activepieces/pieces-framework';
+
+export async function runOnWorker(params: {
+ context: AiStepContext;
+ buildRequest: () => Promise<AiStepRequest>;
+}): Promise<AiStepResult> {
+ const { context } = params;
+
+ if (context.executionType === ExecutionType.RESUME) {
+ return readAnswer(context.resumePayload?.body);
+ }
+
+ const request = await params.buildRequest();
+
+ if (!context.run.canPause) {
+ const { body } = await sendToServer({ context, request });
+ return readAnswer(body);
+ }
+
+ const waitpoint = await context.run.createWaitpoint({
+ type: 'WEBHOOK',
+ resumeDateTime: new Date(Date.now() + AI_STEP_TIMEOUT_MS).toUTCString(),
+ });
+
+ await sendToServer({ context, request, waitpointId: waitpoint.id });
+
+ context.run.waitForWaitpoint(waitpoint.id);
+ return { status: 'paused' };
+}
+
+export async function uploadAiFiles(params: {
+ context: AiStepContext;
+ files: (ApFile | undefined)[];
+ mimeTypeOf: (file: ApFile) => string | undefined;
+}): Promise<AiStepFile[]> {
+ const present = params.files.filter((file): file is ApFile => !isNil(file));
+ return Promise.all(
+ present.map(async (file) => {
+ const uploaded = await params.context.files.upload({
+ fileName: file.filename,
+ data: file.data,
+ });
+ return {
+ fileId: uploaded.id,
+ ...spreadIfDefined('mimeType', params.mimeTypeOf(file)),
+ ...spreadIfDefined('filename', file.filename),
+ };
+ }),
+ );
+}
+
+async function sendToServer(params: {
+ context: AiStepContext;
+ request: AiStepRequest;
+ waitpointId?: string;
+}): Promise<{ body: unknown }> {
+ return httpClient.sendRequest({
+ method: HttpMethod.POST,
+ url: `${params.context.server.apiUrl}v1/ai/execute`,
+ authentication: { type: AuthenticationType.BEARER_TOKEN, token: params.context.server.token },
+ body: {
+ ...params.request,
+ flowId: params.context.flows.current.id,
+ flowRunId: params.context.run.id,
+ ...spreadIfDefined('waitpointId', params.waitpointId),
+ },
+ });
+}
+
+function readAnswer(body: unknown): AiStepResult {
+ if (isNil(body) || typeof body !== 'object') {
+ throw new Error(NOTHING_REPORTED);
+ }
+ const payload = body as Record<string, unknown>;
+ if (typeof payload['failure'] === 'string') {
+ throw new Error(payload['failure']);
+ }
+ const output = payload['output'];
+ if (isNil(output) || typeof output !== 'object') {
+ throw new Error(NOTHING_REPORTED);
+ }
+ return { status: 'done', output: output as AiStepOutput };
+}
+
+const AI_STEP_TIMEOUT_MS = 60 * 60 * 1000;
+const NOTHING_REPORTED = 'The AI step did not report a result before it timed out';
+
+export type AiStepContext = Pick<
+ ActionContext<undefined, InputPropertyMap>,
+ 'server' | 'flows' | 'files' | 'run' | 'executionType'
+> & { resumePayload?: { body?: unknown } };
+
+export type AiStepAction =
+ | 'ASK_AI'
+ | 'SUMMARIZE_TEXT'
+ | 'CLASSIFY_TEXT'
+ | 'EXTRACT_STRUCTURED_DATA'
+ | 'GENERATE_IMAGE';
+
+export type AiStepFile = {
+ fileId: string;
+ mimeType?: string;
+ filename?: string;
+};
+
+export type AiStepRequest = {
+ action: AiStepAction;
+ provider: string;
+ providerConfigId?: string;
+ modelId: string;
+ prompt?: string;
+ text?: string;
+ categories?: string[];
+ files?: AiStepFile[];
+ schema?: { mode: 'simple' | 'advanced'; fields: unknown };
+ advancedOptions?: Record<string, unknown>;
+ conversation?: Record<string, unknown>[];
+ maxOutputTokens?: number;
+ temperature?: number;
+ webSearch?: { enabled: boolean; options?: Record<string, unknown> };
+};
+
+export type AiStepOutput = {
+ answer: unknown;
+ conversation?: Record<string, unknown>[];
+};
+
+export type AiStepResult =
+ | { status: 'paused' }
+ | { status: 'done'; output: AiStepOutput };
diff --git a/packages/pieces/community/ai/src/lib/common/web-search.ts b/packages/pieces/community/ai/src/lib/common/web-search.ts
index b3913e53cad..015f39492c5 100644
--- a/packages/pieces/community/ai/src/lib/common/web-search.ts
+++ b/packages/pieces/community/ai/src/lib/common/web-search.ts
@@ -283,6 +283,74 @@ export function buildWebSearchConfig(params: {
};
}
+export function usesNativeWebSearchTools({ provider, model }: { provider: string; model: string | undefined }): boolean {
+ const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider, model });
+ return NATIVE_WEB_SEARCH_PROVIDERS.has(effectiveProvider ?? provider);
+}
+
+export function sanitizeWebSearchOptions(saved: unknown): Record<string, unknown> {
+ if (saved === null || typeof saved !== 'object') {
+ return {};
+ }
+ const options = saved as Record<string, unknown>;
+ return {
+ ...spreadIfDefined('maxUses', asNumber(options['maxUses'])),
+ ...spreadIfDefined('includeSources', asBoolean(options['includeSources'])),
+ ...spreadIfDefined('userLocationCity', asText(options['userLocationCity'])),
+ ...spreadIfDefined('userLocationRegion', asText(options['userLocationRegion'])),
+ ...spreadIfDefined('userLocationCountry', asText(options['userLocationCountry'])),
+ ...spreadIfDefined('userLocationTimezone', asText(options['userLocationTimezone'])),
+ ...spreadIfDefined('allowedDomains', asDomainList(options['allowedDomains'])),
+ ...spreadIfDefined('blockedDomains', asDomainList(options['blockedDomains'])),
+ ...spreadIfDefined('searchContextSize', asSearchContextSize(options['searchContextSize'])),
+ };
+}
+
+function asNumber(value: unknown): number | undefined {
+ const parsed = Number(value);
+ return typeof value === 'boolean' || value === null || value === '' || value === undefined || Number.isNaN(parsed)
+ ? undefined
+ : parsed;
+}
+
+function asBoolean(value: unknown): boolean | undefined {
+ if (typeof value === 'boolean') {
+ return value;
+ }
+ if (value === 'true' || value === 'false') {
+ return value === 'true';
+ }
+ return undefined;
+}
+
+function asText(value: unknown): string | undefined {
+ return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+function asDomainList(value: unknown): { domain: string }[] | undefined {
+ if (!Array.isArray(value)) {
+ return undefined;
+ }
+ const domains = value.flatMap((entry) => {
+ if (typeof entry === 'string') {
+ return [{ domain: entry }];
+ }
+ const domain = entry && typeof entry === 'object' ? (entry as Record<string, unknown>)['domain'] : undefined;
+ return typeof domain === 'string' ? [{ domain }] : [];
+ });
+ return domains.length === 0 ? undefined : domains;
+}
+
+function asSearchContextSize(value: unknown): 'low' | 'medium' | 'high' | undefined {
+ return value === 'low' || value === 'medium' || value === 'high' ? value : undefined;
+}
+
+const NATIVE_WEB_SEARCH_PROVIDERS: ReadonlySet<string> = new Set([
+ AIProviderName.OPENAI,
+ AIProviderName.ANTHROPIC,
+ AIProviderName.GOOGLE,
+]);
+
type BaseWebSearchOptions = {
maxUses?: number
includeSources?: boolean
diff --git a/packages/pieces/framework/package.json b/packages/pieces/framework/package.json
index 0332864200b..386e8feec59 100644
--- a/packages/pieces/framework/package.json
+++ b/packages/pieces/framework/package.json
@@ -1,6 +1,6 @@
{
"name": "@activepieces/pieces-framework",
- "version": "0.39.0",
+ "version": "0.40.0",
"type": "commonjs",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
diff --git a/packages/pieces/framework/src/lib/context/index.ts b/packages/pieces/framework/src/lib/context/index.ts
index 8d10076a26f..12770382b8c 100644
--- a/packages/pieces/framework/src/lib/context/index.ts
+++ b/packages/pieces/framework/src/lib/context/index.ts
@@ -210,6 +210,7 @@ export type WaitForWaitpointHook = (waitpointId: string) => void;
export type RunContext = {
id: FlowRunId;
+ canPause: boolean;
stop: StopHook;
/** @deprecated Use createWaitpoint + waitForWaitpoint instead */
pause?: PauseHook;
@@ -283,8 +284,20 @@ export interface FilesService {
fileName: string;
data: Buffer | Readable;
}): Promise<string>;
+ upload({
+ fileName,
+ data,
+ }: {
+ fileName: string;
+ data: Buffer | Readable;
+ }): Promise<UploadedFile>;
}
+export type UploadedFile = {
+ id: string;
+ url: string;
+};
+
export interface ConnectionsManager {
get(
key: string
diff --git a/packages/pieces/framework/src/lib/test/index.ts b/packages/pieces/framework/src/lib/test/index.ts
index 05f3ada0179..5500a041a61 100644
--- a/packages/pieces/framework/src/lib/test/index.ts
+++ b/packages/pieces/framework/src/lib/test/index.ts
@@ -1,6 +1,8 @@
-import { ExecutionType, TriggerStrategy } from '@activepieces/core-piece-types';
+import { ExecutionType, ResumePayload, TriggerStrategy } from '@activepieces/core-piece-types';
import {
ActionContext,
+ CreateWaitpointParams,
+ CreateWaitpointResult,
SetScheduleRequest,
TriggerHookContext,
} from '../context';
@@ -10,9 +12,14 @@ export function createMockActionContext<
Props extends InputPropertyMap
>(params: {
propsValue: StaticPropsValue<Props>;
+ resumePayload?: ResumePayload;
+ canPause?: boolean;
+ onCreateWaitpoint?: (waitpoint: CreateWaitpointParams) => void;
+ onWaitForWaitpoint?: (waitpointId: string) => void;
}): ActionContext<undefined, Props> {
return {
- executionType: ExecutionType.BEGIN,
+ executionType: params.resumePayload ? ExecutionType.RESUME : ExecutionType.BEGIN,
+ resumePayload: params.resumePayload,
auth: undefined,
propsValue: params.propsValue,
store: {
@@ -37,6 +44,7 @@ export function createMockActionContext<
},
files: {
write: async () => 'test-file-url',
+ upload: async () => ({ id: 'test-file-id', url: 'test-file-url' }),
},
output: {
update: async () => {
@@ -48,6 +56,7 @@ export function createMockActionContext<
},
run: {
id: 'test-run-id' as string,
+ canPause: params.canPause ?? true,
stop: () => {
return;
},
@@ -57,6 +66,17 @@ export function createMockActionContext<
respond: () => {
return;
},
+ createWaitpoint: async (waitpoint: CreateWaitpointParams): Promise<CreateWaitpointResult> => {
+ params.onCreateWaitpoint?.(waitpoint);
+ return {
+ id: 'test-waitpoint-id',
+ resumeUrl: 'http://localhost:3000/resume',
+ buildResumeUrl: () => 'http://localhost:3000/resume',
+ };
+ },
+ waitForWaitpoint: (waitpointId: string) => {
+ params.onWaitForWaitpoint?.(waitpointId);
+ },
},
project: {
id: 'test-project-id',
diff --git a/packages/server/api/src/app/ai/ai-execute-controller.ts b/packages/server/api/src/app/ai/ai-execute-controller.ts
index 559267e7a40..53620f4ee13 100644
--- a/packages/server/api/src/app/ai/ai-execute-controller.ts
+++ b/packages/server/api/src/app/ai/ai-execute-controller.ts
@@ -1,5 +1,6 @@
-import { ActivepiecesError, AIProviderName, apId, ErrorCode } from '@activepieces/core-utils'
-import { AiStepAction, AiStepFile, AiStepSchema, AiStepWebSearch, ExecuteAiJobData, LATEST_JOB_DATA_SCHEMA_VERSION, maxSocketHttpBufferSizeBytes, PrincipalType, spreadIfDefined, WorkerJobType } from '@activepieces/shared'
+import { ActivepiecesError, AIProviderName, apId, ErrorCode, isNil } from '@activepieces/core-utils'
+import { AiStepAction, AiStepFile, AiStepSchema, AiStepWebSearch, EngineResponseStatus, ExecuteAiJobData, LATEST_JOB_DATA_SCHEMA_VERSION, maxSocketHttpBufferSizeBytes, PrincipalType, WorkerJobType } from '@activepieces/shared'
+import { FastifyBaseLogger } from 'fastify'
import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'
import { StatusCodes } from 'http-status-codes'
import { z } from 'zod'
@@ -7,6 +8,7 @@ import { securityAccess } from '../core/security/authorization/fastify-security'
import { system } from '../helper/system/system'
import { AppSystemProp } from '../helper/system/system-props'
import { assertCreditsAndAppSumoNotExceeded } from '../platform/billing-provider'
+import { engineResponseWatcher } from '../workers/engine-response-watcher'
import { jobQueue, JobType } from '../workers/job-queue/job-queue'
export const aiExecuteController: FastifyPluginAsyncZod = async (app) => {
@@ -25,23 +27,36 @@ export const aiExecuteController: FastifyPluginAsyncZod = async (app) => {
const requestId = apId()
const log = request.log.child({ flowRun: { id: body.flowRunId }, requestId })
+ const answerInThisRequest = isNil(body.waitpointId)
+ const answer = answerInThisRequest ? listenForWorkerAnswer({ requestId, log }) : undefined
await jobQueue(log).add({
id: apId(),
type: JobType.ONE_TIME,
- data: aiJobFor({ body, requestId, projectId, platformId: platform.id }),
+ data: aiJobFor({
+ body,
+ requestId,
+ projectId,
+ platformId: platform.id,
+ webserverId: answerInThisRequest ? engineResponseWatcher(log).getServerId() : undefined,
+ }),
})
+ if (!isNil(answer)) {
+ return reply.status(StatusCodes.OK).send({ requestId, ...await answer })
+ }
+
log.info({ project: { id: projectId } }, '[aiExecuteController] Enqueued AI step')
return reply.status(StatusCodes.OK).send({ requestId })
})
}
-function aiJobFor({ body, requestId, projectId, platformId }: {
+function aiJobFor({ body, requestId, projectId, platformId, webserverId }: {
body: z.infer<typeof ExecuteAiRequest>
requestId: string
projectId: string
platformId: string
+ webserverId?: string
}): ExecuteAiJobData {
const shared = {
schemaVersion: LATEST_JOB_DATA_SCHEMA_VERSION,
@@ -52,52 +67,65 @@ function aiJobFor({ body, requestId, projectId, platformId }: {
flowId: body.flowId,
flowRunId: body.flowRunId,
waitpointId: body.waitpointId,
+ webserverId,
provider: body.provider,
modelId: body.modelId,
- ...spreadIfDefined('prompt', body.prompt),
- ...spreadIfDefined('providerConfigId', body.providerConfigId),
- ...spreadIfDefined('maxOutputTokens', body.maxOutputTokens),
- ...spreadIfDefined('temperature', body.temperature),
- ...spreadIfDefined('webSearch', body.webSearch),
+ prompt: body.prompt,
+ providerConfigId: body.providerConfigId,
+ maxOutputTokens: body.maxOutputTokens,
+ temperature: body.temperature,
+ webSearch: body.webSearch,
} as const
switch (body.action) {
case AiStepAction.ASK_AI:
- return { ...shared, action: AiStepAction.ASK_AI, ...spreadIfDefined('conversation', body.conversation) }
+ return { ...shared, action: AiStepAction.ASK_AI, conversation: body.conversation }
case AiStepAction.SUMMARIZE_TEXT:
- return { ...shared, action: AiStepAction.SUMMARIZE_TEXT, ...spreadIfDefined('text', body.text) }
+ return { ...shared, action: AiStepAction.SUMMARIZE_TEXT, text: body.text }
case AiStepAction.CLASSIFY_TEXT:
return {
...shared,
action: AiStepAction.CLASSIFY_TEXT,
- ...spreadIfDefined('text', body.text),
- ...spreadIfDefined('categories', body.categories),
+ text: body.text,
+ categories: body.categories,
}
case AiStepAction.EXTRACT_STRUCTURED_DATA:
return {
...shared,
action: AiStepAction.EXTRACT_STRUCTURED_DATA,
- ...spreadIfDefined('text', body.text),
- ...spreadIfDefined('files', body.files),
- ...spreadIfDefined('schema', body.schema),
+ text: body.text,
+ files: body.files,
+ schema: body.schema,
}
case AiStepAction.GENERATE_IMAGE:
return {
...shared,
action: AiStepAction.GENERATE_IMAGE,
- ...spreadIfDefined('files', body.files),
- ...spreadIfDefined('advancedOptions', body.advancedOptions),
+ files: body.files,
+ advancedOptions: body.advancedOptions,
}
}
}
+async function listenForWorkerAnswer({ requestId, log }: { requestId: string, log: FastifyBaseLogger }): Promise<{ output?: unknown, failure?: string }> {
+ const timeoutMs = system.getNumberOrThrow(AppSystemProp.FLOW_TIMEOUT_SECONDS) * 1000
+ const response = await engineResponseWatcher(log).oneTimeListener<WorkerResponse | undefined>(requestId, true, timeoutMs, undefined)
+ if (isNil(response)) {
+ return { failure: 'The AI step did not finish in time' }
+ }
+ if (response.status !== EngineResponseStatus.OK) {
+ return { failure: response.error ?? 'The AI step failed' }
+ }
+ return response.response ?? { failure: 'The AI step reported nothing back' }
+}
+
const RUN_PRINCIPALS = [PrincipalType.ENGINE] as const
const ExecuteAiRequest = z.object({
action: z.enum(AiStepAction),
flowId: z.string(),
flowRunId: z.string(),
- waitpointId: z.string(),
+ waitpointId: z.string().optional(),
provider: z.enum(AIProviderName),
providerConfigId: z.string().optional(),
modelId: z.string(),
@@ -115,6 +143,8 @@ const ExecuteAiRequest = z.object({
const ExecuteAiResponse = z.object({
requestId: z.string(),
+ output: z.unknown().optional(),
+ failure: z.string().optional(),
})
const ExecuteAiRoute = {
@@ -127,3 +157,9 @@ const ExecuteAiRoute = {
response: { [StatusCodes.OK]: ExecuteAiResponse },
},
}
+
+type WorkerResponse = {
+ status: EngineResponseStatus
+ response?: { output?: unknown, failure?: string }
+ error?: string
+}
diff --git a/packages/server/api/src/app/workers/job-queue/job-broker.ts b/packages/server/api/src/app/workers/job-queue/job-broker.ts
index 1b6b2453e1f..9352e9086bf 100644
--- a/packages/server/api/src/app/workers/job-queue/job-broker.ts
+++ b/packages/server/api/src/app/workers/job-queue/job-broker.ts
@@ -11,7 +11,7 @@ import { rateLimiterInterceptor } from './interceptors/rate-limiter-interceptor'
import { zombiePollingInterceptor } from './interceptors/zombie-polling-interceptor'
import { jobAssignmentTracker } from './job-assignment-tracker'
import { InterceptorVerdict, JobInterceptor } from './job-interceptor'
-import { isUserInteractionJobData } from './job-queue'
+import { callerWaitingForResponse } from './job-queue'
import { createQueueDispatcher, QueueDispatcher } from './queue-dispatcher'
const DRAIN_DELAY_SECONDS = 15
@@ -243,17 +243,17 @@ export const jobBroker = (log: FastifyBaseLogger) => ({
}
const jobData = JobData.parse(job.data)
- const userJobData = isUserInteractionJobData(jobData) ? jobData : null
+ const waitingCaller = callerWaitingForResponse(jobData)
const { error } = await tryCatch(async () => {
if (input.status === EngineResponseStatus.INTERNAL_ERROR) {
- if (userJobData) {
+ if (waitingCaller) {
// User-interaction jobs (piece-metadata extraction, validation, property/auth, trigger
// hooks) are synchronous request/response — the caller awaits the result with a timeout.
// Return the error to that caller and COMPLETE the job instead of moving it to failed: the
// exponential-backoff retry only fires long after the caller has timed out, so it serves no
// one and just piles up dead jobs in the failed queue.
- await engineResponseWatcher(log).publish(userJobData.webserverId, userJobData.requestId, {
+ await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {
status: EngineResponseStatus.INTERNAL_ERROR,
response: undefined,
error: input.errorMessage ?? 'Internal error',
@@ -267,8 +267,8 @@ export const jobBroker = (log: FastifyBaseLogger) => ({
}
await job.moveToCompleted({ response: input.response ?? undefined }, input.token, false)
- if (userJobData) {
- await engineResponseWatcher(log).publish(userJobData.webserverId, userJobData.requestId, {
+ if (waitingCaller) {
+ await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {
status: input.status,
response: input.response,
error: input.errorMessage,
@@ -278,8 +278,8 @@ export const jobBroker = (log: FastifyBaseLogger) => ({
})
if (error) {
log.error({ job: { id: input.jobId }, error: String(error), originalError: input.errorMessage }, '[jobBroker] Failed to move job to final state — leaving for stalled-scan recovery')
- if (userJobData) {
- await engineResponseWatcher(log).publish(userJobData.webserverId, userJobData.requestId, {
+ if (waitingCaller) {
+ await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {
status: EngineResponseStatus.INTERNAL_ERROR,
response: undefined,
error: String(error),
diff --git a/packages/server/api/src/app/workers/job-queue/job-queue.ts b/packages/server/api/src/app/workers/job-queue/job-queue.ts
index e7a4021fe31..ad442a20828 100644
--- a/packages/server/api/src/app/workers/job-queue/job-queue.ts
+++ b/packages/server/api/src/app/workers/job-queue/job-queue.ts
@@ -206,6 +206,16 @@ export function isUserInteractionJobData(jobData: JobData): jobData is UserInter
return USER_INTERACTION_JOB_TYPES.has(jobData.jobType)
}
+export function callerWaitingForResponse(jobData: JobData): { webserverId: string, requestId: string } | null {
+ if (isUserInteractionJobData(jobData)) {
+ return { webserverId: jobData.webserverId, requestId: jobData.requestId }
+ }
+ if (jobData.jobType === WorkerJobType.EXECUTE_AI && !isNil(jobData.webserverId)) {
+ return { webserverId: jobData.webserverId, requestId: jobData.requestId }
+ }
+ return null
+}
+
export const PROJECT_GROUP_ROUTABLE_JOB_TYPES = new Set<WorkerJobType>([
WorkerJobType.EXECUTE_FLOW,
WorkerJobType.EXECUTE_WEBHOOK,
diff --git a/packages/server/engine/src/lib/handler/piece-executor.ts b/packages/server/engine/src/lib/handler/piece-executor.ts
index 49070b0e406..45b03d1e0ff 100644
--- a/packages/server/engine/src/lib/handler/piece-executor.ts
+++ b/packages/server/engine/src/lib/handler/piece-executor.ts
@@ -128,6 +128,7 @@ const executeAction: ActionHandler<PieceAction> = async ({ action, executionStat
}),
run: {
id: constants.flowRunId,
+ canPause: !constants.actionRunMode,
stop: createStopHook(params),
respond: createRespondHook(params),
createWaitpoint: createWaitpointHook({ constants, stepName: action.name, hookParams: params }),
diff --git a/packages/server/engine/src/lib/piece-context/file-uploader.ts b/packages/server/engine/src/lib/piece-context/file-uploader.ts
index 0185d5e76c5..4e93c238642 100644
--- a/packages/server/engine/src/lib/piece-context/file-uploader.ts
+++ b/packages/server/engine/src/lib/piece-context/file-uploader.ts
@@ -1,28 +1,30 @@
import { Readable } from 'node:stream'
import { apId } from '@activepieces/core-utils'
-import { FilesService } from '@activepieces/pieces-framework'
+import { FilesService, UploadedFile } from '@activepieces/pieces-framework'
import { EngineGenericError, FileSizeError, FileType } from '@activepieces/shared'
import { engineFileApi } from '../api/engine-file-api'
export function createFileUploader({ engineToken, apiUrl }: CreateFileUploaderParams): FilesService {
+ const upload = async ({ fileName, data }: { fileName: string, data: Buffer | Readable }): Promise<UploadedFile> => {
+ if (!Buffer.isBuffer(data) && !(data instanceof Readable)) {
+ throw new Error(`Expected file data to be a Buffer or Readable stream, but received ${describeType(data)}`)
+ }
+ const maxBytes = resolveMaxFileSizeBytes()
+ const payload = Buffer.isBuffer(data) ? data : await drainToBuffer({ stream: data, maxBytes })
+ assertWithinLimit({ sizeInBytes: payload.length, maxBytes })
+ const { fileId, readUrl } = await engineFileApi.upload({
+ engineToken,
+ apiUrl,
+ fileId: apId(),
+ type: FileType.FLOW_STEP_FILE,
+ fileName,
+ data: payload,
+ })
+ return { id: fileId, url: readUrl }
+ }
return {
- write: async ({ fileName, data }: { fileName: string, data: Buffer | Readable }): Promise<string> => {
- if (!Buffer.isBuffer(data) && !(data instanceof Readable)) {
- throw new Error(`Expected file data to be a Buffer or Readable stream, but received ${describeType(data)}`)
- }
- const maxBytes = resolveMaxFileSizeBytes()
- const payload = Buffer.isBuffer(data) ? data : await drainToBuffer({ stream: data, maxBytes })
- assertWithinLimit({ sizeInBytes: payload.length, maxBytes })
- const { readUrl } = await engineFileApi.upload({
- engineToken,
- apiUrl,
- fileId: apId(),
- type: FileType.FLOW_STEP_FILE,
- fileName,
- data: payload,
- })
- return readUrl
- },
+ upload,
+ write: async (params: { fileName: string, data: Buffer | Readable }): Promise<string> => (await upload(params)).url,
}
}
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
index 92c9e4c7b9b..6408385cfde 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts
@@ -2,16 +2,19 @@ import { AIProviderName, isNil, spreadIfDefined, tryCatch } from '@activepieces/
import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'
import { AiStepAction, ClassifyTextJobData, EngineResponseStatus, ExecuteAiJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse, WorkerJobType } from '@activepieces/shared'
import { generateText, ModelMessage, stepCountIs } from 'ai'
-import { FireAndForgetJobResult, JobContext, JobHandler, JobResultKind } from '../../types'
+import { JobContext, JobHandler, JobResult, JobResultKind } from '../../types'
import { resolveAiFiles } from './ai-files'
import { extractStructuredData } from './extract-structured-data'
import { generateImageStep } from './generate-image'
-export const executeAiJob: JobHandler<ExecuteAiJobData, FireAndForgetJobResult> = {
+export const executeAiJob: JobHandler<ExecuteAiJobData, JobResult> = {
jobType: WorkerJobType.EXECUTE_AI,
- async execute(ctx: JobContext, data: ExecuteAiJobData): Promise<FireAndForgetJobResult> {
- const { data: output, error } = await tryCatch(() => runAiStep(ctx, data))
+ async execute(ctx: JobContext, data: ExecuteAiJobData): Promise<JobResult> {
+ const { data: output, error } = await tryCatch(() => callTheModel({ ctx, data }))
const stepOutput = isNil(error) ? { output } : { failure: toFailureMessage(error) }
+ if (isNil(data.waitpointId)) {
+ return { kind: JobResultKind.SYNCHRONOUS, status: EngineResponseStatus.OK, response: stepOutput }
+ }
const handedBack = await handBackToTheFlow({ ctx, data, waitpointId: data.waitpointId, output: stepOutput })
if (!isNil(error)) {
ctx.log.warn({ flowRun: { id: data.flowRunId }, requestId: data.requestId, error }, '[executeAiJob] Handed the failure back to the flow')
@@ -47,7 +50,7 @@ async function waitBeforeRetry(attempt: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, attempt * RETRY_DELAY_MS))
}
-async function runAiStep(ctx: JobContext, data: ExecuteAiJobData): Promise<unknown> {
+async function callTheModel({ ctx, data }: { ctx: JobContext, data: ExecuteAiJobData }): Promise<unknown> {
const resolved = await ctx.apiClient.resolveAiProvider({
projectId: data.projectId,
platformId: data.platformId,
@@ -90,6 +93,7 @@ async function runTextStep({ data, resolved, flowStep }: { data: ExecuteAiJobDat
tools,
...spreadIfDefined('maxOutputTokens', data.maxOutputTokens),
...spreadIfDefined('temperature', data.temperature),
+ ...spreadIfDefined('providerOptions', reasoningEffortFor({ action: data.action, provider })),
...spreadIfDefined('stopWhen', Object.keys(tools).length === 0 ? undefined : stepCountIs(webSearchOptions?.maxUses ?? DEFAULT_WEB_SEARCH_STEPS)),
})
@@ -126,6 +130,13 @@ function classificationPrompt(data: ClassifyTextJobData): string {
Text to classify: "${data.text ?? ''}"`
}
+function reasoningEffortFor({ action, provider }: { action: AiStepAction, provider: AIProviderName }): Record<string, Record<string, string>> | undefined {
+ if (action !== AiStepAction.SUMMARIZE_TEXT || provider !== AIProviderName.OPENAI) {
+ return undefined
+ }
+ return { [AIProviderName.OPENAI]: { reasoning_effort: 'minimal' } }
+}
+
function toStepOutput({ data, text, sources }: { data: ExecuteAiJobData, text: string, sources: unknown }): unknown {
switch (data.action) {
case AiStepAction.ASK_AI: {
diff --git a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
index 2557b4ee6f1..99a6e6f0b17 100644
--- a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
+++ b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts
@@ -10,9 +10,6 @@ export async function extractStructuredData({ data, resolved, flowStep, files }:
flowStep: FlowStepMetadata
files: ResolvedAiFile[]
}): Promise<unknown> {
- if (isNil(data.text) && files.length === 0) {
- throw new Error('Please provide text or image/PDF to extract data from.')
- }
const model = aiUtils.createModel({
provider: resolved.provider,
auth: resolved.auth,
@@ -67,11 +64,11 @@ function buildSchema(data: ExtractStructuredDataJobData): { schemaDefinition: Re
}
function buildMessages({ data, files }: { data: ExtractStructuredDataJobData, files: ResolvedAiFile[] }): ModelMessage[] {
- let textContent = data.prompt ?? 'Extract the following data from the provided data.'
- if (data.text) {
- textContent += `\n\nText to analyze:\n${data.text}`
- }
- const contentParts: UserModelMessage['content'] = [{ type: 'text', text: textContent }]
+ const prompt = data.prompt ?? ''
+ const guide = prompt.length === 0 ? 'Extract the following data from the provided data.' : prompt
+ const text = data.text ?? ''
+ const textContent = text.length === 0 ? guide : `${guide}\n\nText to analyze:\n${text}`
+ const contentParts: Exclude<UserModelMessage['content'], string> = [{ type: 'text', text: textContent }]
for (const file of files) {
if (file.base64.length === 0) {
continue