AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
feat(ai): the five simple AI steps run on the worker, not in the sandbox (#15493)
Abdulcc09b97f8679
diff --git a/packages/core/execution/src/lib/workers/job-data.ts b/packages/core/execution/src/lib/workers/job-data.tsindex 57f97da272a..f8fa01b0a25 100644--- a/packages/core/execution/src/lib/workers/job-data.ts+++ b/packages/core/execution/src/lib/workers/job-data.ts@@ -418,7 +418,8 @@ const AiStepJobBase = z.object({platformId: z.string(),flowId: z.string(),flowRunId: z.string(),- waitpointId: z.string(),+ waitpointId: z.string().optional(),+ webserverId: z.string().optional(),provider: z.enum(AIProviderName),providerConfigId: z.string().optional(),modelId: z.string(),diff --git a/packages/pieces/community/ai/package.json b/packages/pieces/community/ai/package.jsonindex dcf8940898c..03281a67820 100644--- a/packages/pieces/community/ai/package.json+++ b/packages/pieces/community/ai/package.json@@ -1,6 +1,6 @@{"name": "@activepieces/piece-ai",- "version": "0.10.3",+ "version": "0.11.0","type": "commonjs","main": "./dist/src/index.js","types": "./dist/src/index.d.ts",diff --git a/packages/pieces/community/ai/src/lib/actions/image/generate-image.ts b/packages/pieces/community/ai/src/lib/actions/image/generate-image.tsindex 4a194288f9a..1fdb5f3e25c 100644--- a/packages/pieces/community/ai/src/lib/actions/image/generate-image.ts+++ b/packages/pieces/community/ai/src/lib/actions/image/generate-image.ts@@ -1,25 +1,17 @@import {+ AIProviderName,ApFile,createAction,DynamicPropsValue,+ getEffectiveProviderAndModel,InputPropertyMap,+ isNil,PieceAuth,Property,+ spreadIfDefined,} from '@activepieces/pieces-framework';-import {- GeneratedFile,- generateText,- GenerateTextResult,- ImagePart,- LanguageModel,- ToolSet,-} from 'ai';-import { generateImage } from 'ai';import mime from 'mime-types';-import { isNil } from '@activepieces/pieces-framework';-import { getEffectiveProviderAndModel, spreadIfDefined } from '@activepieces/pieces-framework';-import { createAIModel } from '../../common/ai-sdk';-import { AIProviderName } from '@activepieces/pieces-framework';+import { runOnWorker, uploadAiFiles } from '../../common/ai-step';import { aiProps, aiProviderSelection } from '../../common/props';export const generateImageAction = createAction({@@ -142,39 +134,46 @@ export const generateImageAction = createAction({},async run(context) {const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);- const modelId = context.propsValue.model;const inputImages = collectInputImages({inputImages: context.propsValue.inputImages,advancedOptions: context.propsValue.advancedOptions,});- const image = await getGeneratedImage({- provider,- ...spreadIfDefined('configId', configId),- modelId,- engineToken: context.server.token,- apiUrl: context.server.apiUrl,- prompt: context.propsValue.prompt,- inputImages,- projectId: context.project.id,- flowId: context.flows.current.id,- runId: context.run.id,- advancedOptions: context.propsValue.advancedOptions,+ const result = await runOnWorker({+ context,+ buildRequest: async () => ({+ action: 'GENERATE_IMAGE',+ provider,+ ...spreadIfDefined('providerConfigId', configId),+ modelId: context.propsValue.model,+ prompt: context.propsValue.prompt,+ files: await uploadAiFiles({ context, files: inputImages, mimeTypeOf: inputImageMimeType }),+ ...spreadIfDefined('advancedOptions', withoutInputImages(context.propsValue.advancedOptions)),+ }),});- const imageData =- image.base64 && image.base64.length > 0- ? Buffer.from(image.base64, 'base64')- : Buffer.from(image.uint8Array);+ if (result.status === 'paused') {+ return {};+ }- return context.files.write({- data: imageData,- fileName: 'image.png',- });+ return result.output.answer;},});+const withoutInputImages = (advancedOptions: DynamicPropsValue | undefined): Record<string, unknown> | undefined => {+ if (isNil(advancedOptions)) {+ return undefined;+ }+ const { image: _inputImages, ...rest } = advancedOptions as Record<string, unknown>;+ return rest;+};++const inputImageMimeType = (file: ApFile): string => {+ const detected = file.extension ? mime.lookup(file.extension) : false;+ return detected && ALLOWED_IMAGE_MIME_TYPES.has(detected) ? detected : 'image/jpeg';+};+const collectInputImages = ({inputImages,advancedOptions,@@ -206,150 +205,6 @@ const extractImageFiles = (value: unknown): ApFile[] => {});};-const getGeneratedImage = async ({- provider,- configId,- modelId,- engineToken,- apiUrl,- prompt,- inputImages,- projectId,- flowId,- runId,- advancedOptions,-}: {- provider: AIProviderName;- configId?: string;- modelId: string;- engineToken: string;- apiUrl: string;- prompt: string;- inputImages: ApFile[];- projectId: string;- flowId: string;- runId: string;- advancedOptions?: DynamicPropsValue;-}): Promise<GeneratedFile> => {- const model = await createAIModel({- provider,- ...spreadIfDefined('configId', configId),- modelId,- engineToken,- apiUrl,- projectId,- flowId,- runId,- isImage: true,- });-- const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider, model: modelId });- const resolvedProvider = (effectiveProvider ?? provider) as AIProviderName;-- const hasInputImages = inputImages.length > 0;-- return withImageInputErrorContext({ modelId, hasInputImages }, async () => {- switch (resolvedProvider) {- case AIProviderName.GOOGLE:- case AIProviderName.ACTIVEPIECES:- case AIProviderName.OPENROUTER:- case AIProviderName.CLOUDFLARE_GATEWAY:- return generateImageUsingGenerateText({- model: model as unknown as LanguageModel,- prompt,- inputImages,- });- default: {- const sanitizedAdvancedOptions = stripLegacyImageField(advancedOptions);- const sdkImages = inputImages.map((file) =>- Buffer.from(file.base64, 'base64'),- );- const { image } = await generateImage({- model,- prompt: hasInputImages- ? { text: prompt, images: sdkImages }- : prompt,- providerOptions: {- [resolvedProvider]: { ...sanitizedAdvancedOptions },- },- });- return image;- }- }- });-};--const withImageInputErrorContext = async <T>(- { modelId, hasInputImages }: { modelId: string; hasInputImages: boolean },- run: () => Promise<T>,-): Promise<T> => {- try {- return await run();- } catch (error) {- if (!hasInputImages) {- throw error;- }- const original = error instanceof Error ? error.message : String(error);- throw new Error(- `Image generation failed for model "${modelId}". ` +- `This model may not support input images. Try a model that supports image editing — ` +- `for example gpt-image-1, dall-e-2, or a Gemini Nano Banana model — or remove the input images. ` +- `Original error: ${original}`,- );- }-};--const generateImageUsingGenerateText = async ({- model,- prompt,- inputImages,-}: {- model: LanguageModel;- prompt: string;- inputImages: ApFile[];-}): Promise<GeneratedFile> => {- const imageFiles = inputImages.map<ImagePart>((file) => {- const detected = file.extension ? mime.lookup(file.extension) : false;- const fileType =- detected && ALLOWED_IMAGE_MIME_TYPES.has(detected)- ? detected- : 'image/jpeg';-- return {- type: 'image',- image: `data:${fileType};base64,${file.base64}`,- };- });-- const result = await generateText({- model,- providerOptions: {- google: { responseModalities: ['TEXT', 'IMAGE'] },- openrouter: { modalities: ['image', 'text'] },- },- messages: [- {- role: 'user',- content: [{ type: 'text', text: prompt }, ...imageFiles],- },- ],- });-- assertImageGenerationSuccess(result);-- return result.files[0];-};--const stripLegacyImageField = (- advancedOptions: DynamicPropsValue | undefined,-): DynamicPropsValue | undefined => {- if (isNil(advancedOptions)) {- return advancedOptions;- }- const { image: _legacy, ...rest } = advancedOptions as Record<string, unknown>;- return rest as DynamicPropsValue;-};-const ALLOWED_IMAGE_MIME_TYPES: ReadonlySet<string> = new Set(['image/jpeg','image/png',@@ -358,34 +213,6 @@ const ALLOWED_IMAGE_MIME_TYPES: ReadonlySet<string> = new Set(['image/avif',]);-const assertImageGenerationSuccess = (- result: GenerateTextResult<ToolSet, never>-): void => {- const responseBody =- result.response.body &&- typeof result.response.body === 'object' &&- 'candidates' in result.response.body- ? result.response.body- : { candidates: [] };-- const responseCandidates = Array.isArray(responseBody?.candidates)- ? responseBody?.candidates- : [];-- responseCandidates.forEach((candidate: { finishReason: string }) => {- if (candidate.finishReason !== 'STOP') {- throw new Error(- 'Image generation failed Reason:\n ' +- JSON.stringify(responseCandidates, null, 2)- );- }- });-- if (isNil(result.files) || result.files.length === 0) {- throw new Error('No image generated');- }-};-function isGptImageModel({ modelId }: { modelId: string }): boolean {return modelId.startsWith('gpt-image');}diff --git a/packages/pieces/community/ai/src/lib/actions/output-shape.test.ts b/packages/pieces/community/ai/src/lib/actions/output-shape.test.tsnew file mode 100644index 00000000000..ea8e9d09126--- /dev/null+++ b/packages/pieces/community/ai/src/lib/actions/output-shape.test.ts@@ -0,0 +1,326 @@+import { httpClient } from '@activepieces/pieces-common';+import { createMockActionContext } from '@activepieces/pieces-framework';+import { beforeEach, describe, expect, it, vi } from 'vitest';+import { generateImageAction } from './image/generate-image';+import { askAI } from './text/ask-ai';+import { summarizeText } from './text/summarize-text';+import { classifyText } from './utility/classify-text';+import { extractStructuredData } from './utility/extract-structured-data';++vi.mock('@activepieces/pieces-common', async (importOriginal) => {+ const original = await importOriginal<Record<string, unknown>>();+ return {+ ...original,+ httpClient: { sendRequest: vi.fn(async () => ({ body: { requestId: 'request-1' } })) },+ };+});++const sendRequest = vi.mocked(httpClient.sendRequest);++const provider = { provider: 'openai', configId: 'config-1' };++function answered(answer: unknown, conversation?: unknown) {+ return { body: { output: { answer, conversation } } };+}++function requestBody(callIndex = 0): Record<string, unknown> {+ return sendRequest.mock.calls[callIndex][0].body as Record<string, unknown>;+}++beforeEach(() => {+ sendRequest.mockClear();+ sendRequest.mockResolvedValue({ body: { requestId: 'request-1' } } as never);+});++describe('the shape every AI action returns once the worker answers', () => {+ it('Ask AI returns the answer as a bare string', async () => {+ const output = await askAI.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },+ resumePayload: answered('sunny'),+ }));++ expect(output).toBe('sunny');+ });++ it('Ask AI returns text and sources together when web search asks for sources', async () => {+ const output = await askAI.run(createMockActionContext({+ propsValue: {+ provider,+ model: 'gpt-test',+ prompt: 'weather?',+ webSearch: true,+ webSearchOptions: { includeSources: true },+ maxOutputTokens: 2000,+ },+ resumePayload: answered({ text: 'sunny', sources: [{ id: 'source-1' }] }),+ }));++ expect(output).toEqual({ text: 'sunny', sources: [{ id: 'source-1' }] });+ });++ it('Summarize Text returns the summary as a bare string', async () => {+ const output = await summarizeText.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', text: 'a long article', prompt: 'Summarize', maxOutputTokens: 2000 },+ resumePayload: answered('the short version'),+ }));++ expect(output).toBe('the short version');+ });++ it('Classify Text returns the chosen label as a bare string', async () => {+ const output = await classifyText.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', text: 'it is warm today', categories: ['sunny', 'rainy'] },+ resumePayload: answered('sunny'),+ }));++ expect(output).toBe('sunny');+ });++ it('Extract Structured Data returns the extracted object itself', async () => {+ const output = await extractStructuredData.run(createMockActionContext({+ propsValue: {+ provider,+ model: 'gpt-test',+ text: 'an invoice totalling 42',+ mode: 'simple',+ schema: { fields: [{ name: 'total', type: 'number', isRequired: true }] },+ maxOutputTokens: 2000,+ },+ resumePayload: answered({ total: 42 }),+ }));++ expect(output).toEqual({ total: 42 });+ });++ it('Generate Image returns the stored file url as a bare string', async () => {+ const output = await generateImageAction.run(createMockActionContext({+ propsValue: { provider, model: 'dall-e-3', prompt: 'a cat', advancedOptions: {} },+ resumePayload: answered('test-file-url'),+ }));++ expect(output).toBe('test-file-url');+ });+});++describe('handing the step to the worker', () => {+ it('pauses the flow on the waitpoint it just created', async () => {+ const waitedOn: string[] = [];++ const output = await askAI.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },+ onWaitForWaitpoint: (waitpointId) => waitedOn.push(waitpointId),+ }));++ expect(output).toEqual({});+ expect(waitedOn).toEqual(['test-waitpoint-id']);+ expect(requestBody()['waitpointId']).toBe('test-waitpoint-id');+ });++ it('gives the waitpoint a backstop so a dead worker cannot hang the run forever', async () => {+ const created: { resumeDateTime?: string }[] = [];++ await askAI.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },+ onCreateWaitpoint: (waitpoint) => created.push(waitpoint),+ }));++ expect(created).toHaveLength(1);+ expect(new Date(created[0].resumeDateTime as string).getTime()).toBeGreaterThan(Date.now());+ });++ it('asks for the answer in the same request when the step cannot pause', async () => {+ sendRequest.mockResolvedValue(answered('sunny') as never);++ const output = await askAI.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },+ canPause: false,+ }));++ expect(output).toBe('sunny');+ expect(requestBody()['waitpointId']).toBeUndefined();+ });++ it('raises the failure the worker reported', async () => {+ await expect(askAI.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },+ resumePayload: { body: { failure: 'the model refused' } },+ }))).rejects.toThrow('the model refused');+ });++ it('raises a clear error when the backstop fires with nothing to report', async () => {+ await expect(askAI.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', webSearch: false, maxOutputTokens: 2000 },+ resumePayload: { body: undefined },+ }))).rejects.toThrow('did not report a result');+ });+});++describe('what a saved flow sends over the wire', () => {+ it('turns categories saved as numbers into strings', async () => {+ await classifyText.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', text: 'two', categories: [1, 2] },+ }));++ expect(requestBody()['categories']).toEqual(['1', '2']);+ });++ it('skips an empty file row instead of sending it', async () => {+ await extractStructuredData.run(createMockActionContext({+ propsValue: {+ provider,+ model: 'gpt-test',+ text: 'an invoice',+ files: [{ file: undefined }],+ mode: 'simple',+ schema: { fields: [] },+ },+ }));++ expect(requestBody()['files']).toEqual([]);+ });++ it('reads an Ask AI history stored in the legacy messages wrapper', async () => {+ const context = createMockActionContext({+ propsValue: { provider, model: 'gpt-test', prompt: 'and then?', conversationKey: 'thread-1', webSearch: false },+ });+ context.store.get = (async () => ({ messages: [{ role: 'user', content: 'hello' }] })) as typeof context.store.get;++ await askAI.run(context);++ expect(requestBody()['conversation']).toEqual([{ role: 'user', content: 'hello' }]);+ });++ it('still sends an input image pinned under the legacy advanced option', async () => {+ await generateImageAction.run(createMockActionContext({+ propsValue: {+ provider,+ model: 'gemini-2.5-flash-image',+ prompt: 'a cat',+ advancedOptions: { image: [{ file: { filename: 'cat.png', data: Buffer.from('png'), extension: 'png' } }] },+ },+ }));++ expect(requestBody()['files']).toEqual([{ fileId: 'test-file-id', mimeType: 'image/png', filename: 'cat.png' }]);+ expect(requestBody()['advancedOptions']).toEqual({});+ });++ it('turns a web search limit saved as text into the number the schema accepts', async () => {+ await askAI.run(createMockActionContext({+ propsValue: {+ provider,+ model: 'gpt-test',+ prompt: 'weather?',+ webSearch: true,+ webSearchOptions: { maxUses: '7', allowedDomains: ['example.com'] },+ },+ }));++ const webSearch = requestBody()['webSearch'] as { options: Record<string, unknown> };+ expect(webSearch.options['maxUses']).toBe(7);+ expect(webSearch.options['allowedDomains']).toEqual([{ domain: 'example.com' }]);+ });++ it('drops a stale Include Sources left behind by a provider that has no native search', async () => {+ await askAI.run(createMockActionContext({+ propsValue: {+ provider: { provider: 'openrouter', configId: 'config-1' },+ model: 'some/model',+ prompt: 'weather?',+ webSearch: true,+ webSearchOptions: { includeSources: true },+ },+ }));++ const webSearch = requestBody()['webSearch'] as { options: Record<string, unknown> };+ expect(webSearch.options).not.toHaveProperty('includeSources');+ });++ it('still runs an Extract whose only file row is empty, as it did before', async () => {+ await extractStructuredData.run(createMockActionContext({+ propsValue: {+ provider,+ model: 'gpt-test',+ files: [{ file: undefined }],+ mode: 'simple',+ schema: { fields: [] },+ },+ }));++ expect(requestBody()['files']).toEqual([]);+ });++ it('sends the creativity as a temperature the model understands', async () => {+ await askAI.run(createMockActionContext({+ propsValue: { provider, model: 'gpt-test', prompt: 'weather?', creativity: 50, webSearch: false },+ }));++ expect(requestBody()['temperature']).toBe(0.5);+ });+});++describe('what a resumed step does before it reads its answer', () => {+ function fileProp() {+ return { file: { filename: 'invoice.pdf', extension: 'pdf', base64: 'eA==', data: Buffer.from('x') } };+ }++ function countingUploads<T extends { files: { upload: unknown } }>(context: T): { context: T, uploads: () => number } {+ let uploads = 0;+ context.files.upload = async () => {+ uploads += 1;+ return { id: 'test-file-id', url: 'test-file-url' };+ };+ return { context, uploads: () => uploads };+ }++ it('Extract Structured Data does not upload its attachments a second time', async () => {+ const { context, uploads } = countingUploads(createMockActionContext({+ propsValue: {+ provider,+ model: 'gpt-test',+ text: 'an invoice totalling 42',+ files: [fileProp()],+ mode: 'simple',+ schema: { fields: [{ name: 'total', type: 'number', isRequired: true }] },+ maxOutputTokens: 2000,+ },+ resumePayload: answered({ total: 42 }),+ }));++ const output = await extractStructuredData.run(context);++ expect(output).toEqual({ total: 42 });+ expect(uploads()).toBe(0);+ expect(sendRequest).not.toHaveBeenCalled();+ });++ it('Generate Image does not upload its input images a second time', async () => {+ const { context, uploads } = countingUploads(createMockActionContext({+ propsValue: { provider, model: 'dall-e-3', prompt: 'a cat', inputImages: [fileProp()], advancedOptions: {} },+ resumePayload: answered('test-file-url'),+ }));++ const output = await generateImageAction.run(context);++ expect(output).toBe('test-file-url');+ expect(uploads()).toBe(0);+ expect(sendRequest).not.toHaveBeenCalled();+ });++ it('still uploads on the first run, so the worker has something to read', async () => {+ const { context, uploads } = countingUploads(createMockActionContext({+ propsValue: {+ provider,+ model: 'gpt-test',+ text: 'an invoice totalling 42',+ files: [fileProp()],+ mode: 'simple',+ schema: { fields: [{ name: 'total', type: 'number', isRequired: true }] },+ maxOutputTokens: 2000,+ },+ }));++ await extractStructuredData.run(context);++ expect(uploads()).toBe(1);+ });+});diff --git a/packages/pieces/community/ai/src/lib/actions/text/ask-ai.ts b/packages/pieces/community/ai/src/lib/actions/text/ask-ai.tsindex 03de83ac509..773ef92e11e 100644--- a/packages/pieces/community/ai/src/lib/actions/text/ask-ai.ts+++ b/packages/pieces/community/ai/src/lib/actions/text/ask-ai.ts@@ -1,8 +1,7 @@-import { ModelMessage, generateText, stepCountIs } from 'ai';-import { AIProviderName, createAction, getEffectiveProviderAndModel, isNil, Property, spreadIfDefined } from '@activepieces/pieces-framework';+import { createAction, isNil, Property, spreadIfDefined } from '@activepieces/pieces-framework';+import { runOnWorker } from '../../common/ai-step';import { aiProps, aiProviderSelection } from '../../common/props';-import { createAIModel } from '../../common/ai-sdk';-import { buildWebSearchOptionsProperty, buildWebSearchConfig, WebSearchOptions } from '../../common/web-search';+import { buildWebSearchOptionsProperty, sanitizeWebSearchOptions, usesNativeWebSearchTools } from '../../common/web-search';export const askAI = createAction({audience: 'both',@@ -50,84 +49,67 @@ export const askAI = createAction({},async run(context) {const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);- const modelId = context.propsValue.model;- const storage = context.store;const webSearchEnabled = !!context.propsValue.webSearch;- const webSearchOptions = (context.propsValue.webSearchOptions ?? {}) as WebSearchOptions;-- const { tools: webSearchTools, providerOptions } = buildWebSearchConfig({+ const webSearchOptions = webSearchOptionsToSend({provider,- model: modelId,- webSearchEnabled,- webSearchOptions,+ model: context.propsValue.model,+ saved: context.propsValue.webSearchOptions,});+ const storageKey = conversationStorageKey(context.propsValue.conversationKey);- const { provider: effectiveProvider } = getEffectiveProviderAndModel({- provider,- model: modelId,- });- const model = await createAIModel({- provider,- ...spreadIfDefined('configId', configId),- modelId,- engineToken: context.server.token,- apiUrl: context.server.apiUrl,- projectId: context.project.id,- flowId: context.flows.current.id,- runId: context.run.id,- ...spreadIfDefined('openaiResponsesModel', webSearchEnabled && effectiveProvider === AIProviderName.OPENAI ? true : undefined),+ const result = await runOnWorker({+ context,+ buildRequest: async () => ({+ action: 'ASK_AI',+ provider,+ ...spreadIfDefined('providerConfigId', configId),+ modelId: context.propsValue.model,+ prompt: context.propsValue.prompt,+ ...spreadIfDefined('maxOutputTokens', context.propsValue.maxOutputTokens),+ ...spreadIfDefined('temperature', isNil(context.propsValue.creativity) ? undefined : context.propsValue.creativity / 100),+ ...spreadIfDefined('conversation', isNil(storageKey) ? undefined : await readConversation(context.store, storageKey)),+ webSearch: { enabled: webSearchEnabled, options: webSearchOptions },+ }),});- const conversationKey = context.propsValue.conversationKey- ? `ask-ai-conversation:${context.propsValue.conversationKey}`- : null;+ if (result.status === 'paused') {+ return {};+ }- let conversation = null;- if (conversationKey) {- conversation = (await storage.get<ModelMessage[]>(conversationKey)) ?? [];- if (!conversation) {- await storage.put(conversationKey, { messages: [] });- }+ if (!isNil(storageKey) && !isNil(result.output.conversation)) {+ await context.store.put(storageKey, result.output.conversation);}- const stopWhen = webSearchTools- ? stepCountIs(webSearchOptions?.maxUses ?? 5)- : undefined;+ return result.output.answer;+ },+});- const response = await generateText({- model,- messages: [- ...(conversation ?? []),- {- role: 'user',- content: context.propsValue.prompt,- },- ],- maxOutputTokens: context.propsValue.maxOutputTokens,- ...spreadIfDefined('temperature', isNil(context.propsValue.creativity) ? undefined : context.propsValue.creativity / 100),- tools: webSearchTools,- stopWhen,- providerOptions,- });+function webSearchOptionsToSend({ provider, model, saved }: { provider: string; model: string; saved: unknown }): Record<string, unknown> {+ const options = sanitizeWebSearchOptions(saved);+ if (usesNativeWebSearchTools({ provider, model })) {+ return options;+ }+ const { includeSources: _onlyWithNativeTools, ...rest } = options;+ return rest;+}- conversation?.push({- role: 'user',- content: context.propsValue.prompt,- });+function conversationStorageKey(conversationKey: string | undefined): string | null {+ return conversationKey ? `ask-ai-conversation:${conversationKey}` : null;+}- conversation?.push({- role: 'assistant',- content: response.text ?? '',- });+async function readConversation(store: ConversationStore, storageKey: string): Promise<Record<string, unknown>[]> {+ const stored = await store.get<StoredConversation>(storageKey);+ if (Array.isArray(stored)) {+ return stored;+ }+ if (!isNil(stored) && Array.isArray(stored.messages)) {+ return stored.messages;+ }+ return [];+}- if (conversationKey) {- await storage.put(conversationKey, conversation);- }+type StoredConversation = Record<string, unknown>[] | { messages?: Record<string, unknown>[] };- const includeSources = webSearchTools && webSearchOptions.includeSources;- if (includeSources) {- return { text: response.text, sources: response.sources };- }- return response.text;- },-});+type ConversationStore = {+ get<T>(key: string): Promise<T | null>;+};diff --git a/packages/pieces/community/ai/src/lib/actions/text/summarize-text.ts b/packages/pieces/community/ai/src/lib/actions/text/summarize-text.tsindex e60eb26c65e..b85d4b48f89 100644--- a/packages/pieces/community/ai/src/lib/actions/text/summarize-text.ts+++ b/packages/pieces/community/ai/src/lib/actions/text/summarize-text.ts@@ -1,7 +1,5 @@-import { AIProviderName, spreadIfDefined } from '@activepieces/pieces-framework';-import { createAIModel } from '../../common/ai-sdk';-import { createAction, Property } from '@activepieces/pieces-framework';-import { generateText } from 'ai';+import { createAction, Property, spreadIfDefined } from '@activepieces/pieces-framework';+import { runOnWorker } from '../../common/ai-step';import { aiProps, aiProviderSelection } from '../../common/props';export const summarizeText = createAction({@@ -32,35 +30,24 @@ export const summarizeText = createAction({},async run(context) {const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);- const modelId = context.propsValue.model;- const model = await createAIModel({- provider,- ...spreadIfDefined('configId', configId),- modelId,- engineToken: context.server.token,- apiUrl: context.server.apiUrl,- projectId: context.project.id,- flowId: context.flows.current.id,- runId: context.run.id,+ const result = await runOnWorker({+ context,+ buildRequest: async () => ({+ action: 'SUMMARIZE_TEXT',+ provider,+ ...spreadIfDefined('providerConfigId', configId),+ modelId: context.propsValue.model,+ prompt: context.propsValue.prompt,+ text: context.propsValue.text,+ ...spreadIfDefined('maxOutputTokens', context.propsValue.maxOutputTokens),+ }),});- const response = await generateText({- model,- messages: [- {- role: 'user',- content: `${context.propsValue.prompt} Summarize the following text : ${context.propsValue.text}`- },- ],- maxOutputTokens: context.propsValue.maxOutputTokens,- providerOptions: {- [provider]: {- ...(provider === AIProviderName.OPENAI ? { reasoning_effort: 'minimal' } : {}),- }- }- });+ if (result.status === 'paused') {+ return {};+ }- return response.text ?? '';+ return result.output.answer;},});diff --git a/packages/pieces/community/ai/src/lib/actions/text/temperature.test.ts b/packages/pieces/community/ai/src/lib/actions/text/temperature.test.tsindex 7bd72926d73..946480e50b4 100644--- a/packages/pieces/community/ai/src/lib/actions/text/temperature.test.ts+++ b/packages/pieces/community/ai/src/lib/actions/text/temperature.test.ts@@ -1,19 +1,18 @@-import { beforeEach, describe, expect, it, vi } from 'vitest';+import { httpClient } from '@activepieces/pieces-common';import { createMockActionContext } from '@activepieces/pieces-framework';-import { generateText } from 'ai';+import { beforeEach, describe, expect, it, vi } from 'vitest';import { askAI } from './ask-ai';import { summarizeText } from './summarize-text';-vi.mock('ai', () => ({- generateText: vi.fn(async () => ({ text: 'ok', sources: [] })),- stepCountIs: vi.fn(),-}));--vi.mock('../../common/ai-sdk', () => ({- createAIModel: vi.fn(async () => ({})),-}));+vi.mock('@activepieces/pieces-common', async (importOriginal) => {+ const original = await importOriginal<Record<string, unknown>>();+ return {+ ...original,+ httpClient: { sendRequest: vi.fn(async () => ({ body: { requestId: 'request-1' } })) },+ };+});-const generateTextMock = vi.mocked(generateText);+const sendRequest = vi.mocked(httpClient.sendRequest);const baseProps = {provider: { provider: 'openai', configId: 'config1' },@@ -21,41 +20,41 @@ const baseProps = {maxOutputTokens: 2000,};-async function askAiGenerateTextArgs({ creativity }: { creativity: number | null | undefined }) {+async function askAiRequestBody({ creativity }: { creativity: number | null | undefined }) {await askAI.run(createMockActionContext({propsValue: { ...baseProps, prompt: 'hello', webSearch: false, creativity },}));- return generateTextMock.mock.calls[0][0];+ return sendRequest.mock.calls[0][0].body as Record<string, unknown>;}beforeEach(() => {- generateTextMock.mockClear();+ sendRequest.mockClear();});describe('askAI temperature', () => {it('omits temperature when creativity is not set', async () => {- const args = await askAiGenerateTextArgs({ creativity: undefined });- expect(args).not.toHaveProperty('temperature');+ const body = await askAiRequestBody({ creativity: undefined });+ expect(body).not.toHaveProperty('temperature');});it('omits temperature when creativity is null', async () => {- const args = await askAiGenerateTextArgs({ creativity: null });- expect(args).not.toHaveProperty('temperature');+ const body = await askAiRequestBody({ creativity: null });+ expect(body).not.toHaveProperty('temperature');});it('sends temperature scaled from an explicit creativity', async () => {- const args = await askAiGenerateTextArgs({ creativity: 50 });- expect(args.temperature).toBe(0.5);+ const body = await askAiRequestBody({ creativity: 50 });+ expect(body['temperature']).toBe(0.5);});it('sends temperature 1 for the previously seeded default of 100', async () => {- const args = await askAiGenerateTextArgs({ creativity: 100 });- expect(args.temperature).toBe(1);+ const body = await askAiRequestBody({ creativity: 100 });+ expect(body['temperature']).toBe(1);});it('sends temperature 0 when creativity is 0', async () => {- const args = await askAiGenerateTextArgs({ creativity: 0 });- expect(args.temperature).toBe(0);+ const body = await askAiRequestBody({ creativity: 0 });+ expect(body['temperature']).toBe(0);});});@@ -64,6 +63,6 @@ describe('summarizeText temperature', () => {await summarizeText.run(createMockActionContext({propsValue: { ...baseProps, text: 'long text', prompt: 'Summarize' },}));- expect(generateTextMock.mock.calls[0][0]).not.toHaveProperty('temperature');+ expect(sendRequest.mock.calls[0][0].body).not.toHaveProperty('temperature');});});diff --git a/packages/pieces/community/ai/src/lib/actions/utility/classify-text.ts b/packages/pieces/community/ai/src/lib/actions/utility/classify-text.tsindex 7bfd502cdab..714559bb1e5 100644--- a/packages/pieces/community/ai/src/lib/actions/utility/classify-text.ts+++ b/packages/pieces/community/ai/src/lib/actions/utility/classify-text.ts@@ -1,8 +1,6 @@-import { createAction, Property } from '@activepieces/pieces-framework';-import { generateText } from 'ai';-import { createAIModel } from '../../common/ai-sdk';+import { createAction, Property, spreadIfDefined } from '@activepieces/pieces-framework';+import { runOnWorker } from '../../common/ai-step';import { aiProps, aiProviderSelection } from '../../common/props';-import { spreadIfDefined } from '@activepieces/pieces-framework';export const classifyText = createAction({audience: 'both',@@ -25,37 +23,25 @@ export const classifyText = createAction({}),},async run(context) {- const categories = (context.propsValue.categories as string[]) ?? [];-const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);- const modelId = context.propsValue.model;-- const model = await createAIModel({- provider,- ...spreadIfDefined('configId', configId),- modelId,- engineToken: context.server.token,- apiUrl: context.server.apiUrl,- projectId: context.project.id,- flowId: context.flows.current.id,- runId: context.run.id,- });+ const categories = ((context.propsValue.categories as unknown[]) ?? []).map((category) => String(category));- const response = await generateText({- model,- prompt: `As a text classifier, your task is to assign one of the following categories to the provided text: ${categories.join(- ', '- )}. Please respond with only the selected category as a single word, and nothing else.- Text to classify: "${context.propsValue.text}"`,+ const result = await runOnWorker({+ context,+ buildRequest: async () => ({+ action: 'CLASSIFY_TEXT',+ provider,+ ...spreadIfDefined('providerConfigId', configId),+ modelId: context.propsValue.model,+ text: context.propsValue.text,+ categories,+ }),});- const result = response.text.trim();- if (!categories.includes(result)) {- throw new Error(- 'Unable to classify the text into the provided categories.'- );+ if (result.status === 'paused') {+ return {};}- return result;+ return result.output.answer;},});diff --git a/packages/pieces/community/ai/src/lib/actions/utility/extract-structured-data.ts b/packages/pieces/community/ai/src/lib/actions/utility/extract-structured-data.tsindex 36dbeff5c14..cfef9bb5269 100644--- a/packages/pieces/community/ai/src/lib/actions/utility/extract-structured-data.ts+++ b/packages/pieces/community/ai/src/lib/actions/utility/extract-structured-data.ts@@ -1,10 +1,8 @@-import { ApFile, createAction, PieceAuth, Property } from '@activepieces/pieces-framework';-import { createAIModel } from '../../common/ai-sdk';-import { generateText, tool, jsonSchema, ModelMessage, UserModelMessage } from 'ai';-import mime from 'mime-types';+import { ApFile, createAction, PieceAuth, Property, spreadIfDefined } from '@activepieces/pieces-framework';import Ajv from 'ajv';+import mime from 'mime-types';+import { runOnWorker, uploadAiFiles } from '../../common/ai-step';import { aiProps, aiProviderSelection } from '../../common/props';-import { spreadIfDefined } from '@activepieces/pieces-framework';export const extractStructuredData = createAction({audience: 'both',@@ -129,164 +127,55 @@ export const extractStructuredData = createAction({},async run(context) {const { provider, configId } = aiProviderSelection.resolveOrThrow(context.propsValue.provider);- const modelId = context.propsValue.model;- const text = context.propsValue.text;- const files = (context.propsValue.files as Array<{ file: ApFile }>) ?? [];- const prompt = context.propsValue.prompt;- const schema = context.propsValue.schema;- const maxOutputTokens = context.propsValue.maxOutputTokens;+ const attachments = ((context.propsValue.files as { file?: ApFile }[] | undefined) ?? []).map((row) => row?.file);- if (!text && !files.length) {+ if (!context.propsValue.text && attachments.length === 0) {throw new Error('Please provide text or image/PDF to extract data from.');}- const model = await createAIModel({- provider,- ...spreadIfDefined('configId', configId),- modelId,- engineToken: context.server.token,- apiUrl: context.server.apiUrl,- projectId: context.project.id,- flowId: context.flows.current.id,- runId: context.run.id,- });-- let schemaDefinition: any;- // Track sanitized-to-original name mapping to restore output keys.- const sanitizedNameMap: Record<string, string> = {};-if (context.propsValue.mode === 'advanced') {- const ajv = new Ajv();- const isValidSchema = ajv.validateSchema(schema['fields']);-- if (!isValidSchema) {- throw new Error(- JSON.stringify({- message: 'Invalid JSON schema',- errors: ajv.errors,- }),- );- }-- schemaDefinition = jsonSchema(schema['fields'] as any);- } else {- const fields = schema['fields'] as Array<{- name: string;- description?: string;- type: string;- isRequired: boolean;- }>;-- const properties: Record<string, any> = {};- const required: string[] = [];-- fields.forEach((field) => {- const sanitizedFieldName = field.name.replace(/[^a-zA-Z0-9_.-]/g, '_');- sanitizedNameMap[sanitizedFieldName] = field.name;-- properties[sanitizedFieldName] = {- type: field.type,- description: field.description,- };-- if (field.isRequired) {- required.push(sanitizedFieldName);- }- });-- const jsonSchemaObject = {- type: 'object' as const,- properties,- required,- };-- schemaDefinition = jsonSchema(jsonSchemaObject);- }-- const extractionTool = tool({- description: 'Extract structured data from the provided content',- inputSchema: schemaDefinition,- execute: async (data) => {- return data;- },- });-- const messages: Array<ModelMessage> = [];-- const contentParts: UserModelMessage['content'] = [];-- let textContent = prompt || 'Extract the following data from the provided data.';- if (text) {- textContent += `\n\nText to analyze:\n${text}`;+ assertValidJsonSchema(context.propsValue.schema['fields']);}- contentParts.push({- type: 'text',- text: textContent,+ const result = await runOnWorker({+ context,+ buildRequest: async () => ({+ action: 'EXTRACT_STRUCTURED_DATA',+ provider,+ ...spreadIfDefined('providerConfigId', configId),+ modelId: context.propsValue.model,+ ...spreadIfDefined('text', context.propsValue.text),+ ...spreadIfDefined('prompt', context.propsValue.prompt),+ files: await uploadAiFiles({ context, files: attachments, mimeTypeOf: documentMimeType }),+ schema: { mode: context.propsValue.mode ?? 'simple', fields: context.propsValue.schema['fields'] },+ ...spreadIfDefined('maxOutputTokens', context.propsValue.maxOutputTokens),+ }),});- if (files.length > 0) {- for (const fileWrapper of files) {- const file = fileWrapper.file;- if (!file) {- continue;- }- const fileType = file.extension ? mime.lookup(file.extension) : 'image/jpeg';-- if (fileType && fileType.startsWith('image') && file.base64) {- contentParts.push({- type: 'image',- image: `data:${fileType};base64,${file.base64}`,- });- } else if (fileType && fileType.startsWith('application/pdf') && file.base64) {- contentParts.push({- type: 'file',- data: `data:${fileType};base64,${file.base64}`,- mediaType: fileType,- filename: file.filename,- });- }- }+ if (result.status === 'paused') {+ return {};}- messages.push({- role: 'user',- content: contentParts,- });-- try {- const result = await generateText({- model,- maxOutputTokens,- tools: {- extractData: extractionTool,- },- toolChoice: 'required',- messages,- });-- const toolCalls = result.toolCalls;- if (!toolCalls || toolCalls.length === 0) {- throw new Error('No structured data could be extracted from the input.');- }-- const extractedData = toolCalls[0].input;-- if (Object.keys(sanitizedNameMap).length > 0 && extractedData && typeof extractedData === 'object') {- const restoredData: Record<string, unknown> = {};- for (const [key, value] of Object.entries(extractedData)) {- const originalName = sanitizedNameMap[key] ?? key;- restoredData[originalName] = value;- }- return restoredData;- }-- return extractedData;-- } catch (error) {- throw new Error(`Failed to extract structured data: ${error instanceof Error ? error.message : 'Unknown error'}`);- }+ return result.output.answer;},});-+function assertValidJsonSchema(fields: unknown): void {+ const ajv = new Ajv();+ if (!ajv.validateSchema(fields as Parameters<Ajv['validateSchema']>[0])) {+ throw new Error(+ JSON.stringify({+ message: 'Invalid JSON schema',+ errors: ajv.errors,+ }),+ );+ }+}++function documentMimeType(file: ApFile): string | undefined {+ if (!file.extension) {+ return 'image/jpeg';+ }+ const detected = mime.lookup(file.extension);+ return detected === false ? undefined : detected;+}diff --git a/packages/pieces/community/ai/src/lib/common/ai-step.ts b/packages/pieces/community/ai/src/lib/common/ai-step.tsnew file mode 100644index 00000000000..10eaba85803--- /dev/null+++ b/packages/pieces/community/ai/src/lib/common/ai-step.ts@@ -0,0 +1,131 @@+import { AuthenticationType, httpClient, HttpMethod } from '@activepieces/pieces-common';+import { ActionContext, ApFile, ExecutionType, InputPropertyMap, isNil, spreadIfDefined } from '@activepieces/pieces-framework';++export async function runOnWorker(params: {+ context: AiStepContext;+ buildRequest: () => Promise<AiStepRequest>;+}): Promise<AiStepResult> {+ const { context } = params;++ if (context.executionType === ExecutionType.RESUME) {+ return readAnswer(context.resumePayload?.body);+ }++ const request = await params.buildRequest();++ if (!context.run.canPause) {+ const { body } = await sendToServer({ context, request });+ return readAnswer(body);+ }++ const waitpoint = await context.run.createWaitpoint({+ type: 'WEBHOOK',+ resumeDateTime: new Date(Date.now() + AI_STEP_TIMEOUT_MS).toUTCString(),+ });++ await sendToServer({ context, request, waitpointId: waitpoint.id });++ context.run.waitForWaitpoint(waitpoint.id);+ return { status: 'paused' };+}++export async function uploadAiFiles(params: {+ context: AiStepContext;+ files: (ApFile | undefined)[];+ mimeTypeOf: (file: ApFile) => string | undefined;+}): Promise<AiStepFile[]> {+ const present = params.files.filter((file): file is ApFile => !isNil(file));+ return Promise.all(+ present.map(async (file) => {+ const uploaded = await params.context.files.upload({+ fileName: file.filename,+ data: file.data,+ });+ return {+ fileId: uploaded.id,+ ...spreadIfDefined('mimeType', params.mimeTypeOf(file)),+ ...spreadIfDefined('filename', file.filename),+ };+ }),+ );+}++async function sendToServer(params: {+ context: AiStepContext;+ request: AiStepRequest;+ waitpointId?: string;+}): Promise<{ body: unknown }> {+ return httpClient.sendRequest({+ method: HttpMethod.POST,+ url: `${params.context.server.apiUrl}v1/ai/execute`,+ authentication: { type: AuthenticationType.BEARER_TOKEN, token: params.context.server.token },+ body: {+ ...params.request,+ flowId: params.context.flows.current.id,+ flowRunId: params.context.run.id,+ ...spreadIfDefined('waitpointId', params.waitpointId),+ },+ });+}++function readAnswer(body: unknown): AiStepResult {+ if (isNil(body) || typeof body !== 'object') {+ throw new Error(NOTHING_REPORTED);+ }+ const payload = body as Record<string, unknown>;+ if (typeof payload['failure'] === 'string') {+ throw new Error(payload['failure']);+ }+ const output = payload['output'];+ if (isNil(output) || typeof output !== 'object') {+ throw new Error(NOTHING_REPORTED);+ }+ return { status: 'done', output: output as AiStepOutput };+}++const AI_STEP_TIMEOUT_MS = 60 * 60 * 1000;+const NOTHING_REPORTED = 'The AI step did not report a result before it timed out';++export type AiStepContext = Pick<+ ActionContext<undefined, InputPropertyMap>,+ 'server' | 'flows' | 'files' | 'run' | 'executionType'+> & { resumePayload?: { body?: unknown } };++export type AiStepAction =+ | 'ASK_AI'+ | 'SUMMARIZE_TEXT'+ | 'CLASSIFY_TEXT'+ | 'EXTRACT_STRUCTURED_DATA'+ | 'GENERATE_IMAGE';++export type AiStepFile = {+ fileId: string;+ mimeType?: string;+ filename?: string;+};++export type AiStepRequest = {+ action: AiStepAction;+ provider: string;+ providerConfigId?: string;+ modelId: string;+ prompt?: string;+ text?: string;+ categories?: string[];+ files?: AiStepFile[];+ schema?: { mode: 'simple' | 'advanced'; fields: unknown };+ advancedOptions?: Record<string, unknown>;+ conversation?: Record<string, unknown>[];+ maxOutputTokens?: number;+ temperature?: number;+ webSearch?: { enabled: boolean; options?: Record<string, unknown> };+};++export type AiStepOutput = {+ answer: unknown;+ conversation?: Record<string, unknown>[];+};++export type AiStepResult =+ | { status: 'paused' }+ | { status: 'done'; output: AiStepOutput };diff --git a/packages/pieces/community/ai/src/lib/common/web-search.ts b/packages/pieces/community/ai/src/lib/common/web-search.tsindex b3913e53cad..015f39492c5 100644--- a/packages/pieces/community/ai/src/lib/common/web-search.ts+++ b/packages/pieces/community/ai/src/lib/common/web-search.ts@@ -283,6 +283,74 @@ export function buildWebSearchConfig(params: {};}+export function usesNativeWebSearchTools({ provider, model }: { provider: string; model: string | undefined }): boolean {+ const { provider: effectiveProvider } = getEffectiveProviderAndModel({ provider, model });+ return NATIVE_WEB_SEARCH_PROVIDERS.has(effectiveProvider ?? provider);+}++export function sanitizeWebSearchOptions(saved: unknown): Record<string, unknown> {+ if (saved === null || typeof saved !== 'object') {+ return {};+ }+ const options = saved as Record<string, unknown>;+ return {+ ...spreadIfDefined('maxUses', asNumber(options['maxUses'])),+ ...spreadIfDefined('includeSources', asBoolean(options['includeSources'])),+ ...spreadIfDefined('userLocationCity', asText(options['userLocationCity'])),+ ...spreadIfDefined('userLocationRegion', asText(options['userLocationRegion'])),+ ...spreadIfDefined('userLocationCountry', asText(options['userLocationCountry'])),+ ...spreadIfDefined('userLocationTimezone', asText(options['userLocationTimezone'])),+ ...spreadIfDefined('allowedDomains', asDomainList(options['allowedDomains'])),+ ...spreadIfDefined('blockedDomains', asDomainList(options['blockedDomains'])),+ ...spreadIfDefined('searchContextSize', asSearchContextSize(options['searchContextSize'])),+ };+}++function asNumber(value: unknown): number | undefined {+ const parsed = Number(value);+ return typeof value === 'boolean' || value === null || value === '' || value === undefined || Number.isNaN(parsed)+ ? undefined+ : parsed;+}++function asBoolean(value: unknown): boolean | undefined {+ if (typeof value === 'boolean') {+ return value;+ }+ if (value === 'true' || value === 'false') {+ return value === 'true';+ }+ return undefined;+}++function asText(value: unknown): string | undefined {+ return typeof value === 'string' && value.length > 0 ? value : undefined;+}++function asDomainList(value: unknown): { domain: string }[] | undefined {+ if (!Array.isArray(value)) {+ return undefined;+ }+ const domains = value.flatMap((entry) => {+ if (typeof entry === 'string') {+ return [{ domain: entry }];+ }+ const domain = entry && typeof entry === 'object' ? (entry as Record<string, unknown>)['domain'] : undefined;+ return typeof domain === 'string' ? [{ domain }] : [];+ });+ return domains.length === 0 ? undefined : domains;+}++function asSearchContextSize(value: unknown): 'low' | 'medium' | 'high' | undefined {+ return value === 'low' || value === 'medium' || value === 'high' ? value : undefined;+}++const NATIVE_WEB_SEARCH_PROVIDERS: ReadonlySet<string> = new Set([+ AIProviderName.OPENAI,+ AIProviderName.ANTHROPIC,+ AIProviderName.GOOGLE,+]);+type BaseWebSearchOptions = {maxUses?: numberincludeSources?: booleandiff --git a/packages/pieces/framework/package.json b/packages/pieces/framework/package.jsonindex 0332864200b..386e8feec59 100644--- a/packages/pieces/framework/package.json+++ b/packages/pieces/framework/package.json@@ -1,6 +1,6 @@{"name": "@activepieces/pieces-framework",- "version": "0.39.0",+ "version": "0.40.0","type": "commonjs","main": "./dist/src/index.js","types": "./dist/src/index.d.ts",diff --git a/packages/pieces/framework/src/lib/context/index.ts b/packages/pieces/framework/src/lib/context/index.tsindex 8d10076a26f..12770382b8c 100644--- a/packages/pieces/framework/src/lib/context/index.ts+++ b/packages/pieces/framework/src/lib/context/index.ts@@ -210,6 +210,7 @@ export type WaitForWaitpointHook = (waitpointId: string) => void;export type RunContext = {id: FlowRunId;+ canPause: boolean;stop: StopHook;/** @deprecated Use createWaitpoint + waitForWaitpoint instead */pause?: PauseHook;@@ -283,8 +284,20 @@ export interface FilesService {fileName: string;data: Buffer | Readable;}): Promise<string>;+ upload({+ fileName,+ data,+ }: {+ fileName: string;+ data: Buffer | Readable;+ }): Promise<UploadedFile>;}+export type UploadedFile = {+ id: string;+ url: string;+};+export interface ConnectionsManager {get(key: stringdiff --git a/packages/pieces/framework/src/lib/test/index.ts b/packages/pieces/framework/src/lib/test/index.tsindex 05f3ada0179..5500a041a61 100644--- a/packages/pieces/framework/src/lib/test/index.ts+++ b/packages/pieces/framework/src/lib/test/index.ts@@ -1,6 +1,8 @@-import { ExecutionType, TriggerStrategy } from '@activepieces/core-piece-types';+import { ExecutionType, ResumePayload, TriggerStrategy } from '@activepieces/core-piece-types';import {ActionContext,+ CreateWaitpointParams,+ CreateWaitpointResult,SetScheduleRequest,TriggerHookContext,} from '../context';@@ -10,9 +12,14 @@ export function createMockActionContext<Props extends InputPropertyMap>(params: {propsValue: StaticPropsValue<Props>;+ resumePayload?: ResumePayload;+ canPause?: boolean;+ onCreateWaitpoint?: (waitpoint: CreateWaitpointParams) => void;+ onWaitForWaitpoint?: (waitpointId: string) => void;}): ActionContext<undefined, Props> {return {- executionType: ExecutionType.BEGIN,+ executionType: params.resumePayload ? ExecutionType.RESUME : ExecutionType.BEGIN,+ resumePayload: params.resumePayload,auth: undefined,propsValue: params.propsValue,store: {@@ -37,6 +44,7 @@ export function createMockActionContext<},files: {write: async () => 'test-file-url',+ upload: async () => ({ id: 'test-file-id', url: 'test-file-url' }),},output: {update: async () => {@@ -48,6 +56,7 @@ export function createMockActionContext<},run: {id: 'test-run-id' as string,+ canPause: params.canPause ?? true,stop: () => {return;},@@ -57,6 +66,17 @@ export function createMockActionContext<respond: () => {return;},+ createWaitpoint: async (waitpoint: CreateWaitpointParams): Promise<CreateWaitpointResult> => {+ params.onCreateWaitpoint?.(waitpoint);+ return {+ id: 'test-waitpoint-id',+ resumeUrl: 'http://localhost:3000/resume',+ buildResumeUrl: () => 'http://localhost:3000/resume',+ };+ },+ waitForWaitpoint: (waitpointId: string) => {+ params.onWaitForWaitpoint?.(waitpointId);+ },},project: {id: 'test-project-id',diff --git a/packages/server/api/src/app/ai/ai-execute-controller.ts b/packages/server/api/src/app/ai/ai-execute-controller.tsindex 559267e7a40..53620f4ee13 100644--- a/packages/server/api/src/app/ai/ai-execute-controller.ts+++ b/packages/server/api/src/app/ai/ai-execute-controller.ts@@ -1,5 +1,6 @@-import { ActivepiecesError, AIProviderName, apId, ErrorCode } from '@activepieces/core-utils'-import { AiStepAction, AiStepFile, AiStepSchema, AiStepWebSearch, ExecuteAiJobData, LATEST_JOB_DATA_SCHEMA_VERSION, maxSocketHttpBufferSizeBytes, PrincipalType, spreadIfDefined, WorkerJobType } from '@activepieces/shared'+import { ActivepiecesError, AIProviderName, apId, ErrorCode, isNil } from '@activepieces/core-utils'+import { AiStepAction, AiStepFile, AiStepSchema, AiStepWebSearch, EngineResponseStatus, ExecuteAiJobData, LATEST_JOB_DATA_SCHEMA_VERSION, maxSocketHttpBufferSizeBytes, PrincipalType, WorkerJobType } from '@activepieces/shared'+import { FastifyBaseLogger } from 'fastify'import { FastifyPluginAsyncZod } from 'fastify-type-provider-zod'import { StatusCodes } from 'http-status-codes'import { z } from 'zod'@@ -7,6 +8,7 @@ import { securityAccess } from '../core/security/authorization/fastify-security'import { system } from '../helper/system/system'import { AppSystemProp } from '../helper/system/system-props'import { assertCreditsAndAppSumoNotExceeded } from '../platform/billing-provider'+import { engineResponseWatcher } from '../workers/engine-response-watcher'import { jobQueue, JobType } from '../workers/job-queue/job-queue'export const aiExecuteController: FastifyPluginAsyncZod = async (app) => {@@ -25,23 +27,36 @@ export const aiExecuteController: FastifyPluginAsyncZod = async (app) => {const requestId = apId()const log = request.log.child({ flowRun: { id: body.flowRunId }, requestId })+ const answerInThisRequest = isNil(body.waitpointId)+ const answer = answerInThisRequest ? listenForWorkerAnswer({ requestId, log }) : undefinedawait jobQueue(log).add({id: apId(),type: JobType.ONE_TIME,- data: aiJobFor({ body, requestId, projectId, platformId: platform.id }),+ data: aiJobFor({+ body,+ requestId,+ projectId,+ platformId: platform.id,+ webserverId: answerInThisRequest ? engineResponseWatcher(log).getServerId() : undefined,+ }),})+ if (!isNil(answer)) {+ return reply.status(StatusCodes.OK).send({ requestId, ...await answer })+ }+log.info({ project: { id: projectId } }, '[aiExecuteController] Enqueued AI step')return reply.status(StatusCodes.OK).send({ requestId })})}-function aiJobFor({ body, requestId, projectId, platformId }: {+function aiJobFor({ body, requestId, projectId, platformId, webserverId }: {body: z.infer<typeof ExecuteAiRequest>requestId: stringprojectId: stringplatformId: string+ webserverId?: string}): ExecuteAiJobData {const shared = {schemaVersion: LATEST_JOB_DATA_SCHEMA_VERSION,@@ -52,52 +67,65 @@ function aiJobFor({ body, requestId, projectId, platformId }: {flowId: body.flowId,flowRunId: body.flowRunId,waitpointId: body.waitpointId,+ webserverId,provider: body.provider,modelId: body.modelId,- ...spreadIfDefined('prompt', body.prompt),- ...spreadIfDefined('providerConfigId', body.providerConfigId),- ...spreadIfDefined('maxOutputTokens', body.maxOutputTokens),- ...spreadIfDefined('temperature', body.temperature),- ...spreadIfDefined('webSearch', body.webSearch),+ prompt: body.prompt,+ providerConfigId: body.providerConfigId,+ maxOutputTokens: body.maxOutputTokens,+ temperature: body.temperature,+ webSearch: body.webSearch,} as constswitch (body.action) {case AiStepAction.ASK_AI:- return { ...shared, action: AiStepAction.ASK_AI, ...spreadIfDefined('conversation', body.conversation) }+ return { ...shared, action: AiStepAction.ASK_AI, conversation: body.conversation }case AiStepAction.SUMMARIZE_TEXT:- return { ...shared, action: AiStepAction.SUMMARIZE_TEXT, ...spreadIfDefined('text', body.text) }+ return { ...shared, action: AiStepAction.SUMMARIZE_TEXT, text: body.text }case AiStepAction.CLASSIFY_TEXT:return {...shared,action: AiStepAction.CLASSIFY_TEXT,- ...spreadIfDefined('text', body.text),- ...spreadIfDefined('categories', body.categories),+ text: body.text,+ categories: body.categories,}case AiStepAction.EXTRACT_STRUCTURED_DATA:return {...shared,action: AiStepAction.EXTRACT_STRUCTURED_DATA,- ...spreadIfDefined('text', body.text),- ...spreadIfDefined('files', body.files),- ...spreadIfDefined('schema', body.schema),+ text: body.text,+ files: body.files,+ schema: body.schema,}case AiStepAction.GENERATE_IMAGE:return {...shared,action: AiStepAction.GENERATE_IMAGE,- ...spreadIfDefined('files', body.files),- ...spreadIfDefined('advancedOptions', body.advancedOptions),+ files: body.files,+ advancedOptions: body.advancedOptions,}}}+async function listenForWorkerAnswer({ requestId, log }: { requestId: string, log: FastifyBaseLogger }): Promise<{ output?: unknown, failure?: string }> {+ const timeoutMs = system.getNumberOrThrow(AppSystemProp.FLOW_TIMEOUT_SECONDS) * 1000+ const response = await engineResponseWatcher(log).oneTimeListener<WorkerResponse | undefined>(requestId, true, timeoutMs, undefined)+ if (isNil(response)) {+ return { failure: 'The AI step did not finish in time' }+ }+ if (response.status !== EngineResponseStatus.OK) {+ return { failure: response.error ?? 'The AI step failed' }+ }+ return response.response ?? { failure: 'The AI step reported nothing back' }+}+const RUN_PRINCIPALS = [PrincipalType.ENGINE] as constconst ExecuteAiRequest = z.object({action: z.enum(AiStepAction),flowId: z.string(),flowRunId: z.string(),- waitpointId: z.string(),+ waitpointId: z.string().optional(),provider: z.enum(AIProviderName),providerConfigId: z.string().optional(),modelId: z.string(),@@ -115,6 +143,8 @@ const ExecuteAiRequest = z.object({const ExecuteAiResponse = z.object({requestId: z.string(),+ output: z.unknown().optional(),+ failure: z.string().optional(),})const ExecuteAiRoute = {@@ -127,3 +157,9 @@ const ExecuteAiRoute = {response: { [StatusCodes.OK]: ExecuteAiResponse },},}++type WorkerResponse = {+ status: EngineResponseStatus+ response?: { output?: unknown, failure?: string }+ error?: string+}diff --git a/packages/server/api/src/app/workers/job-queue/job-broker.ts b/packages/server/api/src/app/workers/job-queue/job-broker.tsindex 1b6b2453e1f..9352e9086bf 100644--- a/packages/server/api/src/app/workers/job-queue/job-broker.ts+++ b/packages/server/api/src/app/workers/job-queue/job-broker.ts@@ -11,7 +11,7 @@ import { rateLimiterInterceptor } from './interceptors/rate-limiter-interceptor'import { zombiePollingInterceptor } from './interceptors/zombie-polling-interceptor'import { jobAssignmentTracker } from './job-assignment-tracker'import { InterceptorVerdict, JobInterceptor } from './job-interceptor'-import { isUserInteractionJobData } from './job-queue'+import { callerWaitingForResponse } from './job-queue'import { createQueueDispatcher, QueueDispatcher } from './queue-dispatcher'const DRAIN_DELAY_SECONDS = 15@@ -243,17 +243,17 @@ export const jobBroker = (log: FastifyBaseLogger) => ({}const jobData = JobData.parse(job.data)- const userJobData = isUserInteractionJobData(jobData) ? jobData : null+ const waitingCaller = callerWaitingForResponse(jobData)const { error } = await tryCatch(async () => {if (input.status === EngineResponseStatus.INTERNAL_ERROR) {- if (userJobData) {+ if (waitingCaller) {// User-interaction jobs (piece-metadata extraction, validation, property/auth, trigger// hooks) are synchronous request/response — the caller awaits the result with a timeout.// Return the error to that caller and COMPLETE the job instead of moving it to failed: the// exponential-backoff retry only fires long after the caller has timed out, so it serves no// one and just piles up dead jobs in the failed queue.- await engineResponseWatcher(log).publish(userJobData.webserverId, userJobData.requestId, {+ await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {status: EngineResponseStatus.INTERNAL_ERROR,response: undefined,error: input.errorMessage ?? 'Internal error',@@ -267,8 +267,8 @@ export const jobBroker = (log: FastifyBaseLogger) => ({}await job.moveToCompleted({ response: input.response ?? undefined }, input.token, false)- if (userJobData) {- await engineResponseWatcher(log).publish(userJobData.webserverId, userJobData.requestId, {+ if (waitingCaller) {+ await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {status: input.status,response: input.response,error: input.errorMessage,@@ -278,8 +278,8 @@ export const jobBroker = (log: FastifyBaseLogger) => ({})if (error) {log.error({ job: { id: input.jobId }, error: String(error), originalError: input.errorMessage }, '[jobBroker] Failed to move job to final state — leaving for stalled-scan recovery')- if (userJobData) {- await engineResponseWatcher(log).publish(userJobData.webserverId, userJobData.requestId, {+ if (waitingCaller) {+ await engineResponseWatcher(log).publish(waitingCaller.webserverId, waitingCaller.requestId, {status: EngineResponseStatus.INTERNAL_ERROR,response: undefined,error: String(error),diff --git a/packages/server/api/src/app/workers/job-queue/job-queue.ts b/packages/server/api/src/app/workers/job-queue/job-queue.tsindex e7a4021fe31..ad442a20828 100644--- a/packages/server/api/src/app/workers/job-queue/job-queue.ts+++ b/packages/server/api/src/app/workers/job-queue/job-queue.ts@@ -206,6 +206,16 @@ export function isUserInteractionJobData(jobData: JobData): jobData is UserInterreturn USER_INTERACTION_JOB_TYPES.has(jobData.jobType)}+export function callerWaitingForResponse(jobData: JobData): { webserverId: string, requestId: string } | null {+ if (isUserInteractionJobData(jobData)) {+ return { webserverId: jobData.webserverId, requestId: jobData.requestId }+ }+ if (jobData.jobType === WorkerJobType.EXECUTE_AI && !isNil(jobData.webserverId)) {+ return { webserverId: jobData.webserverId, requestId: jobData.requestId }+ }+ return null+}+export const PROJECT_GROUP_ROUTABLE_JOB_TYPES = new Set<WorkerJobType>([WorkerJobType.EXECUTE_FLOW,WorkerJobType.EXECUTE_WEBHOOK,diff --git a/packages/server/engine/src/lib/handler/piece-executor.ts b/packages/server/engine/src/lib/handler/piece-executor.tsindex 49070b0e406..45b03d1e0ff 100644--- a/packages/server/engine/src/lib/handler/piece-executor.ts+++ b/packages/server/engine/src/lib/handler/piece-executor.ts@@ -128,6 +128,7 @@ const executeAction: ActionHandler<PieceAction> = async ({ action, executionStat}),run: {id: constants.flowRunId,+ canPause: !constants.actionRunMode,stop: createStopHook(params),respond: createRespondHook(params),createWaitpoint: createWaitpointHook({ constants, stepName: action.name, hookParams: params }),diff --git a/packages/server/engine/src/lib/piece-context/file-uploader.ts b/packages/server/engine/src/lib/piece-context/file-uploader.tsindex 0185d5e76c5..4e93c238642 100644--- a/packages/server/engine/src/lib/piece-context/file-uploader.ts+++ b/packages/server/engine/src/lib/piece-context/file-uploader.ts@@ -1,28 +1,30 @@import { Readable } from 'node:stream'import { apId } from '@activepieces/core-utils'-import { FilesService } from '@activepieces/pieces-framework'+import { FilesService, UploadedFile } from '@activepieces/pieces-framework'import { EngineGenericError, FileSizeError, FileType } from '@activepieces/shared'import { engineFileApi } from '../api/engine-file-api'export function createFileUploader({ engineToken, apiUrl }: CreateFileUploaderParams): FilesService {+ const upload = async ({ fileName, data }: { fileName: string, data: Buffer | Readable }): Promise<UploadedFile> => {+ if (!Buffer.isBuffer(data) && !(data instanceof Readable)) {+ throw new Error(`Expected file data to be a Buffer or Readable stream, but received ${describeType(data)}`)+ }+ const maxBytes = resolveMaxFileSizeBytes()+ const payload = Buffer.isBuffer(data) ? data : await drainToBuffer({ stream: data, maxBytes })+ assertWithinLimit({ sizeInBytes: payload.length, maxBytes })+ const { fileId, readUrl } = await engineFileApi.upload({+ engineToken,+ apiUrl,+ fileId: apId(),+ type: FileType.FLOW_STEP_FILE,+ fileName,+ data: payload,+ })+ return { id: fileId, url: readUrl }+ }return {- write: async ({ fileName, data }: { fileName: string, data: Buffer | Readable }): Promise<string> => {- if (!Buffer.isBuffer(data) && !(data instanceof Readable)) {- throw new Error(`Expected file data to be a Buffer or Readable stream, but received ${describeType(data)}`)- }- const maxBytes = resolveMaxFileSizeBytes()- const payload = Buffer.isBuffer(data) ? data : await drainToBuffer({ stream: data, maxBytes })- assertWithinLimit({ sizeInBytes: payload.length, maxBytes })- const { readUrl } = await engineFileApi.upload({- engineToken,- apiUrl,- fileId: apId(),- type: FileType.FLOW_STEP_FILE,- fileName,- data: payload,- })- return readUrl- },+ upload,+ write: async (params: { fileName: string, data: Buffer | Readable }): Promise<string> => (await upload(params)).url,}}diff --git a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.tsindex 92c9e4c7b9b..6408385cfde 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/execute-ai.ts@@ -2,16 +2,19 @@ import { AIProviderName, isNil, spreadIfDefined, tryCatch } from '@activepieces/import { aiUtils, FlowStepMetadata } from '@activepieces/server-utils'import { AiStepAction, ClassifyTextJobData, EngineResponseStatus, ExecuteAiJobData, getEffectiveProviderAndModel, ResolveAiProviderResponse, WorkerJobType } from '@activepieces/shared'import { generateText, ModelMessage, stepCountIs } from 'ai'-import { FireAndForgetJobResult, JobContext, JobHandler, JobResultKind } from '../../types'+import { JobContext, JobHandler, JobResult, JobResultKind } from '../../types'import { resolveAiFiles } from './ai-files'import { extractStructuredData } from './extract-structured-data'import { generateImageStep } from './generate-image'-export const executeAiJob: JobHandler<ExecuteAiJobData, FireAndForgetJobResult> = {+export const executeAiJob: JobHandler<ExecuteAiJobData, JobResult> = {jobType: WorkerJobType.EXECUTE_AI,- async execute(ctx: JobContext, data: ExecuteAiJobData): Promise<FireAndForgetJobResult> {- const { data: output, error } = await tryCatch(() => runAiStep(ctx, data))+ async execute(ctx: JobContext, data: ExecuteAiJobData): Promise<JobResult> {+ const { data: output, error } = await tryCatch(() => callTheModel({ ctx, data }))const stepOutput = isNil(error) ? { output } : { failure: toFailureMessage(error) }+ if (isNil(data.waitpointId)) {+ return { kind: JobResultKind.SYNCHRONOUS, status: EngineResponseStatus.OK, response: stepOutput }+ }const handedBack = await handBackToTheFlow({ ctx, data, waitpointId: data.waitpointId, output: stepOutput })if (!isNil(error)) {ctx.log.warn({ flowRun: { id: data.flowRunId }, requestId: data.requestId, error }, '[executeAiJob] Handed the failure back to the flow')@@ -47,7 +50,7 @@ async function waitBeforeRetry(attempt: number): Promise<void> {return new Promise((resolve) => setTimeout(resolve, attempt * RETRY_DELAY_MS))}-async function runAiStep(ctx: JobContext, data: ExecuteAiJobData): Promise<unknown> {+async function callTheModel({ ctx, data }: { ctx: JobContext, data: ExecuteAiJobData }): Promise<unknown> {const resolved = await ctx.apiClient.resolveAiProvider({projectId: data.projectId,platformId: data.platformId,@@ -90,6 +93,7 @@ async function runTextStep({ data, resolved, flowStep }: { data: ExecuteAiJobDattools,...spreadIfDefined('maxOutputTokens', data.maxOutputTokens),...spreadIfDefined('temperature', data.temperature),+ ...spreadIfDefined('providerOptions', reasoningEffortFor({ action: data.action, provider })),...spreadIfDefined('stopWhen', Object.keys(tools).length === 0 ? undefined : stepCountIs(webSearchOptions?.maxUses ?? DEFAULT_WEB_SEARCH_STEPS)),})@@ -126,6 +130,13 @@ function classificationPrompt(data: ClassifyTextJobData): string {Text to classify: "${data.text ?? ''}"`}+function reasoningEffortFor({ action, provider }: { action: AiStepAction, provider: AIProviderName }): Record<string, Record<string, string>> | undefined {+ if (action !== AiStepAction.SUMMARIZE_TEXT || provider !== AIProviderName.OPENAI) {+ return undefined+ }+ return { [AIProviderName.OPENAI]: { reasoning_effort: 'minimal' } }+}+function toStepOutput({ data, text, sources }: { data: ExecuteAiJobData, text: string, sources: unknown }): unknown {switch (data.action) {case AiStepAction.ASK_AI: {diff --git a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.tsindex 2557b4ee6f1..99a6e6f0b17 100644--- a/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts+++ b/packages/server/worker/src/lib/execute/jobs/ai/extract-structured-data.ts@@ -10,9 +10,6 @@ export async function extractStructuredData({ data, resolved, flowStep, files }:flowStep: FlowStepMetadatafiles: ResolvedAiFile[]}): Promise<unknown> {- if (isNil(data.text) && files.length === 0) {- throw new Error('Please provide text or image/PDF to extract data from.')- }const model = aiUtils.createModel({provider: resolved.provider,auth: resolved.auth,@@ -67,11 +64,11 @@ function buildSchema(data: ExtractStructuredDataJobData): { schemaDefinition: Re}function buildMessages({ data, files }: { data: ExtractStructuredDataJobData, files: ResolvedAiFile[] }): ModelMessage[] {- let textContent = data.prompt ?? 'Extract the following data from the provided data.'- if (data.text) {- textContent += `\n\nText to analyze:\n${data.text}`- }- const contentParts: UserModelMessage['content'] = [{ type: 'text', text: textContent }]+ const prompt = data.prompt ?? ''+ const guide = prompt.length === 0 ? 'Extract the following data from the provided data.' : prompt+ const text = data.text ?? ''+ const textContent = text.length === 0 ? guide : `${guide}\n\nText to analyze:\n${text}`+ const contentParts: Exclude<UserModelMessage['content'], string> = [{ type: 'text', text: textContent }]for (const file of files) {if (file.base64.length === 0) {continue