AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
feat(firecrawl): capture browser action outputs & add storeInCache toggle (#15258)
Ada Chia Yong Yun4a38c2f16dc2
diff --git a/bun.lock b/bun.lockindex fb5c75d89fa..ccd24ef3aad 100644--- a/bun.lock+++ b/bun.lock@@ -3227,7 +3227,7 @@},"packages/pieces/community/firecrawl": {"name": "@activepieces/piece-firecrawl",- "version": "0.5.0",+ "version": "0.6.0","dependencies": {"@activepieces/core-piece-types": "workspace:*","@activepieces/core-utils": "workspace:*",diff --git a/docs/install/reference/breaking-changes-unreleased.mdx b/docs/install/reference/breaking-changes-unreleased.mdxindex 2f15170b5a7..d266bc3ec0a 100644--- a/docs/install/reference/breaking-changes-unreleased.mdx+++ b/docs/install/reference/breaking-changes-unreleased.mdx@@ -8,6 +8,20 @@ Entries here move to [Breaking Changes](/install/reference/breaking-changes) aut## What has changed?+#### Firecrawl's Scrape Website stops reusing cached pages, and Crawl keeps its pages when a screenshot fails++Three changes in Firecrawl 0.6.0 alter what an existing step does once it is upgraded to that version. Steps keep running on the version they were built with until you upgrade them.++Scrape Website has a new **Store In Cache** checkbox, off by default. When it is off, the step asks Firecrawl for a fresh copy of the page and tells Firecrawl not to keep one. A step upgraded from 0.5.0 or earlier keeps its saved inputs, which have no value for the checkbox, so it is treated as off: a step that used to be served Firecrawl's cached copy of a page now scrapes it fresh on every run. Runs are slower, and each one costs a full scrape in Firecrawl credits.++Crawl with the Screenshot output format used to leave the raw Firecrawl screenshot link in a page's `screenshot` when saving that screenshot failed, and logged the failure only to the server console. That page now has no `screenshot` and carries a `screenshotError` saying why. The rest of the crawl is returned as before.++Saved screenshot files are now named `firecrawl-<id>.png` instead of `screenshot-<id>.png`. The `fileName` and `fileUrl` keys are unchanged.++#### What you need to do++Nothing to configure or migrate, and no existing step stops working. If a Scrape Website step scrapes the same pages repeatedly and does not need them fresh, tick **Store In Cache** to get the old speed and cost back. If a flow reads `screenshot` from Crawl's pages, check `screenshotError` on pages where it is missing. Only a flow that matches on a saved file's name needs changing.+#### SmartSuite 0.3.0: Upload File returns the file list, and record and selector actions are builder-onlyUpload File used to return the record's fields keyed by field label. From SmartSuite 0.3.0 it returns `{ recordId, field, fileName, files }`, where `files` is the field's file list (handle, name and metadata) after the upload.diff --git a/packages/pieces/community/firecrawl/package.json b/packages/pieces/community/firecrawl/package.jsonindex 7f3c64b3a7c..c6595ef3739 100644--- a/packages/pieces/community/firecrawl/package.json+++ b/packages/pieces/community/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@activepieces/piece-firecrawl",- "version": "0.5.0",+ "version": "0.6.0","main": "./dist/src/index.js","types": "./dist/src/index.d.ts","scripts": {diff --git a/packages/pieces/community/firecrawl/src/i18n/translation.json b/packages/pieces/community/firecrawl/src/i18n/translation.jsonindex 0a4fc165c54..a97eae5210c 100644--- a/packages/pieces/community/firecrawl/src/i18n/translation.json+++ b/packages/pieces/community/firecrawl/src/i18n/translation.json@@ -45,6 +45,7 @@"Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint","Website URL": "Website URL","Timeout (ms)": "Timeout (ms)",+ "Store In Cache": "Store In Cache","Perform Actions Before Scraping": "Perform Actions Before Scraping","Action Properties": "Action Properties","Output Format": "Output Format",@@ -93,6 +94,7 @@"Follow redirects": "Follow redirects","The webpage URL to scrape.": "The webpage URL to scrape.","Maximum time to wait for the page to load (in milliseconds).": "Maximum time to wait for the page to load (in milliseconds).",+ "If enabled, the page will be stored in the Firecrawl index and cache. Disabling this is useful if your scraping activity may have data protection concerns.": "If enabled, the page will be stored in the Firecrawl index and cache. Disabling this is useful if your scraping activity may have data protection concerns.","Enable to perform a sequence of actions on the page before scraping (like clicking buttons, filling forms, etc.). See [Firecrawl Actions Documentation](https://docs.firecrawl.dev/api-reference/endpoint/scrape#body-actions) for details on available actions and their parameters.": "Enable to perform a sequence of actions on the page before scraping (like clicking buttons, filling forms, etc.). See [Firecrawl Actions Documentation](https://docs.firecrawl.dev/api-reference/endpoint/scrape#body-actions) for details on available actions and their parameters.","Properties for actions that will be performed on the page.": "Properties for actions that will be performed on the page.","Choose what format you want your output in.": "Choose what format you want your output in.",diff --git a/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts b/packages/pieces/community/firecrawl/src/lib/actions/crawl.tsindex e3df594532d..d5e83dcc226 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts@@ -1,8 +1,8 @@-import { createAction, Property, InputPropertyMap } from '@activepieces/pieces-framework';-import { httpClient, HttpMethod } from '@activepieces/pieces-common';+import { createAction, Property, InputPropertyMap, tryCatch } from '@activepieces/pieces-framework';+import { httpClient, HttpError, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { forScreenshotOutputFormat, forSimpleOutputFormat, forJsonOutputFormat, polling, downloadAndSaveCrawlScreenshots, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';-import { crawlWebsiteActionOutputSchema } from '../output-schemas';+import { forScreenshotOutputFormat, forSimpleOutputFormat, forJsonOutputFormat, polling, saveFirecrawlFile, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';+import { crawlActionOutputSchema } from '../output-schemas';function webhookConfig(useWebhook: boolean, webhookProperties: any): any {if (!useWebhook || !webhookProperties) {@@ -40,7 +40,7 @@ export const crawl = createAction({displayName: 'Crawl',description: 'Crawl multiple pages from a website based on specified rules and patterns.',audience: 'human',- outputSchema: crawlWebsiteActionOutputSchema,+ outputSchema: crawlActionOutputSchema,aiMetadata: { description: 'Starts from a base URL, discovers and follows links across the site, and returns the content of many pages in a chosen format (markdown, HTML, links, summary, screenshot, or AI-extracted JSON). Choose this to gather content from a whole site or section; use Scrape for a single page or Map to only list URLs without fetching content. Bounded by the page limit and timeout, and can deliver results to a webhook. Read-only against the site, so re-running the same call is safe.', idempotent: true },props: {url: Property.ShortText({@@ -315,10 +315,33 @@ export const crawl = createAction({const timeoutSeconds = propsValue.timeout || 300;const result = await polling(jobId, auth.secret_text, timeoutSeconds, 'crawl');- if (propsValue.formats === 'screenshot') {- await downloadAndSaveCrawlScreenshots(result, context);+ if (propsValue.formats === 'screenshot' && Array.isArray(result.data)) {+ const savedPages = [];+ for (const page of result.data) {+ if (!page.screenshot) {+ savedPages.push(page);+ continue;+ }+ const { screenshot, ...pageWithoutScreenshot } = page;+ const { data: savedScreenshot, error } = await tryCatch(() =>+ saveFirecrawlFile({ context, firecrawlFileUrl: screenshot })+ );+ savedPages.push(+ error+ ? { ...pageWithoutScreenshot, screenshotError: screenshotErrorMessage(error) }+ : { ...page, screenshot: savedScreenshot }+ );+ }+ result.data = savedPages;}return result;},-});\ No newline at end of file+});++function screenshotErrorMessage(error: unknown): string {+ if (error instanceof HttpError) {+ return `Screenshot download failed (HTTP ${error.response.status})`;+ }+ return error instanceof Error ? error.message : String(error);+}diff --git a/packages/pieces/community/firecrawl/src/lib/actions/scrape-url.ts b/packages/pieces/community/firecrawl/src/lib/actions/scrape-url.tsindex 49983906769..f0619b82539 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/scrape-url.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/scrape-url.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { downloadAndSaveScreenshot, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';+import { saveFirecrawlFile, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { scrapeUrlActionOutputSchema } from '../output-schemas';export const scrapeUrl = createAction({@@ -87,7 +87,7 @@ export const scrapeUrl = createAction({const result = response.body;if (format === 'screenshot' && result?.data?.screenshot) {- await downloadAndSaveScreenshot(result.data, context);+ result.data.screenshot = await saveFirecrawlFile({ context, firecrawlFileUrl: result.data.screenshot });}return result;} catch (error: any) {diff --git a/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts b/packages/pieces/community/firecrawl/src/lib/actions/scrape.tsindex 055ad731ad9..79ead4ac357 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts@@ -1,8 +1,8 @@import { createAction, Property, InputPropertyMap } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { forScreenshotOutputFormat, forSimpleOutputFormat, downloadAndSaveScreenshot, forJsonOutputFormat, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';-import { scrapeUrlActionOutputSchema } from '../output-schemas';+import { forScreenshotOutputFormat, forSimpleOutputFormat, saveFirecrawlFile, forJsonOutputFormat, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';+import { scrapeActionOutputSchema } from '../output-schemas';function forDefaultScreenshot(): any {return {@@ -18,7 +18,7 @@ export const scrape = createAction({displayName: 'Scrape Website',description: 'Scrape a website by performing a series of actions like clicking, typing, taking screenshots, and extracting data.',audience: 'human',- outputSchema: scrapeUrlActionOutputSchema,+ outputSchema: scrapeActionOutputSchema,aiMetadata: { description: 'Fetches the content of a single web page and returns it in a chosen format (markdown, HTML, links, summary, screenshot, or AI-extracted JSON). Choose this to read one specific URL; for many pages use Crawl, and for a structured data pull across several known URLs use Extract Structured Data. Optionally runs browser actions (click, type, wait, screenshot) before scraping. Read-only against the target, so repeating the same call is safe.', idempotent: true },props: {url: Property.ShortText({@@ -32,6 +32,12 @@ export const scrape = createAction({required: false,defaultValue: 60000,}),+ storeInCache: Property.Checkbox({+ displayName: 'Store In Cache',+ description: 'If enabled, the page will be stored in the Firecrawl index and cache. Disabling this is useful if your scraping activity may have data protection concerns.',+ required: false,+ defaultValue: false,+ }),useActions: Property.Checkbox({displayName: 'Perform Actions Before Scraping',description: 'Enable to perform a sequence of actions on the page before scraping (like clicking buttons, filling forms, etc.). See [Firecrawl Actions Documentation](https://docs.firecrawl.dev/api-reference/endpoint/scrape#body-actions) for details on available actions and their parameters.',@@ -73,6 +79,10 @@ export const scrape = createAction({{type: 'screenshot',},+ {+ type: 'pdf',+ format: 'A4',+ },],}),};@@ -95,7 +105,7 @@ export const scrape = createAction({{ label: 'Links', value: 'links' },{ label: 'Images', value: 'images' },{ label: 'Screenshot', value: 'screenshot' },- { label: 'JSON', value: 'json' }+ { label: 'JSON', value: 'json' },]};},@@ -235,18 +245,24 @@ export const scrape = createAction({},async run(context) {const { auth, propsValue } = context;+ const shouldStoreInCache = propsValue.storeInCache ?? false;const body: Record<string, any> = {url: propsValue.url,origin: FIRECRAWL_ORIGIN,timeout: propsValue.timeout,+ storeInCache: shouldStoreInCache,};-++ if (!shouldStoreInCache) {+ body['maxAge'] = 0;+ }+if (propsValue.useActions && propsValue.actionProperties && propsValue.actionProperties['actions']) {body['actions'] = propsValue.actionProperties['actions'] || [];}-+const format = propsValue.formats as string;- const formatsArray: any[] = [];+ const formatsArray: Array<string | Record<string, unknown>> = [];// user selectionif (format === 'screenshot') {@@ -286,15 +302,44 @@ export const scrape = createAction({});const result = response.body;- await downloadAndSaveScreenshot(result.data, context);+ if (!result?.data) {+ throw new Error(`Firecrawl returned no page data${result?.error ? `: ${result.error}` : ''}`);+ }+ const savedScreenshot = result.data.screenshot+ ? await saveFirecrawlFile({ context, firecrawlFileUrl: result.data.screenshot })+ : undefined;+ const savedPdfs = await Promise.all(+ (result.data.actions?.pdfs ?? []).map((pdfUrl: string) =>+ saveFirecrawlFile({ context, firecrawlFileUrl: pdfUrl })+ )+ );+ const savedActionScreenshots = await Promise.all(+ (result.data.actions?.screenshots ?? []).map((screenshotUrl: string) =>+ saveFirecrawlFile({ context, firecrawlFileUrl: screenshotUrl })+ )+ );+ const javascriptReturns = result.data.actions?.javascriptReturns ?? [];+ const scrapes = result.data.actions?.scrapes ?? [];- // reorder the data object to put screenshot first, then user's selected format only- result.data = {- screenshot: result.data.screenshot,- [format]: result.data[format],- metadata: result.data.metadata+ const output: { success: boolean; data: Record<string, unknown> } = {+ success: result.success,+ data: {+ screenshot: savedScreenshot,+ actions: {+ pdfs: savedPdfs,+ screenshots: savedActionScreenshots,+ javascriptReturns,+ scrapes,+ },+ metadata: result.data.metadata,+ warning: result.data.warning,+ },};- return result;+ if (format !== 'screenshot') {+ output.data[format] = result.data[format];+ }++ return output;},-});\ No newline at end of file+});diff --git a/packages/pieces/community/firecrawl/src/lib/common/common.ts b/packages/pieces/community/firecrawl/src/lib/common/common.tsindex c34ff812df6..7d14d2dfbbf 100644--- a/packages/pieces/community/firecrawl/src/lib/common/common.ts+++ b/packages/pieces/community/firecrawl/src/lib/common/common.ts@@ -2,6 +2,7 @@ import {httpClient,HttpMethod,} from '@activepieces/pieces-common';+import { FilesService } from '@activepieces/pieces-framework';import Ajv from 'ajv';import { randomUUID } from 'crypto';@@ -19,42 +20,36 @@ export const forSimpleOutputFormat = (format: string): string => {return format;}-// Download and save screenshot(s) - works for both single scrape and crawl results-export async function downloadAndSaveScreenshot(screenshotTarget: any, context: any): Promise<void> {- const screenshotUrl = screenshotTarget.screenshot;+const FILE_EXTENSIONS_BY_MIME_TYPE: Record<string, string> = {+ 'image/png': 'png',+ 'image/jpeg': 'jpg',+ 'image/webp': 'webp',+ 'image/gif': 'gif',+ 'application/pdf': 'pdf',+};++export async function saveFirecrawlFile({+ context,+ firecrawlFileUrl,+}: {+ context: { files: FilesService };+ firecrawlFileUrl: string;+}): Promise<{ fileName: string; fileUrl: string }> {const response = await httpClient.sendRequest({method: HttpMethod.GET,- url: screenshotUrl,+ url: firecrawlFileUrl,responseType: 'arraybuffer'});- const fileName = `screenshot-${randomUUID()}.png`;+ const contentTypeHeader = response.headers?.['content-type'];+ const fileMimetype = typeof contentTypeHeader === 'string' ? contentTypeHeader.split(';')[0].trim().toLowerCase() : '';+ const fileName = `firecrawl-${randomUUID()}.${FILE_EXTENSIONS_BY_MIME_TYPE[fileMimetype] ?? 'bin'}`;const fileUrl = await context.files.write({fileName: fileName,data: Buffer.from(response.body),});- screenshotTarget.screenshot = {- fileName: fileName,- fileUrl: fileUrl,- };-}--export async function downloadAndSaveCrawlScreenshots(crawlResult: any, context: any): Promise<void> {-- if (!crawlResult.data || !Array.isArray(crawlResult.data)) {- return;- }-- for (const data of crawlResult.data) {- if (data.screenshot) {- try {- await downloadAndSaveScreenshot(data, context);- } catch (error) {- console.error(`Failed to download screenshot for page: ${error}`);- }- }- }+ return { fileName, fileUrl };}// scrape, extract and crawl uses this functiondiff --git a/packages/pieces/community/firecrawl/src/lib/output-schemas.ts b/packages/pieces/community/firecrawl/src/lib/output-schemas.tsindex bb1e01ba4cc..b0b0986367f 100644--- a/packages/pieces/community/firecrawl/src/lib/output-schemas.ts+++ b/packages/pieces/community/firecrawl/src/lib/output-schemas.ts@@ -1,6 +1,6 @@import { OutputSchema } from '@activepieces/pieces-framework';-const scrapeResultFields: OutputSchema['fields'] = [+const pageContentFields: OutputSchema['fields'] = [{ key: 'markdown', label: 'Markdown' },{ key: 'html', label: 'HTML', format: 'html' },{ key: 'rawHtml', label: 'Raw HTML', format: 'html' },@@ -14,28 +14,91 @@ const scrapeResultFields: OutputSchema['fields'] = [{ key: 'fileUrl', label: 'File URL', format: 'url' },],},- {- key: 'metadata',- label: 'Metadata',- children: [- { key: 'title', label: 'Title' },- { key: 'language', label: 'Language' },- { key: 'sourceURL', label: 'Source URL', format: 'url' },- { key: 'url', label: 'Final URL', format: 'url' },- { key: 'statusCode', label: 'Status Code', format: 'number' },- { key: 'favicon', label: 'Favicon', format: 'image' },- { key: 'creditsUsed', label: 'Credits Used', format: 'number' },- ],- },];-export const scrapeUrlActionOutputSchema: OutputSchema = {+const pageMetadataField: OutputSchema['fields'][number] = {+ key: 'metadata',+ label: 'Metadata',+ children: [+ { key: 'title', label: 'Title' },+ { key: 'language', label: 'Language' },+ { key: 'sourceURL', label: 'Source URL', format: 'url' },+ { key: 'url', label: 'Final URL', format: 'url' },+ { key: 'statusCode', label: 'Status Code', format: 'number' },+ { key: 'favicon', label: 'Favicon', format: 'image' },+ { key: 'creditsUsed', label: 'Credits Used', format: 'number' },+ ],+};++const scrapeActionsField: OutputSchema['fields'][number] = {+ key: 'actions',+ label: 'Actions',+ children: [+ {+ key: 'pdfs',+ label: 'PDF',+ listItems: [+ { key: 'fileName', label: 'File Name' },+ { key: 'fileUrl', label: 'File URL', format: 'url' },+ ],+ },+ {+ key: 'screenshots',+ label: 'Screenshots',+ listItems: [+ { key: 'fileName', label: 'File Name' },+ { key: 'fileUrl', label: 'File URL', format: 'url' },+ ],+ },+ {+ key: 'javascriptReturns',+ label: 'JavaScript Returns',+ listItems: [+ { key: 'type', label: 'Type' },+ { key: 'value', label: 'Value' },+ ],+ },+ {+ key: 'scrapes',+ label: 'Scrapes',+ labelKey: 'url',+ listItems: [+ { key: 'url', label: 'URL', format: 'url' },+ { key: 'html', label: 'HTML', format: 'html' },+ ],+ },+ ],+};++const pageWarningField: OutputSchema['fields'][number] = { key: 'warning', label: 'Warning' };++const pageResultFields: OutputSchema['fields'] = [+ ...pageContentFields,+ pageWarningField,+ pageMetadataField,+];++const scrapeResultFields: OutputSchema['fields'] = [+ ...pageContentFields,+ scrapeActionsField,+ pageWarningField,+ pageMetadataField,+];++export const scrapeActionOutputSchema: OutputSchema = {fields: [{ key: 'success', label: 'Success', format: 'boolean' },{ key: 'data', label: 'Data', children: scrapeResultFields },],};+export const scrapeUrlActionOutputSchema: OutputSchema = {+ fields: [+ { key: 'success', label: 'Success', format: 'boolean' },+ { key: 'data', label: 'Data', children: pageResultFields },+ ],+};+const jobResultFields: OutputSchema['fields'] = [{ key: 'success', label: 'Success', format: 'boolean' },{ key: 'status', label: 'Status' },@@ -44,13 +107,24 @@ const jobResultFields: OutputSchema['fields'] = [{ key: 'creditsUsed', label: 'Credits Used', format: 'number' },{ key: 'expiresAt', label: 'Expires At', format: 'datetime' },{ key: 'next', label: 'Next Page URL', format: 'url' },- { key: 'data', label: 'Pages', listItems: scrapeResultFields },+ { key: 'data', label: 'Pages', listItems: pageResultFields },];export const crawlWebsiteActionOutputSchema: OutputSchema = {fields: jobResultFields,};+export const crawlActionOutputSchema: OutputSchema = {+ fields: [+ ...jobResultFields.filter((field) => field.key !== 'data'),+ {+ key: 'data',+ label: 'Pages',+ listItems: [...pageResultFields, { key: 'screenshotError', label: 'Screenshot Error' }],+ },+ ],+};+export const getCrawlResultsActionOutputSchema: OutputSchema = {fields: jobResultFields,};