AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
feat(firecrawl): return page content from Search Web, add news/images sources and time filter (#16173)
Juampi7eded14caf23
diff --git a/bun.lock b/bun.lockindex 844c20030c6..f79006079c7 100644--- a/bun.lock+++ b/bun.lock@@ -3219,7 +3219,7 @@},"packages/pieces/community/firecrawl": {"name": "@activepieces/piece-firecrawl",- "version": "0.4.1",+ "version": "0.5.0","dependencies": {"@activepieces/core-piece-types": "workspace:*","@activepieces/core-utils": "workspace:*",diff --git a/packages/pieces/community/firecrawl/package.json b/packages/pieces/community/firecrawl/package.jsonindex 623bc97b748..7f3c64b3a7c 100644--- a/packages/pieces/community/firecrawl/package.json+++ b/packages/pieces/community/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@activepieces/piece-firecrawl",- "version": "0.4.1",+ "version": "0.5.0","main": "./dist/src/index.js","types": "./dist/src/index.d.ts","scripts": {diff --git a/packages/pieces/community/firecrawl/src/i18n/translation.json b/packages/pieces/community/firecrawl/src/i18n/translation.jsonindex 4d4ceeffe10..0a4fc165c54 100644--- a/packages/pieces/community/firecrawl/src/i18n/translation.json+++ b/packages/pieces/community/firecrawl/src/i18n/translation.json@@ -1,6 +1,6 @@{- "Extract structured data from websites using AI with natural language prompts": "Extract structured data from websites using AI with natural language prompts",- "\nFollow these steps to obtain your Firecrawl API Key:\n\n1. Visit [Firecrawl](https://firecrawl.dev) and create an account.\n2. Log in and navigate to your dashboard.\n3. Locate and copy your API key from the API settings section.\n": "\nFollow these steps to obtain your Firecrawl API Key:\n\n1. Visit [Firecrawl](https://firecrawl.dev) and create an account.\n2. Log in and navigate to your dashboard.\n3. Locate and copy your API key from the API settings section.\n",+ "The context API to search, scrape, and interact with the web at scale.": "The context API to search, scrape, and interact with the web at scale.",+ "\nFollow these steps to obtain your Firecrawl API Key:\n\n1. Sign in or create an account at [Firecrawl](https://www.firecrawl.dev/app/api-keys?utm_source=activepieces&utm_medium=integration).\n2. Copy your API key from the API Keys page.\n": "\nFollow these steps to obtain your Firecrawl API Key:\n\n1. Sign in or create an account at [Firecrawl](https://www.firecrawl.dev/app/api-keys?utm_source=activepieces&utm_medium=integration).\n2. Copy your API key from the API Keys page.\n","Scrape Website": "Scrape Website","Extract Structured Data": "Extract Structured Data","Crawl": "Crawl",@@ -39,7 +39,7 @@"List the URLs that failed or were blocked in a crawl job.": "List the URLs that failed or were blocked in a crawl job.","Cancel an in-flight crawl job by its ID.": "Cancel an in-flight crawl job by its ID.","List the account's in-flight crawl jobs.": "List the account's in-flight crawl jobs.",- "Search the web for a keyword query and return matching result URLs.": "Search the web for a keyword query and return matching result URLs.",+ "Search the web, news, or images for a query. Optionally include each result page as markdown.": "Search the web, news, or images for a query. Optionally include each result page as markdown.","Start an autonomous Firecrawl (FIRE-1) agent job from a natural-language prompt.": "Start an autonomous Firecrawl (FIRE-1) agent job from a natural-language prompt.","Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.": "Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.","Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint",@@ -74,6 +74,10 @@"Query": "Query","Language": "Language","Country": "Country",+ "Sources": "Sources",+ "Time Filter": "Time Filter",+ "Location": "Location",+ "Include Page Content": "Include Page Content","Result Schema": "Result Schema","Max Credits": "Max Credits","Strictly Constrain to URLs": "Strictly Constrain to URLs",@@ -83,9 +87,9 @@"Query Parameters": "Query Parameters","Body Type": "Body Type","Body": "Body",- "Response is Binary ?": "Response is Binary ?",- "No Error on Failure": "No Error on Failure",- "Timeout (in seconds)": "Timeout (in seconds)",+ "Response is Binary": "Response is Binary",+ "Return Error as Output": "Return Error as Output",+ "Timeout": "Timeout","Follow redirects": "Follow redirects","The webpage URL to scrape.": "The webpage URL to scrape.","Maximum time to wait for the page to load (in milliseconds).": "Maximum time to wait for the page to load (in milliseconds).",@@ -136,10 +140,14 @@"The ID of the crawl job whose errors to list. Obtain it from the Crawl Website action result or List Active Crawls.": "The ID of the crawl job whose errors to list. Obtain it from the Crawl Website action result or List Active Crawls.","The ID of the crawl job to cancel. Obtain it from the Crawl Website action result or List Active Crawls.": "The ID of the crawl job to cancel. Obtain it from the Crawl Website action result or List Active Crawls.","The search query (keywords, like a Google search).": "The search query (keywords, like a Google search).",- "Maximum number of results to return.": "Maximum number of results to return.",+ "Maximum number of results to return for each source (max 100).": "Maximum number of results to return for each source (max 100).","Optional ISO language code to bias results (e.g. \"en\").": "Optional ISO language code to bias results (e.g. \"en\").","Optional ISO country code to bias results (e.g. \"us\").": "Optional ISO country code to bias results (e.g. \"us\").",- "Maximum time to wait for the search, in milliseconds.": "Maximum time to wait for the search, in milliseconds.",+ "Maximum time to wait for the search, in milliseconds. Allow more time when Include Page Content is on.": "Maximum time to wait for the search, in milliseconds. Allow more time when Include Page Content is on.",+ "Where to search: \"web\", \"news\" or \"images\". Defaults to web only.": "Where to search: \"web\", \"news\" or \"images\". Defaults to web only.",+ "Only return results from this time range: qdr:h (past hour), qdr:d (past day), qdr:w (past week), qdr:m (past month) or qdr:y (past year). Leave empty for any time.": "Only return results from this time range: qdr:h (past hour), qdr:d (past day), qdr:w (past week), qdr:m (past month) or qdr:y (past year). Leave empty for any time.",+ "Optional location to search from (e.g. \"San Francisco,California,United States\").": "Optional location to search from (e.g. \"San Francisco,California,United States\").",+ "Also return each result page as markdown (for image results, the page the image is on). Each result with page content uses scrape credits.": "Also return each result page as markdown (for image results, the page the image is on). Each result with page content uses scrape credits.","Natural-language instruction describing what the agent should accomplish.": "Natural-language instruction describing what the agent should accomplish.","Optional starting URLs for the agent. Obtain them from Search Web or Map Website if needed.": "Optional starting URLs for the agent. Obtain them from Search Web or Map Website if needed.","Optional JSON Schema describing the structured result to produce.": "Optional JSON Schema describing the structured result to produce.",@@ -147,7 +155,11 @@"If enabled, the agent may only visit the provided URLs and will not navigate elsewhere.": "If enabled, the agent may only visit the provided URLs and will not navigate elsewhere.","The ID of the agent job to check. Obtain it from the Start Agent action result.": "The ID of the agent job to check. Obtain it from the Start Agent action result.","Authorization headers are injected automatically from your connection.": "Authorization headers are injected automatically from your connection.",+ "Appended to the URL as ?key=value.": "Appended to the URL as ?key=value.","Enable for files like PDFs, images, etc.": "Enable for files like PDFs, images, etc.",+ "On a failed request, output the error instead of failing the step.": "On a failed request, output the error instead of failing the step.",+ "Seconds to wait for a response. Empty: up to the flow limit (10 min).": "Seconds to wait for a response. Empty: up to the flow limit (10 min).",+ "Follow 3xx redirects instead of returning them as the response.": "Follow 3xx redirects instead of returning them as the response.","Simple": "Simple","Advanced": "Advanced","Markdown": "Markdown",@@ -156,14 +168,18 @@"Links": "Links","Summary": "Summary","Screenshot": "Screenshot",+ "Web": "Web",+ "News": "News",+ "Images": "Images",+ "Past hour": "Past hour",+ "Past day": "Past day",+ "Past week": "Past week",+ "Past month": "Past month",+ "Past year": "Past year","GET": "GET","POST": "POST","PATCH": "PATCH","PUT": "PUT","DELETE": "DELETE",- "HEAD": "HEAD",- "None": "None",- "JSON": "JSON",- "Form Data": "Form Data",- "Raw": "Raw"+ "HEAD": "HEAD"}\ No newline at end of filediff --git a/packages/pieces/community/firecrawl/src/index.ts b/packages/pieces/community/firecrawl/src/index.tsindex 46bf9998d29..223f109b47c 100644--- a/packages/pieces/community/firecrawl/src/index.ts+++ b/packages/pieces/community/firecrawl/src/index.ts@@ -27,7 +27,7 @@ import { firecrawlAuth } from './lib/auth';export const firecrawl = createPiece({displayName: 'Firecrawl',- description: 'Extract structured data from websites using AI with natural language prompts',+ description: 'The context API to search, scrape, and interact with the web at scale.',minimumSupportedRelease: '0.86.4',logoUrl: 'https://cdn.activepieces.com/pieces/firecrawl.png',categories: [PieceCategory.ARTIFICIAL_INTELLIGENCE],diff --git a/packages/pieces/community/firecrawl/src/lib/actions/batch-scrape.ts b/packages/pieces/community/firecrawl/src/lib/actions/batch-scrape.tsindex c59867c8d89..5d92b6d3261 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/batch-scrape.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/batch-scrape.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { FIRECRAWL_API_BASE_URL } from '../common/common';+import { FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { batchScrapeActionOutputSchema } from '../output-schemas';export const batchScrape = createAction({@@ -63,6 +63,7 @@ export const batchScrape = createAction({const body: Record<string, any> = {urls: urlsArray,formats: [format],+ origin: FIRECRAWL_ORIGIN,};if (propsValue.onlyMainContent !== undefined) {body['onlyMainContent'] = propsValue.onlyMainContent;diff --git a/packages/pieces/community/firecrawl/src/lib/actions/crawl-website.ts b/packages/pieces/community/firecrawl/src/lib/actions/crawl-website.tsindex 4aae34eeeed..6342f51a9f1 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/crawl-website.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/crawl-website.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { polling, FIRECRAWL_API_BASE_URL } from '../common/common';+import { polling, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { crawlWebsiteActionOutputSchema } from '../output-schemas';export const crawlWebsite = createAction({@@ -75,6 +75,7 @@ export const crawlWebsite = createAction({crawlEntireDomain: false,maxDiscoveryDepth: 10,scrapeOptions,+ origin: FIRECRAWL_ORIGIN,};if (propsValue.limit !== undefined) {body['limit'] = propsValue.limit;diff --git a/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts b/packages/pieces/community/firecrawl/src/lib/actions/crawl.tsindex 5173c11da1a..e3df594532d 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts@@ -1,7 +1,7 @@import { createAction, Property, InputPropertyMap } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { forScreenshotOutputFormat, forSimpleOutputFormat, forJsonOutputFormat, polling, downloadAndSaveCrawlScreenshots, FIRECRAWL_API_BASE_URL } from '../common/common';+import { forScreenshotOutputFormat, forSimpleOutputFormat, forJsonOutputFormat, polling, downloadAndSaveCrawlScreenshots, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { crawlWebsiteActionOutputSchema } from '../output-schemas';function webhookConfig(useWebhook: boolean, webhookProperties: any): any {@@ -254,6 +254,7 @@ export const crawl = createAction({sitemap: "include",crawlEntireDomain: false,maxDiscoveryDepth: 10,+ origin: FIRECRAWL_ORIGIN,};if (propsValue.limit !== undefined) {diff --git a/packages/pieces/community/firecrawl/src/lib/actions/extract-data.ts b/packages/pieces/community/firecrawl/src/lib/actions/extract-data.tsindex d50ddca355a..e69f884ee04 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/extract-data.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/extract-data.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { polling, FIRECRAWL_API_BASE_URL } from '../common/common';+import { polling, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { extractDataActionOutputSchema } from '../output-schemas';export const extractData = createAction({@@ -61,6 +61,7 @@ export const extractData = createAction({const body: Record<string, any> = {urls: urlsArray,schema: propsValue.schema,+ origin: FIRECRAWL_ORIGIN,};if (propsValue.prompt) {body['prompt'] = propsValue.prompt;diff --git a/packages/pieces/community/firecrawl/src/lib/actions/extract.ts b/packages/pieces/community/firecrawl/src/lib/actions/extract.tsindex 3d9759a7e46..e138c385f26 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/extract.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/extract.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { forJsonOutputFormat, polling, FIRECRAWL_API_BASE_URL } from '../common/common';+import { forJsonOutputFormat, polling, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { extractDataActionOutputSchema } from '../output-schemas';export const extract = createAction({@@ -146,6 +146,7 @@ export const extract = createAction({urls: urlsArray,prompt: jsonFormat.prompt,schema: jsonFormat.schema,+ origin: FIRECRAWL_ORIGIN,};if (propsValue.enableWebSearch) {diff --git a/packages/pieces/community/firecrawl/src/lib/actions/map-website.ts b/packages/pieces/community/firecrawl/src/lib/actions/map-website.tsindex c9091a776b2..928d48beb00 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/map-website.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/map-website.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { FIRECRAWL_API_BASE_URL } from '../common/common';+import { FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { mapWebsiteActionOutputSchema } from '../output-schemas';export const mapWebsite = createAction({@@ -47,6 +47,7 @@ export const mapWebsite = createAction({sitemap: 'include',includeSubdomains: propsValue.includeSubdomains,limit: propsValue.limit,+ origin: FIRECRAWL_ORIGIN,};if (propsValue.search) {body['search'] = propsValue.search;diff --git a/packages/pieces/community/firecrawl/src/lib/actions/map.ts b/packages/pieces/community/firecrawl/src/lib/actions/map.tsindex f6895f47ad0..8020166eb11 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/map.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/map.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { FIRECRAWL_API_BASE_URL } from '../common/common';+import { FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { mapWebsiteActionOutputSchema } from '../output-schemas';export const map = createAction({@@ -40,6 +40,7 @@ export const map = createAction({sitemap: 'include',includeSubdomains: propsValue.subdomain,limit: propsValue.limit,+ origin: FIRECRAWL_ORIGIN,};const response = await httpClient.sendRequest({diff --git a/packages/pieces/community/firecrawl/src/lib/actions/scrape-url.ts b/packages/pieces/community/firecrawl/src/lib/actions/scrape-url.tsindex 3dbc5ea9de5..49983906769 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/scrape-url.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/scrape-url.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { downloadAndSaveScreenshot, FIRECRAWL_API_BASE_URL } from '../common/common';+import { downloadAndSaveScreenshot, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { scrapeUrlActionOutputSchema } from '../output-schemas';export const scrapeUrl = createAction({@@ -68,6 +68,7 @@ export const scrapeUrl = createAction({url: propsValue.url,formats: formatsArray,timeout: propsValue.timeout,+ origin: FIRECRAWL_ORIGIN,};if (propsValue.onlyMainContent !== undefined) {body['onlyMainContent'] = propsValue.onlyMainContent;diff --git a/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts b/packages/pieces/community/firecrawl/src/lib/actions/scrape.tsindex 2478db9031e..055ad731ad9 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts@@ -1,7 +1,7 @@import { createAction, Property, InputPropertyMap } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { forScreenshotOutputFormat, forSimpleOutputFormat, downloadAndSaveScreenshot, forJsonOutputFormat, FIRECRAWL_API_BASE_URL } from '../common/common';+import { forScreenshotOutputFormat, forSimpleOutputFormat, downloadAndSaveScreenshot, forJsonOutputFormat, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { scrapeUrlActionOutputSchema } from '../output-schemas';function forDefaultScreenshot(): any {@@ -237,6 +237,7 @@ export const scrape = createAction({const { auth, propsValue } = context;const body: Record<string, any> = {url: propsValue.url,+ origin: FIRECRAWL_ORIGIN,timeout: propsValue.timeout,};diff --git a/packages/pieces/community/firecrawl/src/lib/actions/search-web.ts b/packages/pieces/community/firecrawl/src/lib/actions/search-web.tsindex 8a1c2247760..fed91e85d34 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/search-web.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/search-web.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { FIRECRAWL_API_BASE_URL } from '../common/common';+import { FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { searchWebActionOutputSchema } from '../output-schemas';export const searchWeb = createAction({@@ -9,12 +9,12 @@ export const searchWeb = createAction({name: 'search_web',classification: 'SEARCH',displayName: 'Search Web',- description: 'Search the web for a keyword query and return matching result URLs.',+ description: 'Search the web, news, or images for a query. Optionally include each result page as markdown.',audience: 'ai',outputSchema: searchWebActionOutputSchema,aiMetadata: {description:- 'Runs a keyword web search (a SERP wrapper, not semantic/embedding search) and returns the matching result URLs and snippets. Pick this as the entry point when you have a query but no URL yet; then feed a chosen URL into Scrape URL, Crawl Website, Map Website, or Extract Data. Read-only, so repeating the call is safe.',+ 'Searches the web, and optionally news and images, for a query; the entry point when you have a query but no URL yet. Set includePageContent to true to also get each result page as markdown in the same call instead of following up with Scrape URL (each page uses scrape credits). Read-only, so repeating the call is safe.',idempotent: true,},props: {@@ -25,7 +25,7 @@ export const searchWeb = createAction({}),limit: Property.Number({displayName: 'Limit',- description: 'Maximum number of results to return.',+ description: 'Maximum number of results to return for each source (max 100).',required: false,defaultValue: 5,}),@@ -41,13 +41,53 @@ export const searchWeb = createAction({}),timeout: Property.Number({displayName: 'Timeout (ms)',- description: 'Maximum time to wait for the search, in milliseconds.',+ description: 'Maximum time to wait for the search, in milliseconds. Allow more time when Include Page Content is on.',required: false,}),+ sources: Property.StaticMultiSelectDropdown({+ displayName: 'Sources',+ description: 'Where to search: "web", "news" or "images". Defaults to web only.',+ required: false,+ options: {+ disabled: false,+ options: [+ { label: 'Web', value: 'web' },+ { label: 'News', value: 'news' },+ { label: 'Images', value: 'images' },+ ],+ },+ }),+ timeFilter: Property.StaticDropdown({+ displayName: 'Time Filter',+ description: 'Only return results from this time range: qdr:h (past hour), qdr:d (past day), qdr:w (past week), qdr:m (past month) or qdr:y (past year). Leave empty for any time.',+ required: false,+ options: {+ disabled: false,+ options: [+ { label: 'Past hour', value: 'qdr:h' },+ { label: 'Past day', value: 'qdr:d' },+ { label: 'Past week', value: 'qdr:w' },+ { label: 'Past month', value: 'qdr:m' },+ { label: 'Past year', value: 'qdr:y' },+ ],+ },+ }),+ location: Property.ShortText({+ displayName: 'Location',+ description: 'Optional location to search from (e.g. "San Francisco,California,United States").',+ required: false,+ }),+ includePageContent: Property.Checkbox({+ displayName: 'Include Page Content',+ description: 'Also return each result page as markdown (for image results, the page the image is on). Each result with page content uses scrape credits.',+ required: false,+ defaultValue: false,+ }),},async run({ auth, propsValue }) {const body: Record<string, any> = {query: propsValue.query,+ origin: FIRECRAWL_ORIGIN,};if (propsValue.limit !== undefined) {body['limit'] = propsValue.limit;@@ -61,6 +101,18 @@ export const searchWeb = createAction({if (propsValue.timeout !== undefined) {body['timeout'] = propsValue.timeout;}+ if (propsValue.sources && propsValue.sources.length > 0) {+ body['sources'] = propsValue.sources.map((type) => ({ type }));+ }+ if (propsValue.timeFilter) {+ body['tbs'] = propsValue.timeFilter;+ }+ if (propsValue.location) {+ body['location'] = propsValue.location;+ }+ if (propsValue.includePageContent) {+ body['scrapeOptions'] = { formats: ['markdown'], onlyMainContent: true };+ }try {const response = await httpClient.sendRequest({@@ -75,8 +127,14 @@ export const searchWeb = createAction({return response.body;} catch (error: any) {const status = error?.response?.status;+ if (status === 401) {+ throw new Error('Invalid Firecrawl API key (401): check the key on your connection.');+ }+ if (status === 402) {+ throw new Error(`Firecrawl could not run this search (402): ${error?.response?.body?.error ?? 'not enough credits'}. Lower Limit or turn off Include Page Content, or top up at https://www.firecrawl.dev/pricing?utm_source=activepieces&utm_medium=integration.`);+ }if (status === 403) {- throw new Error('Firecrawl denied the request (403): your API key plan does not permit search or has insufficient credits.');+ throw new Error(`Firecrawl denied the request (403): ${error?.response?.body?.error ?? 'this API key is not allowed to run this search'}`);}if (status === 429) {throw new Error('Firecrawl rate limit reached (429): slow down requests or upgrade your plan, then retry.');diff --git a/packages/pieces/community/firecrawl/src/lib/actions/start-agent.ts b/packages/pieces/community/firecrawl/src/lib/actions/start-agent.tsindex 281b743eb8d..d3c8a8d5af4 100644--- a/packages/pieces/community/firecrawl/src/lib/actions/start-agent.ts+++ b/packages/pieces/community/firecrawl/src/lib/actions/start-agent.ts@@ -1,7 +1,7 @@import { createAction, Property } from '@activepieces/pieces-framework';import { httpClient, HttpMethod } from '@activepieces/pieces-common';import { firecrawlAuth } from '../auth';-import { FIRECRAWL_API_BASE_URL } from '../common/common';+import { FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';import { startAgentActionOutputSchema } from '../output-schemas';export const startAgent = createAction({@@ -48,6 +48,7 @@ export const startAgent = createAction({async run({ auth, propsValue }) {const body: Record<string, any> = {prompt: propsValue.prompt,+ origin: FIRECRAWL_ORIGIN,};if (propsValue.urls && (propsValue.urls as unknown[]).length > 0) {body['urls'] = (propsValue.urls as unknown[]).map((u) => String(u));diff --git a/packages/pieces/community/firecrawl/src/lib/auth.ts b/packages/pieces/community/firecrawl/src/lib/auth.tsindex 5f900506517..58fc46027a3 100644--- a/packages/pieces/community/firecrawl/src/lib/auth.ts+++ b/packages/pieces/community/firecrawl/src/lib/auth.ts@@ -5,9 +5,8 @@ import { FIRECRAWL_API_BASE_URL } from './common/common';const markdownDescription = `Follow these steps to obtain your Firecrawl API Key:-1. Visit [Firecrawl](https://firecrawl.dev) and create an account.-2. Log in and navigate to your dashboard.-3. Locate and copy your API key from the API settings section.+1. Sign in or create an account at [Firecrawl](https://www.firecrawl.dev/app/api-keys?utm_source=activepieces&utm_medium=integration).+2. Copy your API key from the API Keys page.`;export const firecrawlAuth = PieceAuth.SecretText({diff --git a/packages/pieces/community/firecrawl/src/lib/common/common.ts b/packages/pieces/community/firecrawl/src/lib/common/common.tsindex 6ee3cd63e6e..c34ff812df6 100644--- a/packages/pieces/community/firecrawl/src/lib/common/common.ts+++ b/packages/pieces/community/firecrawl/src/lib/common/common.ts@@ -157,4 +157,6 @@ export async function polling(// exit loop time outthrow new Error(`${actionType.charAt(0).toUpperCase()}. job timed out after ${timeoutSeconds} second(s)`);-}\ No newline at end of file+}++export const FIRECRAWL_ORIGIN = 'activepieces';diff --git a/packages/pieces/community/firecrawl/src/lib/output-schemas.ts b/packages/pieces/community/firecrawl/src/lib/output-schemas.tsindex 5c3d291cf53..bb1e01ba4cc 100644--- a/packages/pieces/community/firecrawl/src/lib/output-schemas.ts+++ b/packages/pieces/community/firecrawl/src/lib/output-schemas.ts@@ -126,6 +126,67 @@ export const searchWebActionOutputSchema: OutputSchema = {{ key: 'title', label: 'Title' },{ key: 'description', label: 'Description' },{ key: 'position', label: 'Position', format: 'number' },+ { key: 'markdown', label: 'Markdown' },+ {+ key: 'metadata',+ label: 'Metadata',+ children: [+ { key: 'title', label: 'Title' },+ { key: 'sourceURL', label: 'Source URL', format: 'url' },+ { key: 'url', label: 'Final URL', format: 'url' },+ { key: 'statusCode', label: 'Status Code', format: 'number' },+ ],+ },+ ],+ },+ {+ key: 'news',+ label: 'News Results',+ labelKey: 'title',+ listItems: [+ { key: 'url', label: 'URL', format: 'url' },+ { key: 'title', label: 'Title' },+ { key: 'snippet', label: 'Snippet' },+ { key: 'description', label: 'Description' },+ { key: 'date', label: 'Date' },+ { key: 'imageUrl', label: 'Image URL', format: 'image' },+ { key: 'position', label: 'Position', format: 'number' },+ { key: 'markdown', label: 'Markdown' },+ {+ key: 'metadata',+ label: 'Metadata',+ children: [+ { key: 'title', label: 'Title' },+ { key: 'sourceURL', label: 'Source URL', format: 'url' },+ { key: 'url', label: 'Final URL', format: 'url' },+ { key: 'statusCode', label: 'Status Code', format: 'number' },+ ],+ },+ ],+ },+ {+ key: 'images',+ label: 'Image Results',+ labelKey: 'title',+ listItems: [+ { key: 'title', label: 'Title' },+ { key: 'imageUrl', label: 'Image URL', format: 'image' },+ { key: 'imageWidth', label: 'Image Width', format: 'number' },+ { key: 'imageHeight', label: 'Image Height', format: 'number' },+ { key: 'url', label: 'Page URL', format: 'url' },+ { key: 'description', label: 'Description' },+ { key: 'position', label: 'Position', format: 'number' },+ { key: 'markdown', label: 'Markdown' },+ {+ key: 'metadata',+ label: 'Metadata',+ children: [+ { key: 'title', label: 'Title' },+ { key: 'sourceURL', label: 'Source URL', format: 'url' },+ { key: 'url', label: 'Final URL', format: 'url' },+ { key: 'statusCode', label: 'Status Code', format: 'number' },+ ],+ },],},],