Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

feat(firecrawl): ui improvement (#16601)

Talal Jaber539a4cb4e96f
diff --git a/bun.lock b/bun.lock
index 7bdeb4b7046..e2405fbf441 100644
--- a/bun.lock
+++ b/bun.lock
@@ -3248,7 +3248,7 @@
},
"packages/pieces/community/firecrawl": {
"name": "@activepieces/piece-firecrawl",
- "version": "0.6.1",
+ "version": "0.7.0",
"dependencies": {
"@activepieces/core-piece-types": "workspace:*",
"@activepieces/core-utils": "workspace:*",
@@ -3258,6 +3258,7 @@
},
"devDependencies": {
"tslib": "2.6.2",
+ "vitest": "3.2.6",
},
},
"packages/pieces/community/fireflies-ai": {
diff --git a/packages/pieces/community/firecrawl/package.json b/packages/pieces/community/firecrawl/package.json
index 0f88be671be..ee45ea2d5bf 100644
--- a/packages/pieces/community/firecrawl/package.json
+++ b/packages/pieces/community/firecrawl/package.json
@@ -1,12 +1,13 @@
{
"name": "@activepieces/piece-firecrawl",
- "version": "0.6.1",
+ "version": "0.7.0",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
"scripts": {
"build": "tsc -p tsconfig.lib.json && cp package.json dist/",
"bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle",
- "lint": "eslint 'src/**/*.ts'"
+ "lint": "eslint 'src/**/*.ts'",
+ "test": "vitest run"
},
"dependencies": {
"@activepieces/pieces-common": "workspace:*",
@@ -16,6 +17,7 @@
"@activepieces/core-utils": "workspace:*"
},
"devDependencies": {
- "tslib": "2.6.2"
+ "tslib": "2.6.2",
+ "vitest": "3.2.6"
}
}
diff --git a/packages/pieces/community/firecrawl/src/i18n/de.json b/packages/pieces/community/firecrawl/src/i18n/de.json
index 85cf617468d..1b9700f676b 100644
--- a/packages/pieces/community/firecrawl/src/i18n/de.json
+++ b/packages/pieces/community/firecrawl/src/i18n/de.json
@@ -9,7 +9,7 @@
"Scrape URL": "Scrape URL",
"Crawl Website": "Crawl Website",
"Get Crawl Results": "Get Crawl Results",
- "Map Website": "Map Website",
+ "Map Website": "Website kartieren",
"Extract Data": "Extract Data",
"Batch Scrape": "Batch Scrape",
"Get Batch Scrape Results": "Get Batch Scrape Results",
@@ -67,7 +67,7 @@
"Include subdomain": "Subdomain einbeziehen",
"Page Limit": "Page Limit",
"Search Filter": "Search Filter",
- "Include Subdomains": "Include Subdomains",
+ "Include Subdomains": "Subdomains einbeziehen",
"JSON Schema": "JSON Schema",
"Max Concurrency": "Max Concurrency",
"Per-Page Timeout (ms)": "Per-Page Timeout (ms)",
@@ -183,5 +183,39 @@
"PATCH": "PATCH",
"PUT": "PUT",
"DELETE": "LÖSCHEN",
- "HEAD": "HEAD"
+ "HEAD": "HEAD",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "Eine Webseite als Markdown, HTML, Links, Screenshot oder JSON lesen.",
+ "Pull structured data from one or more pages with AI.": "Strukturierte Daten aus einer oder mehreren Seiten per KI abrufen.",
+ "Read many pages of a site by following links from a start URL.": "Viele Seiten einer Website lesen, indem Links ab einer Start-URL gefolgt wird.",
+ "Get the status and pages of a crawl started earlier.": "Status und Seiten eines zuvor gestarteten Crawls abrufen.",
+ "List the links found on a website.": "Die Links einer Website auflisten.",
+ "Run Browser Actions": "Browser-Aktionen ausführen",
+ "Use Cache": "Cache verwenden",
+ "Start URL": "Start-URL",
+ "Pages to Crawl": "Zu crawlende Seiten",
+ "Max Pages": "Max. Seiten",
+ "Send Results to a Webhook": "Ergebnisse an einen Webhook senden",
+ "Max Links": "Max. Links",
+ "The page to read.": "Die zu lesende Seite.",
+ "Click, type or wait on the page before it is read.": "Vor dem Lesen auf der Seite klicken, tippen oder warten.",
+ "A full-page screenshot is also saved with every scrape.": "Bei jedem Scrape wird zusätzlich ein ganzseitiger Screenshot gespeichert.",
+ "What the step returns for the page.": "Was der Schritt für die Seite zurückgibt.",
+ "Milliseconds to wait for the page to load.": "Millisekunden, die auf das Laden der Seite gewartet wird.",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "An: Firecrawl speichert die Seite und liefert ggf. eine Kopie aus dem Cache.",
+ "Pages to read. End a URL with /* to cover a whole site.": "Zu lesende Seiten. Mit /* am Ende wird die ganze Website erfasst.",
+ "Also search the web for related pages and context.": "Zusätzlich im Web nach verwandten Seiten und Kontext suchen.",
+ "What to pull from the page, in plain words.": "Was aus der Seite geholt werden soll, in einfachen Worten.",
+ "Seconds to wait. The job keeps running after a timeout.": "Wartezeit in Sekunden. Der Job läuft nach einem Timeout weiter.",
+ "The crawl follows links from this page.": "Der Crawl folgt den Links ab dieser Seite.",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "Welche Seiten gelesen werden, in einfachen Worten. Leer: Seiten unterhalb der Start-URL.",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "Höchstzahl zu lesender Seiten. Leer: Firecrawl-Limit von 10.000.",
+ "Seconds to wait. The crawl keeps running after a timeout.": "Wartezeit in Sekunden. Der Crawl läuft nach einem Timeout weiter.",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "In den letzten 2 Tagen gelesene Seiten können aus dem Firecrawl-Cache stammen.",
+ "What the step returns for each page.": "Was der Schritt für jede Seite zurückgibt.",
+ "Skip headers, menus and footers.": "Kopfzeilen, Menüs und Fußzeilen überspringen.",
+ "Firecrawl posts each event to your URL as the crawl runs.": "Firecrawl sendet jedes Ereignis während des Crawls an Ihre URL.",
+ "The id in the webhook events of a Crawl step.": "Die id aus den Webhook-Ereignissen eines Crawl-Schritts.",
+ "The site to list links from.": "Die Website, deren Links aufgelistet werden.",
+ "Also list pages on subdomains, like blog.example.com.": "Auch Seiten auf Subdomains auflisten, z. B. blog.example.com.",
+ "Most links to return, up to 100,000.": "Höchstzahl zurückgegebener Links, bis zu 100.000."
}
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/i18n/es.json b/packages/pieces/community/firecrawl/src/i18n/es.json
index f980b3da6b3..380d6a4d6b3 100644
--- a/packages/pieces/community/firecrawl/src/i18n/es.json
+++ b/packages/pieces/community/firecrawl/src/i18n/es.json
@@ -9,7 +9,7 @@
"Scrape URL": "Scrape URL",
"Crawl Website": "Crawl Website",
"Get Crawl Results": "Get Crawl Results",
- "Map Website": "Map Website",
+ "Map Website": "Mapear sitio web",
"Extract Data": "Extract Data",
"Batch Scrape": "Batch Scrape",
"Get Batch Scrape Results": "Get Batch Scrape Results",
@@ -67,7 +67,7 @@
"Include subdomain": "Incluye subdominio",
"Page Limit": "Page Limit",
"Search Filter": "Search Filter",
- "Include Subdomains": "Include Subdomains",
+ "Include Subdomains": "Incluir subdominios",
"JSON Schema": "JSON Schema",
"Max Concurrency": "Max Concurrency",
"Per-Page Timeout (ms)": "Per-Page Timeout (ms)",
@@ -183,5 +183,39 @@
"PATCH": "PATCH",
"PUT": "PUT",
"DELETE": "BORRAR",
- "HEAD": "LIMPIO"
+ "HEAD": "LIMPIO",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "Lee una página web como Markdown, HTML, enlaces, captura o JSON.",
+ "Pull structured data from one or more pages with AI.": "Extrae datos estructurados de una o varias páginas con IA.",
+ "Read many pages of a site by following links from a start URL.": "Lee muchas páginas de un sitio siguiendo enlaces desde una URL inicial.",
+ "Get the status and pages of a crawl started earlier.": "Obtén el estado y las páginas de un rastreo iniciado antes.",
+ "List the links found on a website.": "Lista los enlaces encontrados en un sitio web.",
+ "Run Browser Actions": "Ejecutar acciones del navegador",
+ "Use Cache": "Usar caché",
+ "Start URL": "URL inicial",
+ "Pages to Crawl": "Páginas a rastrear",
+ "Max Pages": "Máx. páginas",
+ "Send Results to a Webhook": "Enviar resultados a un webhook",
+ "Max Links": "Máx. enlaces",
+ "The page to read.": "La página que se leerá.",
+ "Click, type or wait on the page before it is read.": "Haz clic, escribe o espera en la página antes de leerla.",
+ "A full-page screenshot is also saved with every scrape.": "Cada scrape también guarda una captura de la página completa.",
+ "What the step returns for the page.": "Lo que devuelve el paso para la página.",
+ "Milliseconds to wait for the page to load.": "Milisegundos de espera para que cargue la página.",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "Activado: Firecrawl guarda la página y puede servir una copia en caché.",
+ "Pages to read. End a URL with /* to cover a whole site.": "Páginas a leer. Termina una URL con /* para cubrir todo el sitio.",
+ "Also search the web for related pages and context.": "Busca también en la web páginas y contexto relacionados.",
+ "What to pull from the page, in plain words.": "Qué extraer de la página, en palabras sencillas.",
+ "Seconds to wait. The job keeps running after a timeout.": "Segundos de espera. La tarea sigue en marcha tras el tiempo límite.",
+ "The crawl follows links from this page.": "El rastreo sigue los enlaces desde esta página.",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "Qué páginas leer, en palabras sencillas. Vacío: páginas bajo la URL inicial.",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "Máximo de páginas a leer. Vacío: límite de Firecrawl de 10.000.",
+ "Seconds to wait. The crawl keeps running after a timeout.": "Segundos de espera. El rastreo sigue en marcha tras el tiempo límite.",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "Las páginas leídas en los últimos 2 días pueden venir de la caché de Firecrawl.",
+ "What the step returns for each page.": "Lo que devuelve el paso para cada página.",
+ "Skip headers, menus and footers.": "Omite encabezados, menús y pies de página.",
+ "Firecrawl posts each event to your URL as the crawl runs.": "Firecrawl envía cada evento a tu URL mientras dura el rastreo.",
+ "The id in the webhook events of a Crawl step.": "El id en los eventos de webhook de un paso Crawl.",
+ "The site to list links from.": "El sitio del que se listan los enlaces.",
+ "Also list pages on subdomains, like blog.example.com.": "Lista también páginas en subdominios, como blog.example.com.",
+ "Most links to return, up to 100,000.": "Máximo de enlaces a devolver, hasta 100.000."
}
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/i18n/fr.json b/packages/pieces/community/firecrawl/src/i18n/fr.json
index d83caf022d2..896c3f36aba 100644
--- a/packages/pieces/community/firecrawl/src/i18n/fr.json
+++ b/packages/pieces/community/firecrawl/src/i18n/fr.json
@@ -9,7 +9,7 @@
"Scrape URL": "Scrape URL",
"Crawl Website": "Site Web de Crawl",
"Get Crawl Results": "Get Crawl Results",
- "Map Website": "Map Website",
+ "Map Website": "Cartographier un site web",
"Extract Data": "Extract Data",
"Batch Scrape": "Batch Scrape",
"Get Batch Scrape Results": "Get Batch Scrape Results",
@@ -67,7 +67,7 @@
"Include subdomain": "Inclure le sous-domaine",
"Page Limit": "Limite de page",
"Search Filter": "Filtre de recherche",
- "Include Subdomains": "Include Subdomains",
+ "Include Subdomains": "Inclure les sous-domaines",
"JSON Schema": "JSON Schema",
"Max Concurrency": "Max Concurrency",
"Per-Page Timeout (ms)": "Per-Page Timeout (ms)",
@@ -183,5 +183,39 @@
"PATCH": "PATCH",
"PUT": "EFFACER",
"DELETE": "SUPPRIMER",
- "HEAD": "TÊTE"
+ "HEAD": "TÊTE",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "Lire une page web en Markdown, HTML, liens, capture ou JSON.",
+ "Pull structured data from one or more pages with AI.": "Extraire des données structurées d'une ou plusieurs pages avec l'IA.",
+ "Read many pages of a site by following links from a start URL.": "Lire plusieurs pages d'un site en suivant les liens depuis une URL de départ.",
+ "Get the status and pages of a crawl started earlier.": "Obtenir l'état et les pages d'un crawl lancé auparavant.",
+ "List the links found on a website.": "Lister les liens trouvés sur un site web.",
+ "Run Browser Actions": "Exécuter des actions navigateur",
+ "Use Cache": "Utiliser le cache",
+ "Start URL": "URL de départ",
+ "Pages to Crawl": "Pages à parcourir",
+ "Max Pages": "Pages max.",
+ "Send Results to a Webhook": "Envoyer les résultats à un webhook",
+ "Max Links": "Liens max.",
+ "The page to read.": "La page à lire.",
+ "Click, type or wait on the page before it is read.": "Cliquer, saisir ou attendre sur la page avant sa lecture.",
+ "A full-page screenshot is also saved with every scrape.": "Une capture pleine page est aussi enregistrée à chaque scrape.",
+ "What the step returns for the page.": "Ce que l'étape renvoie pour la page.",
+ "Milliseconds to wait for the page to load.": "Millisecondes à attendre le chargement de la page.",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "Activé : Firecrawl conserve la page et peut servir une copie en cache.",
+ "Pages to read. End a URL with /* to cover a whole site.": "Pages à lire. Terminez une URL par /* pour couvrir tout le site.",
+ "Also search the web for related pages and context.": "Chercher aussi sur le web des pages et du contexte liés.",
+ "What to pull from the page, in plain words.": "Ce que vous voulez tirer de la page, en mots simples.",
+ "Seconds to wait. The job keeps running after a timeout.": "Secondes d'attente. La tâche continue après le délai.",
+ "The crawl follows links from this page.": "Le crawl suit les liens à partir de cette page.",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "Quelles pages lire, en mots simples. Vide : les pages sous l'URL de départ.",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "Nombre max. de pages à lire. Vide : limite Firecrawl de 10 000.",
+ "Seconds to wait. The crawl keeps running after a timeout.": "Secondes d'attente. Le crawl continue après le délai.",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "Les pages lues ces 2 derniers jours peuvent venir du cache de Firecrawl.",
+ "What the step returns for each page.": "Ce que l'étape renvoie pour chaque page.",
+ "Skip headers, menus and footers.": "Ignorer en-têtes, menus et pieds de page.",
+ "Firecrawl posts each event to your URL as the crawl runs.": "Firecrawl envoie chaque événement à votre URL pendant le crawl.",
+ "The id in the webhook events of a Crawl step.": "L'id présent dans les événements webhook d'une étape Crawl.",
+ "The site to list links from.": "Le site dont lister les liens.",
+ "Also list pages on subdomains, like blog.example.com.": "Lister aussi les pages des sous-domaines, comme blog.example.com.",
+ "Most links to return, up to 100,000.": "Nombre max. de liens renvoyés, jusqu'à 100 000."
}
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/i18n/ja.json b/packages/pieces/community/firecrawl/src/i18n/ja.json
index a44bdbe4f51..e45aa23da31 100644
--- a/packages/pieces/community/firecrawl/src/i18n/ja.json
+++ b/packages/pieces/community/firecrawl/src/i18n/ja.json
@@ -9,7 +9,7 @@
"Scrape URL": "Scrape URL",
"Crawl Website": "Crawl Website",
"Get Crawl Results": "Get Crawl Results",
- "Map Website": "Map Website",
+ "Map Website": "ウェブサイトをマップ",
"Extract Data": "Extract Data",
"Batch Scrape": "Batch Scrape",
"Get Batch Scrape Results": "Get Batch Scrape Results",
@@ -43,7 +43,7 @@
"Start an autonomous Firecrawl (FIRE-1) agent job from a natural-language prompt.": "Start an autonomous Firecrawl (FIRE-1) agent job from a natural-language prompt.",
"Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.": "Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.",
"Make a custom API call to a specific endpoint": "特定のエンドポイントへのカスタム API コールを実行します。",
- "Website URL": "Website URL",
+ "Website URL": "ウェブサイト URL",
"Timeout (ms)": "タイムアウト (ミリ秒)",
"Store In Cache": "Store In Cache",
"Perform Actions Before Scraping": "スクラップする前にアクションを実行",
@@ -67,7 +67,7 @@
"Include subdomain": "サブドメインを含める",
"Page Limit": "Page Limit",
"Search Filter": "Search Filter",
- "Include Subdomains": "Include Subdomains",
+ "Include Subdomains": "サブドメインを含める",
"JSON Schema": "JSON Schema",
"Max Concurrency": "Max Concurrency",
"Per-Page Timeout (ms)": "Per-Page Timeout (ms)",
@@ -183,5 +183,39 @@
"PATCH": "PATCH",
"PUT": "PUT",
"DELETE": "削除",
- "HEAD": "頭"
+ "HEAD": "頭",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "1つのウェブページを Markdown、HTML、リンク、スクリーンショット、JSON で読み取ります。",
+ "Pull structured data from one or more pages with AI.": "AI で1つ以上のページから構造化データを取り出します。",
+ "Read many pages of a site by following links from a start URL.": "開始 URL からリンクをたどり、サイトの多くのページを読み取ります。",
+ "Get the status and pages of a crawl started earlier.": "以前に開始したクロールの状態とページを取得します。",
+ "List the links found on a website.": "ウェブサイト内で見つかったリンクを一覧にします。",
+ "Run Browser Actions": "ブラウザ操作を実行",
+ "Use Cache": "キャッシュを使用",
+ "Start URL": "開始 URL",
+ "Pages to Crawl": "クロールするページ",
+ "Max Pages": "最大ページ数",
+ "Send Results to a Webhook": "結果を Webhook に送信",
+ "Max Links": "最大リンク数",
+ "The page to read.": "読み取るページ。",
+ "Click, type or wait on the page before it is read.": "読み取る前にページでクリック、入力、待機を行います。",
+ "A full-page screenshot is also saved with every scrape.": "スクレイプのたびにページ全体のスクリーンショットも保存されます。",
+ "What the step returns for the page.": "このステップがページについて返す内容。",
+ "Milliseconds to wait for the page to load.": "ページの読み込みを待つミリ秒数。",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "オン: Firecrawl がページを保存し、キャッシュのコピーを返す場合があります。",
+ "Pages to read. End a URL with /* to cover a whole site.": "読み取るページ。URL の末尾を /* にするとサイト全体が対象になります。",
+ "Also search the web for related pages and context.": "関連するページや文脈をウェブでも検索します。",
+ "What to pull from the page, in plain words.": "ページから取り出す内容を、わかりやすい言葉で。",
+ "Seconds to wait. The job keeps running after a timeout.": "待機する秒数。タイムアウト後もジョブは実行を続けます。",
+ "The crawl follows links from this page.": "クロールはこのページからリンクをたどります。",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "読み取るページを、わかりやすい言葉で。空欄: 開始 URL 配下のページ。",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "読み取る最大ページ数。空欄: Firecrawl の上限 10,000。",
+ "Seconds to wait. The crawl keeps running after a timeout.": "待機する秒数。タイムアウト後もクロールは実行を続けます。",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "過去2日以内に読み取ったページは Firecrawl のキャッシュから返される場合があります。",
+ "What the step returns for each page.": "このステップが各ページについて返す内容。",
+ "Skip headers, menus and footers.": "ヘッダー、メニュー、フッターを除外します。",
+ "Firecrawl posts each event to your URL as the crawl runs.": "クロール中、Firecrawl が各イベントをあなたの URL に送信します。",
+ "The id in the webhook events of a Crawl step.": "Crawl ステップの Webhook イベントに含まれる id。",
+ "The site to list links from.": "リンクを一覧にするサイト。",
+ "Also list pages on subdomains, like blog.example.com.": "blog.example.com のようなサブドメインのページも一覧にします。",
+ "Most links to return, up to 100,000.": "返す最大リンク数 (最大 100,000)。"
}
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/i18n/nl.json b/packages/pieces/community/firecrawl/src/i18n/nl.json
index 0df1d5edf28..54c07beb167 100644
--- a/packages/pieces/community/firecrawl/src/i18n/nl.json
+++ b/packages/pieces/community/firecrawl/src/i18n/nl.json
@@ -9,7 +9,7 @@
"Scrape URL": "Scrape URL",
"Crawl Website": "Crawl Website",
"Get Crawl Results": "Get Crawl Results",
- "Map Website": "Map Website",
+ "Map Website": "Website in kaart brengen",
"Extract Data": "Extract Data",
"Batch Scrape": "Batch Scrape",
"Get Batch Scrape Results": "Get Batch Scrape Results",
@@ -43,7 +43,7 @@
"Start an autonomous Firecrawl (FIRE-1) agent job from a natural-language prompt.": "Start an autonomous Firecrawl (FIRE-1) agent job from a natural-language prompt.",
"Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.": "Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.",
"Make a custom API call to a specific endpoint": "Maak een aangepaste API call naar een specifiek eindpunt",
- "Website URL": "Website URL",
+ "Website URL": "Website-URL",
"Timeout (ms)": "Time-out (ms)",
"Store In Cache": "Store In Cache",
"Perform Actions Before Scraping": "Acties uitvoeren voor scraping",
@@ -67,7 +67,7 @@
"Include subdomain": "Inclusief subdomein",
"Page Limit": "Page Limit",
"Search Filter": "Search Filter",
- "Include Subdomains": "Include Subdomains",
+ "Include Subdomains": "Subdomeinen meenemen",
"JSON Schema": "JSON Schema",
"Max Concurrency": "Max Concurrency",
"Per-Page Timeout (ms)": "Per-Page Timeout (ms)",
@@ -183,5 +183,39 @@
"PATCH": "BEKIJK",
"PUT": "PUT",
"DELETE": "VERWIJDEREN",
- "HEAD": "HOOFD"
+ "HEAD": "HOOFD",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "Lees één webpagina als Markdown, HTML, links, een screenshot of JSON.",
+ "Pull structured data from one or more pages with AI.": "Haal met AI gestructureerde gegevens uit een of meer pagina's.",
+ "Read many pages of a site by following links from a start URL.": "Lees veel pagina's van een site door links te volgen vanaf een start-URL.",
+ "Get the status and pages of a crawl started earlier.": "Haal de status en pagina's op van een eerder gestarte crawl.",
+ "List the links found on a website.": "Toon de links die op een website staan.",
+ "Run Browser Actions": "Browseracties uitvoeren",
+ "Use Cache": "Cache gebruiken",
+ "Start URL": "Start-URL",
+ "Pages to Crawl": "Te crawlen pagina's",
+ "Max Pages": "Max. pagina's",
+ "Send Results to a Webhook": "Resultaten naar een webhook sturen",
+ "Max Links": "Max. links",
+ "The page to read.": "De pagina die gelezen wordt.",
+ "Click, type or wait on the page before it is read.": "Klik, typ of wacht op de pagina voordat die gelezen wordt.",
+ "A full-page screenshot is also saved with every scrape.": "Bij elke scrape wordt ook een screenshot van de hele pagina opgeslagen.",
+ "What the step returns for the page.": "Wat de stap voor de pagina teruggeeft.",
+ "Milliseconds to wait for the page to load.": "Milliseconden wachten tot de pagina geladen is.",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "Aan: Firecrawl bewaart de pagina en kan een kopie uit de cache leveren.",
+ "Pages to read. End a URL with /* to cover a whole site.": "Te lezen pagina's. Eindig een URL op /* voor de hele site.",
+ "Also search the web for related pages and context.": "Zoek ook op het web naar verwante pagina's en context.",
+ "What to pull from the page, in plain words.": "Wat je uit de pagina wilt halen, in gewone woorden.",
+ "Seconds to wait. The job keeps running after a timeout.": "Seconden wachten. De taak loopt door na een time-out.",
+ "The crawl follows links from this page.": "De crawl volgt links vanaf deze pagina.",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "Welke pagina's gelezen worden, in gewone woorden. Leeg: pagina's onder de start-URL.",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "Maximaal aantal te lezen pagina's. Leeg: Firecrawl-limiet van 10.000.",
+ "Seconds to wait. The crawl keeps running after a timeout.": "Seconden wachten. De crawl loopt door na een time-out.",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "Pagina's die in de afgelopen 2 dagen gelezen zijn, kunnen uit de cache van Firecrawl komen.",
+ "What the step returns for each page.": "Wat de stap voor elke pagina teruggeeft.",
+ "Skip headers, menus and footers.": "Sla kopteksten, menu's en voetteksten over.",
+ "Firecrawl posts each event to your URL as the crawl runs.": "Firecrawl stuurt elke gebeurtenis naar je URL terwijl de crawl loopt.",
+ "The id in the webhook events of a Crawl step.": "De id in de webhookgebeurtenissen van een Crawl-stap.",
+ "The site to list links from.": "De site waarvan de links getoond worden.",
+ "Also list pages on subdomains, like blog.example.com.": "Toon ook pagina's op subdomeinen, zoals blog.example.com.",
+ "Most links to return, up to 100,000.": "Maximaal aantal links, tot 100.000."
}
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/i18n/pt.json b/packages/pieces/community/firecrawl/src/i18n/pt.json
index d932788fad3..d07b5aa7253 100644
--- a/packages/pieces/community/firecrawl/src/i18n/pt.json
+++ b/packages/pieces/community/firecrawl/src/i18n/pt.json
@@ -9,7 +9,7 @@
"Scrape URL": "Scrape URL",
"Crawl Website": "Crawl Website",
"Get Crawl Results": "Get Crawl Results",
- "Map Website": "Map Website",
+ "Map Website": "Mapear site",
"Extract Data": "Extract Data",
"Batch Scrape": "Batch Scrape",
"Get Batch Scrape Results": "Get Batch Scrape Results",
@@ -67,7 +67,7 @@
"Include subdomain": "Incluir subdomínio",
"Page Limit": "Page Limit",
"Search Filter": "Search Filter",
- "Include Subdomains": "Include Subdomains",
+ "Include Subdomains": "Incluir subdomínios",
"JSON Schema": "JSON Schema",
"Max Concurrency": "Max Concurrency",
"Per-Page Timeout (ms)": "Per-Page Timeout (ms)",
@@ -183,5 +183,39 @@
"PATCH": "COMPRAR",
"PUT": "COLOCAR",
"DELETE": "EXCLUIR",
- "HEAD": "CABEÇA"
+ "HEAD": "CABEÇA",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "Lê uma página web como Markdown, HTML, links, captura de tela ou JSON.",
+ "Pull structured data from one or more pages with AI.": "Extrai dados estruturados de uma ou mais páginas com IA.",
+ "Read many pages of a site by following links from a start URL.": "Lê muitas páginas de um site seguindo links a partir de uma URL inicial.",
+ "Get the status and pages of a crawl started earlier.": "Obtém o status e as páginas de um rastreamento iniciado antes.",
+ "List the links found on a website.": "Lista os links encontrados em um site.",
+ "Run Browser Actions": "Executar ações do navegador",
+ "Use Cache": "Usar cache",
+ "Start URL": "URL inicial",
+ "Pages to Crawl": "Páginas a rastrear",
+ "Max Pages": "Máx. de páginas",
+ "Send Results to a Webhook": "Enviar resultados para um webhook",
+ "Max Links": "Máx. de links",
+ "The page to read.": "A página a ser lida.",
+ "Click, type or wait on the page before it is read.": "Clique, digite ou aguarde na página antes da leitura.",
+ "A full-page screenshot is also saved with every scrape.": "Cada scrape também salva uma captura da página inteira.",
+ "What the step returns for the page.": "O que a etapa retorna para a página.",
+ "Milliseconds to wait for the page to load.": "Milissegundos de espera pelo carregamento da página.",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "Ativado: o Firecrawl guarda a página e pode servir uma cópia em cache.",
+ "Pages to read. End a URL with /* to cover a whole site.": "Páginas a ler. Termine uma URL com /* para cobrir o site inteiro.",
+ "Also search the web for related pages and context.": "Também pesquisa na web páginas e contexto relacionados.",
+ "What to pull from the page, in plain words.": "O que extrair da página, em palavras simples.",
+ "Seconds to wait. The job keeps running after a timeout.": "Segundos de espera. A tarefa continua rodando após o tempo limite.",
+ "The crawl follows links from this page.": "O rastreamento segue os links a partir desta página.",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "Quais páginas ler, em palavras simples. Vazio: páginas abaixo da URL inicial.",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "Máximo de páginas a ler. Vazio: limite do Firecrawl de 10.000.",
+ "Seconds to wait. The crawl keeps running after a timeout.": "Segundos de espera. O rastreamento continua após o tempo limite.",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "Páginas lidas nos últimos 2 dias podem vir do cache do Firecrawl.",
+ "What the step returns for each page.": "O que a etapa retorna para cada página.",
+ "Skip headers, menus and footers.": "Ignora cabeçalhos, menus e rodapés.",
+ "Firecrawl posts each event to your URL as the crawl runs.": "O Firecrawl envia cada evento para sua URL durante o rastreamento.",
+ "The id in the webhook events of a Crawl step.": "O id nos eventos de webhook de uma etapa Crawl.",
+ "The site to list links from.": "O site cujos links serão listados.",
+ "Also list pages on subdomains, like blog.example.com.": "Também lista páginas em subdomínios, como blog.example.com.",
+ "Most links to return, up to 100,000.": "Máximo de links a retornar, até 100.000."
}
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/i18n/ru.json b/packages/pieces/community/firecrawl/src/i18n/ru.json
index 07fe74536e5..3b509175c43 100644
--- a/packages/pieces/community/firecrawl/src/i18n/ru.json
+++ b/packages/pieces/community/firecrawl/src/i18n/ru.json
@@ -60,5 +60,42 @@
"PATCH": "ПАТЧ",
"PUT": "ПОКУПИТЬ",
"DELETE": "УДАЛИТЬ",
- "HEAD": "HEAD"
+ "HEAD": "HEAD",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "Прочитать одну веб-страницу как Markdown, HTML, ссылки, скриншот или JSON.",
+ "Pull structured data from one or more pages with AI.": "Извлечь структурированные данные с одной или нескольких страниц с помощью ИИ.",
+ "Read many pages of a site by following links from a start URL.": "Прочитать много страниц сайта, переходя по ссылкам с начального URL.",
+ "Get the status and pages of a crawl started earlier.": "Получить статус и страницы ранее запущенного обхода.",
+ "List the links found on a website.": "Перечислить ссылки, найденные на сайте.",
+ "Run Browser Actions": "Выполнить действия в браузере",
+ "Use Cache": "Использовать кэш",
+ "Start URL": "Начальный URL",
+ "Pages to Crawl": "Страницы для обхода",
+ "Max Pages": "Макс. страниц",
+ "Send Results to a Webhook": "Отправлять результаты на вебхук",
+ "Max Links": "Макс. ссылок",
+ "The page to read.": "Страница, которую нужно прочитать.",
+ "Click, type or wait on the page before it is read.": "Нажать, ввести текст или подождать на странице перед чтением.",
+ "A full-page screenshot is also saved with every scrape.": "При каждом scrape также сохраняется скриншот всей страницы.",
+ "What the step returns for the page.": "Что шаг возвращает для страницы.",
+ "Milliseconds to wait for the page to load.": "Сколько миллисекунд ждать загрузки страницы.",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "Вкл.: Firecrawl сохраняет страницу и может вернуть копию из кэша.",
+ "Pages to read. End a URL with /* to cover a whole site.": "Страницы для чтения. Добавьте /* в конце URL, чтобы охватить весь сайт.",
+ "Also search the web for related pages and context.": "Также искать в интернете связанные страницы и контекст.",
+ "What to pull from the page, in plain words.": "Что извлечь со страницы, простыми словами.",
+ "Seconds to wait. The job keeps running after a timeout.": "Сколько секунд ждать. После тайм-аута задача продолжает работать.",
+ "The crawl follows links from this page.": "Обход идёт по ссылкам с этой страницы.",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "Какие страницы читать, простыми словами. Пусто: страницы под начальным URL.",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "Сколько страниц читать максимум. Пусто: лимит Firecrawl в 10 000.",
+ "Seconds to wait. The crawl keeps running after a timeout.": "Сколько секунд ждать. После тайм-аута обход продолжается.",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "Страницы, прочитанные за последние 2 дня, могут браться из кэша Firecrawl.",
+ "What the step returns for each page.": "Что шаг возвращает для каждой страницы.",
+ "Skip headers, menus and footers.": "Пропускать шапки, меню и подвалы.",
+ "Firecrawl posts each event to your URL as the crawl runs.": "Firecrawl отправляет каждое событие на ваш URL во время обхода.",
+ "The id in the webhook events of a Crawl step.": "id из событий вебхука шага Crawl.",
+ "The site to list links from.": "Сайт, ссылки которого нужно перечислить.",
+ "Also list pages on subdomains, like blog.example.com.": "Также перечислять страницы на поддоменах, например blog.example.com.",
+ "Most links to return, up to 100,000.": "Сколько ссылок вернуть максимум, до 100 000.",
+ "Map Website": "Карта сайта",
+ "Include Subdomains": "Включить поддомены",
+ "Timeout": "Тайм-аут"
}
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/i18n/translation.json b/packages/pieces/community/firecrawl/src/i18n/translation.json
index a97eae5210c..96e4dadd9c2 100644
--- a/packages/pieces/community/firecrawl/src/i18n/translation.json
+++ b/packages/pieces/community/firecrawl/src/i18n/translation.json
@@ -5,11 +5,10 @@
"Extract Structured Data": "Extract Structured Data",
"Crawl": "Crawl",
"Crawl Results": "Crawl Results",
- "Map Websites": "Map Websites",
+ "Map Website": "Map Website",
"Scrape URL": "Scrape URL",
"Crawl Website": "Crawl Website",
"Get Crawl Results": "Get Crawl Results",
- "Map Website": "Map Website",
"Extract Data": "Extract Data",
"Batch Scrape": "Batch Scrape",
"Get Batch Scrape Results": "Get Batch Scrape Results",
@@ -22,11 +21,11 @@
"Start Agent": "Start Agent",
"Get Agent Status": "Get Agent Status",
"Custom API Call": "Custom API Call",
- "Scrape a website by performing a series of actions like clicking, typing, taking screenshots, and extracting data.": "Scrape a website by performing a series of actions like clicking, typing, taking screenshots, and extracting data.",
- "Extract structured data from multiple URLs using AI.": "Extract structured data from multiple URLs using AI.",
- "Crawl multiple pages from a website based on specified rules and patterns.": "Crawl multiple pages from a website based on specified rules and patterns.",
- "Get the results of a crawl job.": "Get the results of a crawl job.",
- "Input a website and get all the urls on the website.": "Input a website and get all the urls on the website.",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "Read one web page as Markdown, HTML, links, a screenshot or JSON.",
+ "Pull structured data from one or more pages with AI.": "Pull structured data from one or more pages with AI.",
+ "Read many pages of a site by following links from a start URL.": "Read many pages of a site by following links from a start URL.",
+ "Get the status and pages of a crawl started earlier.": "Get the status and pages of a crawl started earlier.",
+ "List the links found on a website.": "List the links found on a website.",
"Fetch the content of a single web page in a chosen format.": "Fetch the content of a single web page in a chosen format.",
"Crawl a website starting from a base URL and return the content of many pages.": "Crawl a website starting from a base URL and return the content of many pages.",
"Get the status and accumulated page results of a crawl job by its ID.": "Get the status and accumulated page results of a crawl job by its ID.",
@@ -44,30 +43,33 @@
"Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.": "Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.",
"Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint",
"Website URL": "Website URL",
- "Timeout (ms)": "Timeout (ms)",
- "Store In Cache": "Store In Cache",
- "Perform Actions Before Scraping": "Perform Actions Before Scraping",
+ "Run Browser Actions": "Run Browser Actions",
"Action Properties": "Action Properties",
+ "Markdown": "Markdown",
"Output Format": "Output Format",
"Extraction Prompt": "Extraction Prompt",
"Schema Mode": "Schema Mode",
"Data Definition": "Data Definition",
+ "Timeout": "Timeout",
+ "Use Cache": "Use Cache",
"URLs": "URLs",
"Enable Web Search": "Enable Web Search",
- "Timeout (seconds)": "Timeout (seconds)",
"Data Schema Type": "Data Schema Type",
- "URL": "URL",
- "Prompt": "Prompt",
- "Limit": "Limit",
+ "Start URL": "Start URL",
+ "Pages to Crawl": "Pages to Crawl",
+ "Max Pages": "Max Pages",
"Only Main Content": "Only Main Content",
- "Deliver Results to Webhook": "Deliver Results to Webhook",
+ "Send Results to a Webhook": "Send Results to a Webhook",
"Webhook Properties": "Webhook Properties",
"Crawl ID": "Crawl ID",
- "Main Website URL": "Main Website URL",
- "Include subdomain": "Include subdomain",
+ "Include Subdomains": "Include Subdomains",
+ "Max Links": "Max Links",
+ "URL": "URL",
+ "Timeout (ms)": "Timeout (ms)",
"Page Limit": "Page Limit",
+ "Timeout (seconds)": "Timeout (seconds)",
"Search Filter": "Search Filter",
- "Include Subdomains": "Include Subdomains",
+ "Limit": "Limit",
"JSON Schema": "JSON Schema",
"Max Concurrency": "Max Concurrency",
"Per-Page Timeout (ms)": "Per-Page Timeout (ms)",
@@ -79,6 +81,7 @@
"Time Filter": "Time Filter",
"Location": "Location",
"Include Page Content": "Include Page Content",
+ "Prompt": "Prompt",
"Result Schema": "Result Schema",
"Max Credits": "Max Credits",
"Strictly Constrain to URLs": "Strictly Constrain to URLs",
@@ -90,30 +93,33 @@
"Body": "Body",
"Response is Binary": "Response is Binary",
"Return Error as Output": "Return Error as Output",
- "Timeout": "Timeout",
"Follow redirects": "Follow redirects",
- "The webpage URL to scrape.": "The webpage URL to scrape.",
- "Maximum time to wait for the page to load (in milliseconds).": "Maximum time to wait for the page to load (in milliseconds).",
- "If enabled, the page will be stored in the Firecrawl index and cache. Disabling this is useful if your scraping activity may have data protection concerns.": "If enabled, the page will be stored in the Firecrawl index and cache. Disabling this is useful if your scraping activity may have data protection concerns.",
- "Enable to perform a sequence of actions on the page before scraping (like clicking buttons, filling forms, etc.). See [Firecrawl Actions Documentation](https://docs.firecrawl.dev/api-reference/endpoint/scrape#body-actions) for details on available actions and their parameters.": "Enable to perform a sequence of actions on the page before scraping (like clicking buttons, filling forms, etc.). See [Firecrawl Actions Documentation](https://docs.firecrawl.dev/api-reference/endpoint/scrape#body-actions) for details on available actions and their parameters.",
+ "The page to read.": "The page to read.",
+ "Click, type or wait on the page before it is read.": "Click, type or wait on the page before it is read.",
"Properties for actions that will be performed on the page.": "Properties for actions that will be performed on the page.",
- "Choose what format you want your output in.": "Choose what format you want your output in.",
+ "A full-page screenshot is also saved with every scrape.": "A full-page screenshot is also saved with every scrape.",
+ "What the step returns for the page.": "What the step returns for the page.",
"Prompt for extracting data.": "Prompt for extracting data.",
"Data schema type.": "Data schema type.",
- "Add one or more URLs to extract data from.": "Add one or more URLs to extract data from.",
- "Describe what information you want to extract.": "Describe what information you want to extract.",
- "Enable web search to find additional context.": "Enable web search to find additional context.",
- "Timeout in seconds after which the task will be cancelled": "Timeout in seconds after which the task will be cancelled",
- "For complex schema, you can use advanced mode.": "For complex schema, you can use advanced mode.",
- "The base URL to start crawling from.": "The base URL to start crawling from.",
- "Maximum number of pages to crawl. Default limit is 10.": "Maximum number of pages to crawl. Default limit is 10.",
- "Only return the main content of the page, excluding headers, navs, footers, etc.": "Only return the main content of the page, excluding headers, navs, footers, etc.",
- "Enable to send crawl results to a webhook URL.": "Enable to send crawl results to a webhook URL.",
+ "Milliseconds to wait for the page to load.": "Milliseconds to wait for the page to load.",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "On: Firecrawl keeps the page and may serve a cached copy.",
+ "Pages to read. End a URL with /* to cover a whole site.": "Pages to read. End a URL with /* to cover a whole site.",
+ "Also search the web for related pages and context.": "Also search the web for related pages and context.",
+ "What to pull from the page, in plain words.": "What to pull from the page, in plain words.",
+ "Seconds to wait. The job keeps running after a timeout.": "Seconds to wait. The job keeps running after a timeout.",
+ "The crawl follows links from this page.": "The crawl follows links from this page.",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "Which pages to read, in plain words. Empty: pages under the start URL.",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "Most pages to read. Empty: Firecrawl's limit of 10,000.",
+ "Seconds to wait. The crawl keeps running after a timeout.": "Seconds to wait. The crawl keeps running after a timeout.",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "Pages read in the last 2 days may come from Firecrawl's cache.",
+ "What the step returns for each page.": "What the step returns for each page.",
+ "Skip headers, menus and footers.": "Skip headers, menus and footers.",
+ "Firecrawl posts each event to your URL as the crawl runs.": "Firecrawl posts each event to your URL as the crawl runs.",
"Properties for webhook configuration.": "Properties for webhook configuration.",
- "The ID of the crawl job to check.": "The ID of the crawl job to check.",
- "The webpage URL to start scraping from.": "The webpage URL to start scraping from.",
- "Include and crawl pages from subdomains of the target website (e.g., blog.example.com, shop.example.com) in addition to the main domain.": "Include and crawl pages from subdomains of the target website (e.g., blog.example.com, shop.example.com) in addition to the main domain.",
- "Maximum number of links to return (max: 100,000)": "Maximum number of links to return (max: 100,000)",
+ "The id in the webhook events of a Crawl step.": "The id in the webhook events of a Crawl step.",
+ "The site to list links from.": "The site to list links from.",
+ "Also list pages on subdomains, like blog.example.com.": "Also list pages on subdomains, like blog.example.com.",
+ "Most links to return, up to 100,000.": "Most links to return, up to 100,000.",
"The exact web page URL to scrape (e.g. https://example.com/about). Obtain candidate URLs from Search Web or Map Website if you do not already have one.": "The exact web page URL to scrape (e.g. https://example.com/about). Obtain candidate URLs from Search Web or Map Website if you do not already have one.",
"Which representation of the page to return.": "Which representation of the page to return.",
"Return only the main content of the page, excluding headers, navs, footers, etc.": "Return only the main content of the page, excluding headers, navs, footers, etc.",
@@ -164,7 +170,6 @@
"Follow 3xx redirects instead of returning them as the response.": "Follow 3xx redirects instead of returning them as the response.",
"Simple": "Simple",
"Advanced": "Advanced",
- "Markdown": "Markdown",
"HTML": "HTML",
"Raw HTML": "Raw HTML",
"Links": "Links",
diff --git a/packages/pieces/community/firecrawl/src/i18n/zh.json b/packages/pieces/community/firecrawl/src/i18n/zh.json
index 136edb88bc4..8dc21fa3d37 100644
--- a/packages/pieces/community/firecrawl/src/i18n/zh.json
+++ b/packages/pieces/community/firecrawl/src/i18n/zh.json
@@ -9,7 +9,7 @@
"Scrape URL": "Scrape URL",
"Crawl Website": "Crawl Website",
"Get Crawl Results": "Get Crawl Results",
- "Map Website": "Map Website",
+ "Map Website": "网站地图",
"Extract Data": "Extract Data",
"Batch Scrape": "Batch Scrape",
"Get Batch Scrape Results": "Get Batch Scrape Results",
@@ -43,7 +43,7 @@
"Start an autonomous Firecrawl (FIRE-1) agent job from a natural-language prompt.": "Start an autonomous Firecrawl (FIRE-1) agent job from a natural-language prompt.",
"Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.": "Get the status and result of a Firecrawl (FIRE-1) agent job by its ID.",
"Make a custom API call to a specific endpoint": "将一个自定义 API 调用到一个特定的终点",
- "Website URL": "Website URL",
+ "Website URL": "网站 URL",
"Timeout (ms)": "Timeout (ms)",
"Store In Cache": "Store In Cache",
"Perform Actions Before Scraping": "Perform Actions Before Scraping",
@@ -67,7 +67,7 @@
"Include subdomain": "Include subdomain",
"Page Limit": "Page Limit",
"Search Filter": "Search Filter",
- "Include Subdomains": "Include Subdomains",
+ "Include Subdomains": "包含子域名",
"JSON Schema": "JSON Schema",
"Max Concurrency": "Max Concurrency",
"Per-Page Timeout (ms)": "Per-Page Timeout (ms)",
@@ -90,7 +90,7 @@
"Body": "正文内容",
"Response is Binary": "Response is Binary",
"Return Error as Output": "Return Error as Output",
- "Timeout": "Timeout",
+ "Timeout": "超时",
"Follow redirects": "Follow redirects",
"The webpage URL to scrape.": "The webpage URL to scrape.",
"Maximum time to wait for the page to load (in milliseconds).": "Maximum time to wait for the page to load (in milliseconds).",
@@ -183,5 +183,39 @@
"PATCH": "PATCH",
"PUT": "弹出",
"DELETE": "删除",
- "HEAD": "黑色"
+ "HEAD": "黑色",
+ "Read one web page as Markdown, HTML, links, a screenshot or JSON.": "以 Markdown、HTML、链接、截图或 JSON 读取单个网页。",
+ "Pull structured data from one or more pages with AI.": "用 AI 从一个或多个页面提取结构化数据。",
+ "Read many pages of a site by following links from a start URL.": "从起始 URL 跟随链接,读取网站的多个页面。",
+ "Get the status and pages of a crawl started earlier.": "获取先前启动的抓取的状态和页面。",
+ "List the links found on a website.": "列出网站上找到的链接。",
+ "Run Browser Actions": "执行浏览器操作",
+ "Use Cache": "使用缓存",
+ "Start URL": "起始 URL",
+ "Pages to Crawl": "要抓取的页面",
+ "Max Pages": "最大页面数",
+ "Send Results to a Webhook": "将结果发送到 Webhook",
+ "Max Links": "最大链接数",
+ "The page to read.": "要读取的页面。",
+ "Click, type or wait on the page before it is read.": "读取前在页面上点击、输入或等待。",
+ "A full-page screenshot is also saved with every scrape.": "每次抓取都会同时保存一张整页截图。",
+ "What the step returns for the page.": "此步骤为该页面返回的内容。",
+ "Milliseconds to wait for the page to load.": "等待页面加载的毫秒数。",
+ "On: Firecrawl keeps the page and may serve a cached copy.": "开启:Firecrawl 会保存页面,并可能返回缓存副本。",
+ "Pages to read. End a URL with /* to cover a whole site.": "要读取的页面。URL 以 /* 结尾可覆盖整个网站。",
+ "Also search the web for related pages and context.": "同时在网上搜索相关页面和上下文。",
+ "What to pull from the page, in plain words.": "用简单的话说明要从页面提取什么。",
+ "Seconds to wait. The job keeps running after a timeout.": "等待的秒数。超时后任务仍会继续运行。",
+ "The crawl follows links from this page.": "抓取会从此页面开始跟随链接。",
+ "Which pages to read, in plain words. Empty: pages under the start URL.": "用简单的话说明要读取哪些页面。留空:起始 URL 下的页面。",
+ "Most pages to read. Empty: Firecrawl's limit of 10,000.": "最多读取的页面数。留空:Firecrawl 上限 10,000。",
+ "Seconds to wait. The crawl keeps running after a timeout.": "等待的秒数。超时后抓取仍会继续运行。",
+ "Pages read in the last 2 days may come from Firecrawl's cache.": "过去 2 天内读取过的页面可能来自 Firecrawl 的缓存。",
+ "What the step returns for each page.": "此步骤为每个页面返回的内容。",
+ "Skip headers, menus and footers.": "跳过页眉、菜单和页脚。",
+ "Firecrawl posts each event to your URL as the crawl runs.": "抓取运行期间,Firecrawl 会把每个事件发送到你的 URL。",
+ "The id in the webhook events of a Crawl step.": "Crawl 步骤 Webhook 事件中的 id。",
+ "The site to list links from.": "要列出链接的网站。",
+ "Also list pages on subdomains, like blog.example.com.": "同时列出子域名上的页面,例如 blog.example.com。",
+ "Most links to return, up to 100,000.": "最多返回的链接数,上限 100,000。"
}
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/index.ts b/packages/pieces/community/firecrawl/src/index.ts
index 223f109b47c..500ea427f1e 100644
--- a/packages/pieces/community/firecrawl/src/index.ts
+++ b/packages/pieces/community/firecrawl/src/index.ts
@@ -28,7 +28,7 @@ import { firecrawlAuth } from './lib/auth';
export const firecrawl = createPiece({
displayName: 'Firecrawl',
description: 'The context API to search, scrape, and interact with the web at scale.',
- minimumSupportedRelease: '0.86.4',
+ minimumSupportedRelease: '0.88.2',
logoUrl: 'https://cdn.activepieces.com/pieces/firecrawl.png',
categories: [PieceCategory.ARTIFICIAL_INTELLIGENCE],
authors: ["geekyme-fsmk", "geekyme", "arinmakk"],
@@ -63,6 +63,12 @@ export const firecrawl = createPiece({
authMapping: async (auth) => ({
'Authorization': `Bearer ${auth.secret_text}`,
}),
+ props: {
+ url: {
+ description: `Full URL, or a path relative to ${FIRECRAWL_API_BASE_URL}`,
+ placeholder: '/team/credit-usage',
+ },
+ },
}),
],
triggers: [],
diff --git a/packages/pieces/community/firecrawl/src/lib/actions/crawl-results.ts b/packages/pieces/community/firecrawl/src/lib/actions/crawl-results.ts
index 1390d89e09e..72405df5b3e 100644
--- a/packages/pieces/community/firecrawl/src/lib/actions/crawl-results.ts
+++ b/packages/pieces/community/firecrawl/src/lib/actions/crawl-results.ts
@@ -1,6 +1,7 @@
import { createAction, Property } from '@activepieces/pieces-framework';
import { httpClient, HttpMethod } from '@activepieces/pieces-common';
import { firecrawlAuth } from '../auth';
+import { CrawlStatusPage, FIRECRAWL_API_BASE_URL, followCrawlPages } from '../common/common';
import { getCrawlResultsActionOutputSchema } from '../output-schemas';
export const crawlResults = createAction({
@@ -8,26 +9,27 @@ export const crawlResults = createAction({
name: 'crawlResults',
classification: 'READ',
displayName: 'Crawl Results',
- description: 'Get the results of a crawl job.',
+ description: 'Get the status and pages of a crawl started earlier.',
audience: 'human',
outputSchema: getCrawlResultsActionOutputSchema,
aiMetadata: { description: 'Looks up the status and accumulated page results of a previously started crawl job by its crawl ID. Choose this to poll or fetch the output of a Crawl run that delivered an ID rather than waiting inline. Requires a valid crawl ID; it is a read-only lookup, so repeating the call is safe.', idempotent: true },
props: {
crawlId: Property.ShortText({
displayName: 'Crawl ID',
- description: 'The ID of the crawl job to check.',
+ description: 'The id in the webhook events of a Crawl step.',
+ placeholder: '0f1b3e9c-4a6d-4c1e-9a7b-2d8f5e6c7a10',
required: true,
}),
},
async run({ auth, propsValue }) {
- const response = await httpClient.sendRequest({
+ const response = await httpClient.sendRequest<CrawlStatusPage>({
method: HttpMethod.GET,
- url: `https://api.firecrawl.dev/v1/crawl/${propsValue.crawlId}`,
+ url: `${FIRECRAWL_API_BASE_URL}/crawl/${propsValue.crawlId}`,
headers: {
'Authorization': `Bearer ${auth.secret_text}`,
},
});
- return response.body;
+ return followCrawlPages({ firstPage: response.body, auth: auth.secret_text });
},
});
\ No newline at end of file
diff --git a/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts b/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts
index d5e83dcc226..0553bfa4778 100644
--- a/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts
+++ b/packages/pieces/community/firecrawl/src/lib/actions/crawl.ts
@@ -1,7 +1,7 @@
-import { createAction, Property, InputPropertyMap, tryCatch } from '@activepieces/pieces-framework';
+import { createAction, Property, InputPropertyMap, tryCatch, MarkdownVariant } from '@activepieces/pieces-framework';
import { httpClient, HttpError, HttpMethod } from '@activepieces/pieces-common';
import { firecrawlAuth } from '../auth';
-import { forScreenshotOutputFormat, forSimpleOutputFormat, forJsonOutputFormat, polling, saveFirecrawlFile, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';
+import { forScreenshotOutputFormat, forSimpleOutputFormat, forJsonOutputFormat, polling, followCrawlPages, saveFirecrawlFile, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';
import { crawlActionOutputSchema } from '../output-schemas';
function webhookConfig(useWebhook: boolean, webhookProperties: any): any {
@@ -38,32 +38,45 @@ export const crawl = createAction({
name: 'crawl',
classification: 'SEARCH',
displayName: 'Crawl',
- description: 'Crawl multiple pages from a website based on specified rules and patterns.',
+ description: 'Read many pages of a site by following links from a start URL.',
audience: 'human',
outputSchema: crawlActionOutputSchema,
aiMetadata: { description: 'Starts from a base URL, discovers and follows links across the site, and returns the content of many pages in a chosen format (markdown, HTML, links, summary, screenshot, or AI-extracted JSON). Choose this to gather content from a whole site or section; use Scrape for a single page or Map to only list URLs without fetching content. Bounded by the page limit and timeout, and can deliver results to a webhook. Read-only against the site, so re-running the same call is safe.', idempotent: true },
props: {
url: Property.ShortText({
- displayName: 'URL',
- description: 'The base URL to start crawling from.',
+ displayName: 'Start URL',
+ description: 'The crawl follows links from this page.',
+ placeholder: 'https://example.com/blog',
required: true,
}),
prompt: Property.LongText({
- displayName: 'Prompt',
- description: 'Describe what information you want to extract.',
+ displayName: 'Pages to Crawl',
+ description: 'Which pages to read, in plain words. Empty: pages under the start URL.',
required: false,
- defaultValue: 'Get me all of the blog pages on the website, probably localed in /blog'
+ defaultValue: 'Get me all of the blog pages on the website, probably located in /blog'
}),
limit: Property.Number({
- displayName: 'Limit',
- description: 'Maximum number of pages to crawl. Default limit is 10.',
+ displayName: 'Max Pages',
+ description: "Most pages to read. Empty: Firecrawl's limit of 10,000.",
required: false,
defaultValue: 10,
+ width: 'half',
+ }),
+ timeout: Property.Number({
+ displayName: 'Timeout',
+ description: 'Seconds to wait. The crawl keeps running after a timeout.',
+ required: false,
+ defaultValue: 300,
+ width: 'half',
+ }),
+ cache_note: Property.MarkDown({
+ value: "Pages read in the last 2 days may come from Firecrawl's cache.",
+ variant: MarkdownVariant.INFO,
}),
formats: Property.Dropdown({
auth: firecrawlAuth,
displayName: 'Output Format',
- description: 'Choose what format you want your output in.',
+ description: 'What the step returns for each page.',
required: true,
refreshers: [],
options: async () => {
@@ -82,7 +95,7 @@ export const crawl = createAction({
}),
onlyMainContent: Property.Checkbox({
displayName: 'Only Main Content',
- description: 'Only return the main content of the page, excluding headers, navs, footers, etc.',
+ description: 'Skip headers, menus and footers.',
required: false,
defaultValue: false,
}),
@@ -102,14 +115,14 @@ export const crawl = createAction({
const map: InputPropertyMap= {
mode: Property.StaticDropdown<'simple' | 'advanced'>({
displayName: 'Data Schema Type',
- description: 'For complex schema, you can use advanced mode.',
required: true,
defaultValue: 'simple',
+ display: 'cards',
options: {
disabled: false,
options: [
- { label: 'Simple', value: 'simple' },
- { label: 'Advanced', value: 'advanced' },
+ { label: 'Simple', value: 'simple', description: 'A list of fields', icon: 'text' },
+ { label: 'Advanced', value: 'advanced', description: 'A JSON Schema', icon: 'code' },
],
},
}),
@@ -134,8 +147,7 @@ export const crawl = createAction({
return {
fields: Property.Json({
displayName: 'JSON Schema',
- description:
- 'Learn more about JSON Schema here: https://json-schema.org/learn/getting-started-step-by-step',
+ description: 'The shape of the data to return, as a JSON Schema.',
required: true,
defaultValue: {
type: 'object',
@@ -154,24 +166,23 @@ export const crawl = createAction({
}
return {
fields: Property.Array({
- displayName: 'Data Definition',
+ displayName: 'Fields to Extract',
+ description: 'One row per value to pull from the page.',
required: true,
properties: {
name: Property.ShortText({
displayName: 'Name',
- description:
- 'Provide the name of the value you want to extract from the unstructured text. The name should be unique and short. ',
+ description: 'Letters, numbers, _ . or - only.',
+ placeholder: 'price',
required: true,
}),
description: Property.LongText({
displayName: 'Description',
- description:
- 'Brief description of the data, this hints for the AI on what to look for',
+ description: 'What the value is, as a hint for the AI.',
required: false,
}),
type: Property.StaticDropdown({
displayName: 'Data Type',
- description: 'Type of parameter.',
required: true,
defaultValue: 'string',
options: {
@@ -184,7 +195,8 @@ export const crawl = createAction({
},
}),
isRequired: Property.Checkbox({
- displayName: 'Fail if Not present?',
+ displayName: 'Required Field',
+ description: 'The AI must return a value for this field.',
required: true,
defaultValue: false,
}),
@@ -193,15 +205,9 @@ export const crawl = createAction({
};
},
}),
- timeout: Property.Number({
- displayName: 'Timeout (seconds)',
- description: 'Timeout in seconds after which the task will be cancelled',
- required: false,
- defaultValue: 300,
- }),
useWebhook: Property.Checkbox({
- displayName: 'Deliver Results to Webhook',
- description: 'Enable to send crawl results to a webhook URL.',
+ displayName: 'Send Results to a Webhook',
+ description: 'Firecrawl posts each event to your URL as the crawl runs.',
required: false,
defaultValue: false,
}),
@@ -213,15 +219,16 @@ export const crawl = createAction({
auth: firecrawlAuth,
props: async (propsValue): Promise<InputPropertyMap> => {
const useWebhook = propsValue['useWebhook'] as unknown as boolean;
-
+
if (!useWebhook) {
return {};
}
-
+
const map: InputPropertyMap = {
webhookUrl: Property.ShortText({
displayName: 'Webhook URL',
- description: 'The URL to send the webhook to. This will trigger for crawl started (crawl.started), every page crawled (crawl.page) and when the crawl is completed (crawl.completed or crawl.failed).',
+ description: 'Gets an event when the crawl starts, per page, and when it ends.',
+ placeholder: 'https://example.com/webhook',
required: true,
}),
webhookHeaders: Property.Json({
@@ -232,13 +239,13 @@ export const crawl = createAction({
}),
webhookMetadata: Property.Json({
displayName: 'Webhook Metadata',
- description: 'Custom metadata that will be included in all webhook payloads for this crawl.',
+ description: 'Extra data added to every event for this crawl.',
required: false,
defaultValue: {},
}),
webhookEvents: Property.Array({
displayName: 'Webhook Events',
- description: 'Type of events that should be sent to the webhook URL. (default: all)',
+ description: 'Events to send: started, page, completed, failed.',
required: false,
defaultValue: ['completed', 'page', 'failed', 'started'],
}),
@@ -247,11 +254,17 @@ export const crawl = createAction({
},
}),
},
+ propertyGroups: [
+ { key: 'site', display: 'section', label: 'Site', icon: 'file', props: ['url', 'prompt'] },
+ { key: 'limits', display: 'section', label: 'Limits', icon: 'sliders', props: ['limit', 'timeout'] },
+ { key: 'output', display: 'section', label: 'Output', icon: 'code', props: ['cache_note', 'formats', 'onlyMainContent', 'extractMode', 'extractSchema'] },
+ { key: 'webhook', display: 'section', label: 'Webhook', icon: 'send', props: ['useWebhook', 'webhookProperties'] },
+ ],
async run(context) {
const { auth, propsValue } = context;
const body: Record<string, any> = {
url: propsValue.url,
- sitemap: "include",
+ sitemap: "include",
crawlEntireDomain: false,
maxDiscoveryDepth: 10,
origin: FIRECRAWL_ORIGIN,
@@ -261,7 +274,7 @@ export const crawl = createAction({
body['limit'] = propsValue.limit;
}
- if (propsValue.prompt !== undefined) {
+ if (typeof propsValue.prompt === 'string' && propsValue.prompt.trim() !== '') {
body['prompt'] = propsValue.prompt
}
@@ -311,18 +324,20 @@ export const crawl = createAction({
const jobId = response.body.id;
- // polling
const timeoutSeconds = propsValue.timeout || 300;
- const result = await polling(jobId, auth.secret_text, timeoutSeconds, 'crawl');
+ const result = await followCrawlPages({
+ firstPage: await polling(jobId, auth.secret_text, timeoutSeconds, 'crawl'),
+ auth: auth.secret_text,
+ });
if (propsValue.formats === 'screenshot' && Array.isArray(result.data)) {
- const savedPages = [];
+ const savedPages: Record<string, unknown>[] = [];
for (const page of result.data) {
- if (!page.screenshot) {
+ const { screenshot, ...pageWithoutScreenshot } = page;
+ if (typeof screenshot !== 'string' || !screenshot) {
savedPages.push(page);
continue;
}
- const { screenshot, ...pageWithoutScreenshot } = page;
const { data: savedScreenshot, error } = await tryCatch(() =>
saveFirecrawlFile({ context, firecrawlFileUrl: screenshot })
);
diff --git a/packages/pieces/community/firecrawl/src/lib/actions/extract.ts b/packages/pieces/community/firecrawl/src/lib/actions/extract.ts
index e138c385f26..6a7ecbd1fbb 100644
--- a/packages/pieces/community/firecrawl/src/lib/actions/extract.ts
+++ b/packages/pieces/community/firecrawl/src/lib/actions/extract.ts
@@ -9,51 +9,45 @@ export const extract = createAction({
name: 'extract',
classification: 'READ',
displayName: 'Extract Structured Data',
- description: 'Extract structured data from multiple URLs using AI.',
+ description: 'Pull structured data from one or more pages with AI.',
audience: 'human',
outputSchema: extractDataActionOutputSchema,
aiMetadata: { description: 'Pulls structured data from one or more known URLs by running an AI extraction against a natural-language prompt and a field schema (simple field list or advanced JSON Schema). Choose this when you need specific typed values from pages you already have URLs for; use Scrape to read raw page content or Map to discover URLs first. Runs as a polled job bounded by the timeout; read-only against the sites, so repeating the same request is safe.', idempotent: true },
props: {
urls: Property.Array({
displayName: 'URLs',
- description: 'Add one or more URLs to extract data from.',
+ description: 'Pages to read. End a URL with /* to cover a whole site.',
required: true,
properties: {
url: Property.ShortText({
displayName: 'URL',
- description: 'Website URL to extract data from',
+ placeholder: 'https://example.com/pricing',
required: true,
})
}
}),
- prompt: Property.LongText({
- displayName: 'Extraction Prompt',
- description: 'Describe what information you want to extract.',
- required: false,
- defaultValue: 'Extract the following data from the provided content.',
- }),
enableWebSearch: Property.Checkbox({
displayName: 'Enable Web Search',
- description: 'Enable web search to find additional context.',
+ description: 'Also search the web for related pages and context.',
required: false,
defaultValue: false,
}),
- timeout: Property.Number({
- displayName: 'Timeout (seconds)',
- description: 'Timeout in seconds after which the task will be cancelled',
+ prompt: Property.LongText({
+ displayName: 'Extraction Prompt',
+ description: 'What to pull from the page, in plain words.',
required: false,
- defaultValue: 300,
+ defaultValue: 'Extract the following data from the provided content.',
}),
mode: Property.StaticDropdown<'simple' | 'advanced'>({
displayName: 'Data Schema Type',
- description: 'For complex schema, you can use advanced mode.',
required: true,
defaultValue: 'simple',
+ display: 'cards',
options: {
disabled: false,
options: [
- { label: 'Simple', value: 'simple' },
- { label: 'Advanced', value: 'advanced' },
+ { label: 'Simple', value: 'simple', description: 'A list of fields', icon: 'text' },
+ { label: 'Advanced', value: 'advanced', description: 'A JSON Schema', icon: 'code' },
],
},
}),
@@ -68,8 +62,7 @@ export const extract = createAction({
return {
fields: Property.Json({
displayName: 'JSON Schema',
- description:
- 'Learn more about JSON Schema here: https://json-schema.org/learn/getting-started-step-by-step',
+ description: 'The shape of the data to return, as a JSON Schema.',
required: true,
defaultValue: {
type: 'object',
@@ -88,24 +81,23 @@ export const extract = createAction({
}
return {
fields: Property.Array({
- displayName: 'Data Definition',
+ displayName: 'Fields to Extract',
+ description: 'One row per value to pull from the page.',
required: true,
properties: {
name: Property.ShortText({
displayName: 'Name',
- description:
- 'Provide the name of the value you want to extract from the unstructured text. The name should be unique and short. ',
+ description: 'Letters, numbers, _ . or - only.',
+ placeholder: 'price',
required: true,
}),
description: Property.LongText({
displayName: 'Description',
- description:
- 'Brief description of the data, this hints for the AI on what to look for',
+ description: 'What the value is, as a hint for the AI.',
required: false,
}),
type: Property.StaticDropdown({
displayName: 'Data Type',
- description: 'Type of parameter.',
required: true,
defaultValue: 'string',
options: {
@@ -118,7 +110,8 @@ export const extract = createAction({
},
}),
isRequired: Property.Checkbox({
- displayName: 'Fail if Not present?',
+ displayName: 'Required Field',
+ description: 'The AI must return a value for this field.',
required: true,
defaultValue: false,
}),
@@ -127,7 +120,18 @@ export const extract = createAction({
};
},
}),
+ timeout: Property.Number({
+ displayName: 'Timeout',
+ description: 'Seconds to wait. The job keeps running after a timeout.',
+ required: false,
+ defaultValue: 300,
+ advanced: true,
+ }),
},
+ propertyGroups: [
+ { key: 'pages', display: 'section', label: 'Pages', icon: 'file', props: ['urls', 'enableWebSearch'] },
+ { key: 'data', display: 'section', label: 'What to extract', icon: 'code', props: ['prompt', 'mode', 'schema'] },
+ ],
async run( context ) {
const { auth, propsValue } = context;
diff --git a/packages/pieces/community/firecrawl/src/lib/actions/map.ts b/packages/pieces/community/firecrawl/src/lib/actions/map.ts
index 8020166eb11..e43beb15ccb 100644
--- a/packages/pieces/community/firecrawl/src/lib/actions/map.ts
+++ b/packages/pieces/community/firecrawl/src/lib/actions/map.ts
@@ -8,30 +8,34 @@ export const map = createAction({
auth: firecrawlAuth,
name: 'map',
classification: 'SEARCH',
- displayName: 'Map Websites',
- description: 'Input a website and get all the urls on the website.' ,
+ displayName: 'Map Website',
+ description: 'List the links found on a website.',
audience: 'human',
outputSchema: mapWebsiteActionOutputSchema,
aiMetadata: { description: 'Discovers and returns the list of URLs reachable from a given website, optionally including subdomains, up to a configurable limit. Choose this for fast site-map / link discovery when you only need the URLs and not page content; follow with Scrape, Crawl, or Extract to fetch the pages. Read-only against the site, so repeating the call is safe.', idempotent: true },
props: {
url: Property.ShortText({
- displayName: 'Main Website URL',
- description: 'The webpage URL to start scraping from.',
+ displayName: 'Website URL',
+ description: 'The site to list links from.',
+ placeholder: 'https://example.com',
required: true,
}),
subdomain: Property.Checkbox({
- displayName: 'Include subdomain',
- description: 'Include and crawl pages from subdomains of the target website (e.g., blog.example.com, shop.example.com) in addition to the main domain.',
+ displayName: 'Include Subdomains',
+ description: 'Also list pages on subdomains, like blog.example.com.',
required: false,
- defaultValue: false
+ defaultValue: false,
}),
limit: Property.Number({
- displayName: 'Limit',
- description: 'Maximum number of links to return (max: 100,000)',
+ displayName: 'Max Links',
+ description: 'Most links to return, up to 100,000.',
required: false,
defaultValue: 5000,
}),
},
+ propertyGroups: [
+ { key: 'site', display: 'section', label: 'Site', icon: 'file', props: ['url', 'subdomain', 'limit'] },
+ ],
async run(context) {
const { auth, propsValue } = context;
diff --git a/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts b/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts
index 79ead4ac357..537bc5888cb 100644
--- a/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts
+++ b/packages/pieces/community/firecrawl/src/lib/actions/scrape.ts
@@ -1,4 +1,4 @@
-import { createAction, Property, InputPropertyMap } from '@activepieces/pieces-framework';
+import { createAction, Property, InputPropertyMap, MarkdownVariant } from '@activepieces/pieces-framework';
import { httpClient, HttpMethod } from '@activepieces/pieces-common';
import { firecrawlAuth } from '../auth';
import { forScreenshotOutputFormat, forSimpleOutputFormat, saveFirecrawlFile, forJsonOutputFormat, FIRECRAWL_API_BASE_URL, FIRECRAWL_ORIGIN } from '../common/common';
@@ -16,31 +16,20 @@ export const scrape = createAction({
name: 'scrape',
classification: 'READ',
displayName: 'Scrape Website',
- description: 'Scrape a website by performing a series of actions like clicking, typing, taking screenshots, and extracting data.',
+ description: 'Read one web page as Markdown, HTML, links, a screenshot or JSON.',
audience: 'human',
outputSchema: scrapeActionOutputSchema,
aiMetadata: { description: 'Fetches the content of a single web page and returns it in a chosen format (markdown, HTML, links, summary, screenshot, or AI-extracted JSON). Choose this to read one specific URL; for many pages use Crawl, and for a structured data pull across several known URLs use Extract Structured Data. Optionally runs browser actions (click, type, wait, screenshot) before scraping. Read-only against the target, so repeating the same call is safe.', idempotent: true },
props: {
url: Property.ShortText({
displayName: 'Website URL',
- description: 'The webpage URL to scrape.',
+ description: 'The page to read.',
+ placeholder: 'https://example.com/pricing',
required: true,
}),
- timeout: Property.Number({
- displayName: 'Timeout (ms)',
- description: 'Maximum time to wait for the page to load (in milliseconds).',
- required: false,
- defaultValue: 60000,
- }),
- storeInCache: Property.Checkbox({
- displayName: 'Store In Cache',
- description: 'If enabled, the page will be stored in the Firecrawl index and cache. Disabling this is useful if your scraping activity may have data protection concerns.',
- required: false,
- defaultValue: false,
- }),
useActions: Property.Checkbox({
- displayName: 'Perform Actions Before Scraping',
- description: 'Enable to perform a sequence of actions on the page before scraping (like clicking buttons, filling forms, etc.). See [Firecrawl Actions Documentation](https://docs.firecrawl.dev/api-reference/endpoint/scrape#body-actions) for details on available actions and their parameters.',
+ displayName: 'Run Browser Actions',
+ description: 'Click, type or wait on the page before it is read.',
required: false,
defaultValue: false,
}),
@@ -60,7 +49,7 @@ export const scrape = createAction({
return {
actions: Property.Json({
displayName: 'Actions',
- description: 'Sequence of actions to perform on the page.',
+ description: 'Steps run in order. Replace the example with your own.',
required: false,
defaultValue: [
{
@@ -88,11 +77,14 @@ export const scrape = createAction({
};
},
}),
-
+ screenshot_note: Property.MarkDown({
+ value: 'A full-page screenshot is also saved with every scrape.',
+ variant: MarkdownVariant.INFO,
+ }),
formats: Property.Dropdown({
auth: firecrawlAuth,
displayName: 'Output Format',
- description: 'Choose what format you want your output in.',
+ description: 'What the step returns for the page.',
required: true,
refreshers: [],
options: async () => {
@@ -127,7 +119,7 @@ export const scrape = createAction({
const map: InputPropertyMap = {
prompt: Property.LongText({
displayName: 'Extraction Prompt',
- description: 'Describe what information you want to extract.',
+ description: 'What to pull from the page, in plain words.',
required: false,
defaultValue: 'Extract the following data from the provided text.',
}),
@@ -152,14 +144,14 @@ export const scrape = createAction({
return {
mode: Property.StaticDropdown<'simple' | 'advanced'>({
displayName: 'Data Schema Type',
- description: 'For complex schema, you can use advanced mode.',
required: true,
defaultValue: 'simple',
+ display: 'cards',
options: {
disabled: false,
options: [
- { label: 'Simple', value: 'simple' },
- { label: 'Advanced', value: 'advanced' },
+ { label: 'Simple', value: 'simple', description: 'A list of fields', icon: 'text' },
+ { label: 'Advanced', value: 'advanced', description: 'A JSON Schema', icon: 'code' },
],
},
}),
@@ -183,8 +175,7 @@ export const scrape = createAction({
return {
fields: Property.Json({
displayName: 'JSON Schema',
- description:
- 'Learn more about JSON Schema here: https://json-schema.org/learn/getting-started-step-by-step',
+ description: 'The shape of the data to return, as a JSON Schema.',
required: true,
defaultValue: {
type: 'object',
@@ -203,24 +194,23 @@ export const scrape = createAction({
}
return {
fields: Property.Array({
- displayName: 'Data Definition',
+ displayName: 'Fields to Extract',
+ description: 'One row per value to pull from the page.',
required: true,
properties: {
name: Property.ShortText({
displayName: 'Name',
- description:
- 'Provide the name of the value you want to extract from the unstructured text. The name should be unique and short. ',
+ description: 'Letters, numbers, _ . or - only.',
+ placeholder: 'price',
required: true,
}),
description: Property.LongText({
displayName: 'Description',
- description:
- 'Brief description of the data, this hints for the AI on what to look for',
+ description: 'What the value is, as a hint for the AI.',
required: false,
}),
type: Property.StaticDropdown({
displayName: 'Data Type',
- description: 'Type of parameter.',
required: true,
defaultValue: 'string',
options: {
@@ -233,7 +223,8 @@ export const scrape = createAction({
},
}),
isRequired: Property.Checkbox({
- displayName: 'Fail if Not present?',
+ displayName: 'Required Field',
+ description: 'The AI must return a value for this field.',
required: true,
defaultValue: false,
}),
@@ -242,7 +233,25 @@ export const scrape = createAction({
};
},
}),
+ timeout: Property.Number({
+ displayName: 'Timeout',
+ description: 'Milliseconds to wait for the page to load.',
+ required: false,
+ defaultValue: 60000,
+ advanced: true,
+ }),
+ storeInCache: Property.Checkbox({
+ displayName: 'Use Cache',
+ description: 'On: Firecrawl keeps the page and may serve a cached copy.',
+ required: false,
+ defaultValue: false,
+ advanced: true,
+ }),
},
+ propertyGroups: [
+ { key: 'page', display: 'section', label: 'Page', icon: 'file', props: ['url', 'useActions', 'actionProperties'] },
+ { key: 'output', display: 'section', label: 'Output', icon: 'code', props: ['screenshot_note', 'formats', 'extractPrompt', 'extractMode', 'extractSchema'] },
+ ],
async run(context) {
const { auth, propsValue } = context;
const shouldStoreInCache = propsValue.storeInCache ?? false;
diff --git a/packages/pieces/community/firecrawl/src/lib/common/common.ts b/packages/pieces/community/firecrawl/src/lib/common/common.ts
index 7d14d2dfbbf..46ed8d515d7 100644
--- a/packages/pieces/community/firecrawl/src/lib/common/common.ts
+++ b/packages/pieces/community/firecrawl/src/lib/common/common.ts
@@ -146,12 +146,52 @@ export async function polling(
if (jobStatus === 'completed') {
return statusResponse.body;
} else if (jobStatus === 'failed') {
- throw new Error(`${actionType.charAt(0).toUpperCase()}. job failed: ${JSON.stringify(statusResponse.body)}`);
+ throw new Error(`${JOB_NAMES[actionType]} job failed: ${JSON.stringify(statusResponse.body)}`);
}
}
// exit loop time out
- throw new Error(`${actionType.charAt(0).toUpperCase()}. job timed out after ${timeoutSeconds} second(s)`);
+ throw new Error(`${JOB_NAMES[actionType]} job timed out after ${timeoutSeconds} seconds`);
}
+export async function followCrawlPages({
+ firstPage,
+ auth,
+}: {
+ firstPage: CrawlStatusPage;
+ auth: string;
+}): Promise<CrawlStatusPage> {
+ let data = firstPage.data ?? [];
+ let next = firstPage.next;
+ const visited = new Set<string>();
+ while (isFirecrawlApiUrl(next) && !visited.has(next)) {
+ visited.add(next);
+ const response = await httpClient.sendRequest<CrawlStatusPage>({
+ method: HttpMethod.GET,
+ url: next,
+ headers: {
+ 'Authorization': `Bearer ${auth}`,
+ },
+ });
+ data = [...data, ...(response.body.data ?? [])];
+ next = response.body.next;
+ }
+ return { ...firstPage, data, next: null };
+}
+
+function isFirecrawlApiUrl(url: string | null | undefined): url is string {
+ return typeof url === 'string' && url.startsWith(`${FIRECRAWL_API_BASE_URL}/`);
+}
+
+const JOB_NAMES: Record<'extract' | 'crawl', string> = {
+ extract: 'Extract',
+ crawl: 'Crawl',
+};
+
export const FIRECRAWL_ORIGIN = 'activepieces';
+
+export type CrawlStatusPage = {
+ data?: Record<string, unknown>[];
+ next?: string | null;
+ [key: string]: unknown;
+};
diff --git a/packages/pieces/community/firecrawl/test/follow-crawl-pages.test.ts b/packages/pieces/community/firecrawl/test/follow-crawl-pages.test.ts
new file mode 100644
index 00000000000..bfbec11e3ee
--- /dev/null
+++ b/packages/pieces/community/firecrawl/test/follow-crawl-pages.test.ts
@@ -0,0 +1,108 @@
+import '../src';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { followCrawlPages } from '../src/lib/common/common';
+
+const sendRequest = vi.fn();
+
+vi.mock('@activepieces/pieces-common', async (importOriginal) => {
+ const actual = await importOriginal<typeof import('@activepieces/pieces-common')>();
+ return {
+ ...actual,
+ httpClient: {
+ sendRequest: (...args: unknown[]) => sendRequest(...args),
+ },
+ };
+});
+
+const BASE = 'https://api.firecrawl.dev/v2/crawl/job-1';
+
+function respondWith({ pages }: { pages: Record<string, { data: Record<string, unknown>[]; next: string | null }> }) {
+ sendRequest.mockImplementation(async ({ url }: { url: string }) => {
+ const page = pages[url];
+ if (!page) {
+ throw new Error(`unexpected request to ${url}`);
+ }
+ return { status: 200, headers: {}, body: page };
+ });
+}
+
+describe('followCrawlPages', () => {
+ beforeEach(() => {
+ sendRequest.mockReset();
+ });
+
+ it('collects every page until next is empty', async () => {
+ respondWith({
+ pages: {
+ [`${BASE}?skip=1`]: { data: [{ url: 'b' }], next: `${BASE}?skip=2` },
+ [`${BASE}?skip=2`]: { data: [{ url: 'c' }], next: null },
+ },
+ });
+
+ const result = await followCrawlPages({
+ firstPage: { status: 'completed', total: 3, data: [{ url: 'a' }], next: `${BASE}?skip=1` },
+ auth: 'key',
+ });
+
+ expect(result.data).toEqual([{ url: 'a' }, { url: 'b' }, { url: 'c' }]);
+ expect(result.next).toBeNull();
+ expect(result['status']).toBe('completed');
+ expect(result['total']).toBe(3);
+ expect(sendRequest).toHaveBeenCalledTimes(2);
+ expect(sendRequest.mock.calls[0][0].headers).toEqual({ Authorization: 'Bearer key' });
+ });
+
+ it('stops when the next links form a cycle', async () => {
+ respondWith({
+ pages: {
+ [`${BASE}?skip=1`]: { data: [{ url: 'b' }], next: `${BASE}?skip=2` },
+ [`${BASE}?skip=2`]: { data: [{ url: 'c' }], next: `${BASE}?skip=1` },
+ },
+ });
+
+ const result = await followCrawlPages({
+ firstPage: { data: [{ url: 'a' }], next: `${BASE}?skip=1` },
+ auth: 'key',
+ });
+
+ expect(result.data).toEqual([{ url: 'a' }, { url: 'b' }, { url: 'c' }]);
+ expect(sendRequest).toHaveBeenCalledTimes(2);
+ });
+
+ it('stops when next points to itself', async () => {
+ respondWith({
+ pages: {
+ [`${BASE}?skip=1`]: { data: [{ url: 'b' }], next: `${BASE}?skip=1` },
+ },
+ });
+
+ const result = await followCrawlPages({
+ firstPage: { data: [{ url: 'a' }], next: `${BASE}?skip=1` },
+ auth: 'key',
+ });
+
+ expect(result.data).toEqual([{ url: 'a' }, { url: 'b' }]);
+ expect(sendRequest).toHaveBeenCalledTimes(1);
+ });
+
+ it('never sends the key to a host outside the Firecrawl API', async () => {
+ const result = await followCrawlPages({
+ firstPage: { data: [{ url: 'a' }], next: 'https://evil.example.com/v2/crawl/job-1?skip=1' },
+ auth: 'key',
+ });
+
+ expect(result.data).toEqual([{ url: 'a' }]);
+ expect(result.next).toBeNull();
+ expect(sendRequest).not.toHaveBeenCalled();
+ });
+
+ it('returns the first page when there is no next link', async () => {
+ const result = await followCrawlPages({
+ firstPage: { status: 'scraping', data: [{ url: 'a' }] },
+ auth: 'key',
+ });
+
+ expect(result.data).toEqual([{ url: 'a' }]);
+ expect(sendRequest).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/pieces/community/firecrawl/vitest.config.ts b/packages/pieces/community/firecrawl/vitest.config.ts
new file mode 100644
index 00000000000..79c97261904
--- /dev/null
+++ b/packages/pieces/community/firecrawl/vitest.config.ts
@@ -0,0 +1,19 @@
+import path from 'path'
+import { defineConfig } from 'vitest/config'
+
+const repoRoot = path.resolve(__dirname, '../../../..')
+
+export default defineConfig({
+ root: __dirname,
+ test: {
+ globals: true,
+ environment: 'node',
+ include: ['test/**/*.test.ts'],
+ },
+ resolve: {
+ alias: {
+ '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'),
+ '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'),
+ },
+ },
+})