Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

feat(browserless): scrape, search, crawl and capture pages from flows and AI agents (#16215)

Odai Ahmadc9719860d453
diff --git a/bun.lock b/bun.lock
index dc695145beb..56efd082399 100644
--- a/bun.lock
+++ b/bun.lock
@@ -1404,7 +1404,7 @@
},
"packages/pieces/community/browserless": {
"name": "@activepieces/piece-browserless",
- "version": "0.1.8",
+ "version": "0.2.0",
"dependencies": {
"@activepieces/core-piece-types": "workspace:*",
"@activepieces/core-utils": "workspace:*",
@@ -1413,6 +1413,7 @@
},
"devDependencies": {
"tslib": "^2.3.0",
+ "vitest": "3.2.6",
},
},
"packages/pieces/community/bubble": {
diff --git a/packages/pieces/community/browserless/package.json b/packages/pieces/community/browserless/package.json
index 6de9eed49a2..0d232297a71 100644
--- a/packages/pieces/community/browserless/package.json
+++ b/packages/pieces/community/browserless/package.json
@@ -1,6 +1,6 @@
{
"name": "@activepieces/piece-browserless",
- "version": "0.1.8",
+ "version": "0.2.0",
"type": "commonjs",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
@@ -11,11 +11,13 @@
"@activepieces/core-utils": "workspace:*"
},
"devDependencies": {
- "tslib": "^2.3.0"
+ "tslib": "^2.3.0",
+ "vitest": "3.2.6"
},
"scripts": {
"build": "tsc -p tsconfig.lib.json && cp package.json dist/",
"bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle",
- "lint": "eslint 'src/**/*.ts'"
+ "lint": "eslint 'src/**/*.ts'",
+ "test": "vitest run"
}
}
diff --git a/packages/pieces/community/browserless/src/i18n/translation.json b/packages/pieces/community/browserless/src/i18n/translation.json
index f74284bfc3a..4010cb1e0ce 100644
--- a/packages/pieces/community/browserless/src/i18n/translation.json
+++ b/packages/pieces/community/browserless/src/i18n/translation.json
@@ -5,7 +5,7 @@
"Custom Base URL": "Custom Base URL",
"Your Browserless API token (found in your dashboard)": "Your Browserless API token (found in your dashboard)",
"Choose the regional endpoint closest to you for optimal performance": "Choose the regional endpoint closest to you for optimal performance",
- "Enter your custom Browserless endpoint URL": "Enter your custom Browserless endpoint URL",
+ "Only for \"Custom Endpoint\": the address of your dedicated or self-hosted Browserless, for example https://chrome.browserless.io": "Only for \"Custom Endpoint\": the address of your dedicated or self-hosted Browserless, for example https://chrome.browserless.io",
"US West (San Francisco)": "US West (San Francisco)",
"Europe UK (London)": "Europe UK (London)",
"Europe (Amsterdam)": "Europe (Amsterdam)",
@@ -16,11 +16,39 @@
"Scrape URL": "Scrape URL",
"Run BQL Query": "Run BQL Query",
"Get Website Performance": "Get Website Performance",
+ "Get Page Content": "Get Page Content",
+ "Smart Scrape": "Smart Scrape",
+ "Search the Web": "Search the Web",
+ "Map Website URLs": "Map Website URLs",
+ "Start Crawl": "Start Crawl",
+ "Get Crawl Results": "Get Crawl Results",
+ "List Crawls": "List Crawls",
+ "Cancel Crawl": "Cancel Crawl",
+ "Unblock Page": "Unblock Page",
+ "Run Puppeteer Function": "Run Puppeteer Function",
+ "Take Screenshot (File Only)": "Take Screenshot (File Only)",
+ "Create PDF (File Only)": "Create PDF (File Only)",
+ "Audit Page Performance (Summary)": "Audit Page Performance (Summary)",
+ "Custom API Call": "Custom API Call",
"Take a screenshot of a web page": "Take a screenshot of a web page",
"Convert a web page to PDF": "Convert a web page to PDF",
"Extract content from a web page": "Extract content from a web page",
"Execute Browser Query Language (BQL) GraphQL-based queries for advanced browser automation": "Execute Browser Query Language (BQL) GraphQL-based queries for advanced browser automation",
"Analyze website performance metrics using Lighthouse": "Analyze website performance metrics using Lighthouse",
+ "Load a page in a real browser and return its fully rendered HTML, including content added by JavaScript.": "Load a page in a real browser and return its fully rendered HTML, including content added by JavaScript.",
+ "Get a page as markdown, text, HTML or links. Browserless tries a fast fetch first and falls back to a real browser, proxy or CAPTCHA solving when the site blocks it.": "Get a page as markdown, text, HTML or links. Browserless tries a fast fetch first and falls back to a real browser, proxy or CAPTCHA solving when the site blocks it.",
+ "Search the web (pages, news or images) and optionally read each result page as markdown.": "Search the web (pages, news or images) and optionally read each result page as markdown.",
+ "List the URLs of a website from its sitemap and links, optionally ordered by relevance to a search term.": "List the URLs of a website from its sitemap and links, optionally ordered by relevance to a search term.",
+ "Start crawling a website in the background. Browserless follows links and scrapes every page it finds; use Get Crawl Results to read them.": "Start crawling a website in the background. Browserless follows links and scrapes every page it finds; use Get Crawl Results to read them.",
+ "Get the status of a crawl and one page of its scraped pages.": "Get the status of a crawl and one page of its scraped pages.",
+ "List your recent crawls with their status and progress.": "List your recent crawls with their status and progress.",
+ "Stop a running crawl. Pages already scraped stay available.": "Stop a running crawl. Pages already scraped stay available.",
+ "Open a page that blocks bots (Cloudflare, CAPTCHAs and similar) and return its HTML, cookies and a screenshot.": "Open a page that blocks bots (Cloudflare, CAPTCHAs and similar) and return its HTML, cookies and a screenshot.",
+ "Run your own Puppeteer code in a Browserless browser and return what it returns.": "Run your own Puppeteer code in a Browserless browser and return what it returns.",
+ "Screenshot a web page or HTML and return only the stored image file.": "Screenshot a web page or HTML and return only the stored image file.",
+ "Render a web page or HTML to PDF and return only the stored file.": "Render a web page or HTML to PDF and return only the stored file.",
+ "Run a Lighthouse audit and return only the scores, Core Web Vitals and top opportunities.": "Run a Lighthouse audit and return only the scores, Core Web Vitals and top opportunities.",
+ "Make a custom API call to a specific endpoint": "Make a custom API call to a specific endpoint",
"URL": "URL",
"Image Type": "Image Type",
"Quality": "Quality",
@@ -91,9 +119,65 @@
"Locale": "Locale",
"Emulate Media Type": "Emulate Media Type",
"Performance Budgets": "Performance Budgets",
+ "Include Full Report": "Include Full Report",
+ "Page Load Timeout (ms)": "Page Load Timeout (ms)",
+ "Extra Wait (ms)": "Extra Wait (ms)",
+ "Skip Resource Types": "Skip Resource Types",
+ "Maximum Characters": "Maximum Characters",
+ "Formats": "Formats",
+ "Only Main Content": "Only Main Content",
+ "Keep Only These Selectors": "Keep Only These Selectors",
+ "Remove These Selectors": "Remove These Selectors",
+ "Proxy": "Proxy",
+ "Search Query": "Search Query",
+ "Sources": "Sources",
+ "Results per Source": "Results per Source",
+ "Time Range": "Time Range",
+ "Categories": "Categories",
+ "Language": "Language",
+ "Country": "Country",
+ "Location": "Location",
+ "Read Result Pages As": "Read Result Pages As",
+ "Search Term": "Search Term",
+ "Maximum URLs": "Maximum URLs",
+ "Sitemap": "Sitemap",
+ "Include Subdomains": "Include Subdomains",
+ "Ignore Query Parameters": "Ignore Query Parameters",
+ "Maximum Pages": "Maximum Pages",
+ "Maximum Link Depth": "Maximum Link Depth",
+ "Only Paths Matching": "Only Paths Matching",
+ "Skip Paths Matching": "Skip Paths Matching",
+ "Follow Subdomains": "Follow Subdomains",
+ "Follow External Links": "Follow External Links",
+ "Page Formats": "Page Formats",
+ "Crawl ID": "Crawl ID",
+ "Skip": "Skip",
+ "Status": "Status",
+ "Limit": "Limit",
+ "Cursor": "Cursor",
+ "Return HTML": "Return HTML",
+ "Return Cookies": "Return Cookies",
+ "Return Screenshot": "Return Screenshot",
+ "Solve CAPTCHAs": "Solve CAPTCHAs",
+ "Markdown": "Markdown",
+ "Code": "Code",
+ "Context": "Context",
+ "HTML": "HTML",
+ "Element Selector": "Element Selector",
+ "Margin": "Margin",
+ "Device": "Device",
+ "Method": "Method",
+ "Headers": "Headers",
+ "Query Parameters": "Query Parameters",
+ "Body Type": "Body Type",
+ "Body": "Body",
+ "Response is Binary": "Response is Binary",
+ "Return Error as Output": "Return Error as Output",
+ "Timeout": "Timeout",
+ "Follow redirects": "Follow redirects",
"The URL of the page to capture": "The URL of the page to capture",
"Format of the screenshot image": "Format of the screenshot image",
- "Image quality (0-100, only for JPEG)": "Image quality (0-100, only for JPEG)",
+ "Image quality (0-100, only for JPEG and WebP)": "Image quality (0-100, only for JPEG and WebP)",
"Capture the full scrollable page": "Capture the full scrollable page",
"Width of the browser viewport in pixels": "Width of the browser viewport in pixels",
"Height of the browser viewport in pixels": "Height of the browser viewport in pixels",
@@ -128,7 +212,7 @@
"Hide default white background and allow transparent PDFs": "Hide default white background and allow transparent PDFs",
"Generate tagged (accessible) PDF": "Generate tagged (accessible) PDF",
"Generate document outline": "Generate document outline",
- "Maximum time to wait for the page to load": "Maximum time to wait for the page to load",
+ "Maximum time in milliseconds to wait for the page to load": "Maximum time in milliseconds to wait for the page to load",
"JavaScript function to wait for before generating PDF": "JavaScript function to wait for before generating PDF",
"Polling interval for wait function (\"raf\", \"mutation\", or number in ms)": "Polling interval for wait function (\"raf\", \"mutation\", or number in ms)",
"Timeout in milliseconds for wait function (0 to disable)": "Timeout in milliseconds for wait function (0 to disable)",
@@ -146,6 +230,7 @@
"Include network requests in debug output": "Include network requests in debug output",
"Custom user agent string": "Custom user agent string",
"Cookies to set before scraping": "Cookies to set before scraping",
+ "Maximum time to wait for the page to load": "Maximum time to wait for the page to load",
"When to consider navigation complete": "When to consider navigation complete",
"JavaScript function to wait for before scraping (should return true when ready)": "JavaScript function to wait for before scraping (should return true when ready)",
"Browser viewport width in pixels": "Browser viewport width in pixels",
@@ -169,18 +254,103 @@
"Browser viewport height": "Browser viewport height",
"Cookies to set before executing BQL query": "Cookies to set before executing BQL query",
"The URL of the website to analyze": "The URL of the website to analyze",
- "Select which performance categories to analyze (performance, accessibility, best-practices, seo, pwa)": "Select which performance categories to analyze (performance, accessibility, best-practices, seo, pwa)",
+ "Categories to audit (Lighthouse 12+ has no PWA category).": "Categories to audit (Lighthouse 12+ has no PWA category).",
"Device type for performance analysis": "Device type for performance analysis",
"Network throttling simulation": "Network throttling simulation",
"Return only category scores without detailed audit results": "Return only category scores without detailed audit results",
"Locale for the analysis (e.g., en-US, de-DE)": "Locale for the analysis (e.g., en-US, de-DE)",
- "Maximum time to wait for the analysis to complete": "Maximum time to wait for the analysis to complete",
- "CSS selector to wait for before running performance analysis": "CSS selector to wait for before running performance analysis",
- "Emulate CSS media type": "Emulate CSS media type",
+ "Max page load wait in ms during the audit (under 480000).": "Max page load wait in ms during the audit (under 480000).",
+ "Not supported by Lighthouse audits; this setting is ignored.": "Not supported by Lighthouse audits; this setting is ignored.",
"Lighthouse performance budgets for resource sizes": "Lighthouse performance budgets for resource sizes",
"Enable ad blocker during performance analysis": "Enable ad blocker during performance analysis",
+ "Also return the full Lighthouse report (often several MB).": "Also return the full Lighthouse report (often several MB).",
+ "The page to load, for example https://example.com/pricing": "The page to load, for example https://example.com/pricing",
+ "When the page counts as loaded. \"Network Idle\" waits longest and suits pages that load data with JavaScript.": "When the page counts as loaded. \"Network Idle\" waits longest and suits pages that load data with JavaScript.",
+ "How long to wait for the page to load before failing (Browserless default 30000).": "How long to wait for the page to load before failing (Browserless default 30000).",
+ "A CSS selector (for example `#content` or `.price`) to wait for before reading the page.": "A CSS selector (for example `#content` or `.price`) to wait for before reading the page.",
+ "Extra time to wait after the page loads, for animations or late content.": "Extra time to wait after the page loads, for animations or late content.",
+ "Resource types the browser should not download, which makes the page load faster.": "Resource types the browser should not download, which makes the page load faster.",
+ "Continue with whatever has loaded when a wait or the page load times out, instead of failing.": "Continue with whatever has loaded when a wait or the page load times out, instead of failing.",
+ "Load the page with an ad blocker, which is usually faster.": "Load the page with an ad blocker, which is usually faster.",
+ "Cut long text fields to this many characters (default 100000, 0 = no limit).": "Cut long text fields to this many characters (default 100000, 0 = no limit).",
+ "Maximum time the whole request may take on Browserless. Browser time is billed per 30 seconds, so a lower limit caps cost. Keep it under 540000.": "Maximum time the whole request may take on Browserless. Browser time is billed per 30 seconds, so a lower limit caps cost. Keep it under 540000.",
+ "The page to read, for example https://example.com/blog/post": "The page to read, for example https://example.com/blog/post",
+ "What to return. Markdown is the best choice for AI steps.": "What to return. Markdown is the best choice for AI steps.",
+ "Drop navigation, headers, footers and sidebars.": "Drop navigation, headers, footers and sidebars.",
+ "CSS selectors to keep; cannot be combined with Remove or Only Main Content.": "CSS selectors to keep; cannot be combined with Remove or Only Main Content.",
+ "CSS selectors to remove (for example `.cookie-banner`).": "CSS selectors to remove (for example `.cookie-banner`).",
+ "Wait after load (0-30000 ms); above 0 forces a real browser.": "Wait after load (0-30000 ms); above 0 forces a real browser.",
+ "Proxy network used when a plain fetch is blocked. Browserless uses Residential when empty; Datacenter is cheaper.": "Proxy network used when a plain fetch is blocked. Browserless uses Residential when empty; Datacenter is cheaper.",
+ "What to search for, for example `best headless browser 2026`.": "What to search for, for example `best headless browser 2026`.",
+ "Where to search. Web is used when empty.": "Where to search. Web is used when empty.",
+ "Results per source (default 10, capped by your plan).": "Results per source (default 10, capped by your plan).",
+ "Only return results from this period.": "Only return results from this period.",
+ "Narrow results to GitHub repositories, research papers or PDF documents.": "Narrow results to GitHub repositories, research papers or PDF documents.",
+ "Two-letter language code for results, for example `en` or `de` (default `en`).": "Two-letter language code for results, for example `en` or `de` (default `en`).",
+ "Two-letter country code for local results, for example `us` or `gb`.": "Two-letter country code for local results, for example `us` or `gb`.",
+ "City or region to narrow results, used with Country.": "City or region to narrow results, used with Country.",
+ "Also read each result page in these formats (slower, costs more).": "Also read each result page in these formats (slower, costs more).",
+ "When reading result pages, drop navigation, headers and footers.": "When reading result pages, drop navigation, headers and footers.",
+ "Proxy network used to read result pages. Browserless uses Residential when empty; Datacenter is cheaper.": "Proxy network used to read result pages. Browserless uses Residential when empty; Datacenter is cheaper.",
+ "The website to map, for example https://example.com": "The website to map, for example https://example.com",
+ "Order the URLs by relevance to this term, for example `pricing`.": "Order the URLs by relevance to this term, for example `pricing`.",
+ "How many URLs to return (1-5000, default 500).": "How many URLs to return (1-5000, default 500).",
+ "How to use the site's sitemap.": "How to use the site's sitemap.",
+ "Also return URLs on subdomains such as blog.example.com.": "Also return URLs on subdomains such as blog.example.com.",
+ "Skip URLs that only differ by their ?query part.": "Skip URLs that only differ by their ?query part.",
+ "Proxy network for fetching the site. Browserless uses Residential when empty; Datacenter is cheaper.": "Proxy network for fetching the site. Browserless uses Residential when empty; Datacenter is cheaper.",
+ "Where the crawl starts, for example https://example.com/docs": "Where the crawl starts, for example https://example.com/docs",
+ "Stop after this many pages (capped by your plan); each page costs units.": "Stop after this many pages (capped by your plan); each page costs units.",
+ "How many links away from the start URL to follow (0-20, default 5).": "How many links away from the start URL to follow (0-20, default 5).",
+ "Regexes; only matching paths are crawled, e.g. `^/docs/`.": "Regexes; only matching paths are crawled, e.g. `^/docs/`.",
+ "Regular expressions for paths to skip, for example `^/blog/`.": "Regular expressions for paths to skip, for example `^/blog/`.",
+ "Also crawl subdomains such as blog.example.com.": "Also crawl subdomains such as blog.example.com.",
+ "Also follow links to other websites.": "Also follow links to other websites.",
+ "How to use the sitemap to find pages.": "How to use the sitemap to find pages.",
+ "What to store for each page (default markdown).": "What to store for each page (default markdown).",
+ "Drop navigation, headers and footers from each page (Browserless default is on).": "Drop navigation, headers and footers from each page (Browserless default is on).",
+ "Proxy network for fetching pages. Browserless uses Residential when empty; Datacenter is cheaper.": "Proxy network for fetching pages. Browserless uses Residential when empty; Datacenter is cheaper.",
+ "The ID returned by Start Crawl (looks like `crawl_abc123`). List Crawls shows recent IDs.": "The ID returned by Start Crawl (looks like `crawl_abc123`). List Crawls shows recent IDs.",
+ "Pages to skip; use Next Skip from the previous run.": "Pages to skip; use Next Skip from the previous run.",
+ "Only list crawls with this status.": "Only list crawls with this status.",
+ "How many crawls to return in this batch.": "How many crawls to return in this batch.",
+ "Next Cursor from a previous run, to read the next batch.": "Next Cursor from a previous run, to read the next batch.",
+ "The blocked page to open, for example https://example.com/products": "The blocked page to open, for example https://example.com/products",
+ "Return the page HTML.": "Return the page HTML.",
+ "Return the cookies the site set, so later requests can reuse the unblocked session.": "Return the cookies the site set, so later requests can reuse the unblocked session.",
+ "Return a full-page screenshot as a file.": "Return a full-page screenshot as a file.",
+ "Let Browserless solve CAPTCHAs it meets (10 units per successful solve).": "Let Browserless solve CAPTCHAs it meets (10 units per successful solve).",
+ "Proxy network to route the browser through. Residential works best against bot detection.": "Proxy network to route the browser through. Residential works best against bot detection.",
+ "Two-letter country code for the proxy exit, for example `us` or `de`.": "Two-letter country code for the proxy exit, for example `us` or `de`.",
+ "Write an ES module whose default export is an async function. It receives `{ page, context }` and should return `{ data, type }`, for example:\n\n```js\nexport default async function ({ page, context }) {\n await page.goto(context.url);\n return { data: { title: await page.title() }, type: \"application/json\" };\n}\n```": "Write an ES module whose default export is an async function. It receives `{ page, context }` and should return `{ data, type }`, for example:\n\n```js\nexport default async function ({ page, context }) {\n await page.goto(context.url);\n return { data: { title: await page.title() }, type: \"application/json\" };\n}\n```",
+ "The Puppeteer module to run.": "The Puppeteer module to run.",
+ "Values your function receives as `context`.": "Values your function receives as `context`.",
+ "The page to capture, for example https://example.com. Leave empty when you pass HTML.": "The page to capture, for example https://example.com. Leave empty when you pass HTML.",
+ "Raw HTML to render instead of a URL.": "Raw HTML to render instead of a URL.",
+ "Image format (default PNG).": "Image format (default PNG).",
+ "Capture the whole scrollable page instead of the visible viewport.": "Capture the whole scrollable page instead of the visible viewport.",
+ "JPEG/WebP quality 0-100.": "JPEG/WebP quality 0-100.",
+ "Browser width in pixels (set together with height).": "Browser width in pixels (set together with height).",
+ "Browser height in pixels (set together with width).": "Browser height in pixels (set together with width).",
+ "Capture only the element matching this CSS selector.": "Capture only the element matching this CSS selector.",
+ "The page to convert, for example https://example.com. Leave empty when you pass HTML.": "The page to convert, for example https://example.com. Leave empty when you pass HTML.",
+ "Paper size (default A4).": "Paper size (default A4).",
+ "Use landscape orientation.": "Use landscape orientation.",
+ "Include background colours and images (default on).": "Include background colours and images (default on).",
+ "One margin for all sides, for example `10mm` or `0.5in`.": "One margin for all sides, for example `10mm` or `0.5in`.",
+ "The page to audit, for example https://example.com": "The page to audit, for example https://example.com",
+ "Audit as a desktop (default) or mobile visitor.": "Audit as a desktop (default) or mobile visitor.",
+ "Which scores to compute (all four when empty).": "Which scores to compute (all four when empty).",
+ "Max page load wait during the audit (default 45000).": "Max page load wait during the audit (default 45000).",
+ "Authorization headers are injected automatically from your connection.": "Authorization headers are injected automatically from your connection.",
+ "Appended to the URL as ?key=value.": "Appended to the URL as ?key=value.",
+ "Enable for files like PDFs, images, etc.": "Enable for files like PDFs, images, etc.",
+ "On a failed request, output the error instead of failing the step.": "On a failed request, output the error instead of failing the step.",
+ "Seconds to wait for a response. Empty: up to the flow limit (10 min).": "Seconds to wait for a response. Empty: up to the flow limit (10 min).",
+ "Follow 3xx redirects instead of returning them as the response.": "Follow 3xx redirects instead of returning them as the response.",
"PNG": "PNG",
"JPEG": "JPEG",
+ "WebP": "WebP",
"A0": "A0",
"A1": "A1",
"A2": "A2",
@@ -205,5 +375,44 @@
"Regular 4G": "Regular 4G",
"Fast 4G": "Fast 4G",
"Screen": "Screen",
- "Print": "Print"
+ "Print": "Print",
+ "Network Idle (0 connections)": "Network Idle (0 connections)",
+ "Network Idle (2 connections)": "Network Idle (2 connections)",
+ "Images": "Images",
+ "Stylesheets": "Stylesheets",
+ "Fonts": "Fonts",
+ "Media (video/audio)": "Media (video/audio)",
+ "Plain Text": "Plain Text",
+ "Links": "Links",
+ "Screenshot (file)": "Screenshot (file)",
+ "PDF (file)": "PDF (file)",
+ "Datacenter (2 units/MB)": "Datacenter (2 units/MB)",
+ "Residential (6 units/MB)": "Residential (6 units/MB)",
+ "Web": "Web",
+ "News": "News",
+ "Past Day": "Past Day",
+ "Past Week": "Past Week",
+ "Past Month": "Past Month",
+ "Past Year": "Past Year",
+ "GitHub": "GitHub",
+ "Research": "Research",
+ "PDF": "PDF",
+ "Use sitemap and page links (default)": "Use sitemap and page links (default)",
+ "Only the sitemap": "Only the sitemap",
+ "Ignore the sitemap": "Ignore the sitemap",
+ "Auto (default)": "Auto (default)",
+ "In Progress": "In Progress",
+ "Completed": "Completed",
+ "Failed": "Failed",
+ "Cancelled": "Cancelled",
+ "Performance": "Performance",
+ "Accessibility": "Accessibility",
+ "Best Practices": "Best Practices",
+ "SEO": "SEO",
+ "GET": "GET",
+ "POST": "POST",
+ "PATCH": "PATCH",
+ "PUT": "PUT",
+ "DELETE": "DELETE",
+ "HEAD": "HEAD"
}
\ No newline at end of file
diff --git a/packages/pieces/community/browserless/src/index.ts b/packages/pieces/community/browserless/src/index.ts
index 0ce86a3f8b9..9c168af6dfc 100644
--- a/packages/pieces/community/browserless/src/index.ts
+++ b/packages/pieces/community/browserless/src/index.ts
@@ -1,15 +1,29 @@
-import { createPiece } from '@activepieces/pieces-framework';
-import { PieceCategory } from '@activepieces/pieces-framework';
-import { browserlessAuth } from './lib/common/auth';
+import { createCustomApiCallAction } from '@activepieces/pieces-common';
+import { PieceCategory, createPiece } from '@activepieces/pieces-framework';
+import { auditPagePerformance } from './lib/actions/ai/audit-page-performance';
+import { createPdf } from './lib/actions/ai/create-pdf';
+import { takeScreenshot } from './lib/actions/ai/take-screenshot';
+import { cancelCrawl } from './lib/actions/cancel-crawl';
import { captureScreenshot } from './lib/actions/capture-screenshot';
import { generatePdf } from './lib/actions/generate-pdf';
-import { scrapeUrl } from './lib/actions/scrape-url';
-import { runBqlQuery } from './lib/actions/run-bql-query';
+import { getCrawl } from './lib/actions/get-crawl';
+import { getPageContent } from './lib/actions/get-page-content';
import { getWebsitePerformance } from './lib/actions/get-website-performance';
+import { listCrawls } from './lib/actions/list-crawls';
+import { mapWebsite } from './lib/actions/map-website';
+import { runBqlQuery } from './lib/actions/run-bql-query';
+import { runFunction } from './lib/actions/run-function';
+import { scrapeUrl } from './lib/actions/scrape-url';
+import { searchWeb } from './lib/actions/search-web';
+import { smartScrape } from './lib/actions/smart-scrape';
+import { startCrawl } from './lib/actions/start-crawl';
+import { unblockPage } from './lib/actions/unblock-page';
+import { browserlessAuth } from './lib/common/auth';
+import { browserlessApi } from './lib/common/client';
export const browserless = createPiece({
displayName: 'Browserless',
- minimumSupportedRelease: '0.36.1',
+ minimumSupportedRelease: '0.88.2',
logoUrl: 'https://cdn.activepieces.com/pieces/browserless.png',
categories: [PieceCategory.DEVELOPER_TOOLS],
description: 'Browserless is a headless browser automation tool that allows you to scrape websites, take screenshots, and more.',
@@ -21,6 +35,42 @@ export const browserless = createPiece({
scrapeUrl,
runBqlQuery,
getWebsitePerformance,
+ getPageContent,
+ smartScrape,
+ searchWeb,
+ mapWebsite,
+ startCrawl,
+ getCrawl,
+ listCrawls,
+ cancelCrawl,
+ unblockPage,
+ runFunction,
+ takeScreenshot,
+ createPdf,
+ auditPagePerformance,
+ createCustomApiCallAction({
+ auth: browserlessAuth,
+ baseUrl: (auth) => (auth ? browserlessApi.resolveBaseUrl(auth.props) : ''),
+ authLocation: 'queryParams',
+ authMapping: async (auth, propsValue) => {
+ const baseUrl = browserlessApi.resolveBaseUrl(auth.props);
+ const url = readUrlProp(propsValue);
+ if (/^[a-z][a-z0-9+.-]*:/i.test(url) || url.startsWith('//')) {
+ if (!browserlessApi.isSameTarget({ baseUrl, url })) {
+ throw new Error(`Custom API Call only sends your Browserless token to ${baseUrl}. Use a path such as /meta instead of another host.`);
+ }
+ }
+ return { token: auth.props.apiToken.trim() };
+ },
+ }),
],
triggers: [],
-});
\ No newline at end of file
+});
+
+function readUrlProp(propsValue: Record<string, unknown>): string {
+ const holder = propsValue['url'];
+ if (typeof holder === 'object' && holder !== null && 'url' in holder && typeof holder.url === 'string') {
+ return holder.url.trim();
+ }
+ return '';
+}
diff --git a/packages/pieces/community/browserless/src/lib/actions/ai/audit-page-performance.ts b/packages/pieces/community/browserless/src/lib/actions/ai/audit-page-performance.ts
new file mode 100644
index 00000000000..484b835ef87
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/ai/audit-page-performance.ts
@@ -0,0 +1,93 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../../common/auth';
+import { browserlessApi } from '../../common/client';
+import { browserlessLighthouse } from '../../common/lighthouse';
+import { browserlessBody, browserlessProps } from '../../common/props';
+import { browserlessOutputSchemas } from '../../output-schemas';
+
+export const auditPagePerformance = createAction({
+ auth: browserlessAuth,
+ name: 'audit_page_performance',
+ classification: 'READ',
+ displayName: 'Audit Page Performance (Summary)',
+ description: 'Run a Lighthouse audit and return only the scores, Core Web Vitals and top opportunities.',
+ audience: 'ai',
+ aiMetadata: {
+ description:
+ 'Runs a Google Lighthouse audit on a public URL and returns only a compact summary: 0-100 scores for performance, accessibility, best practices and SEO, Core Web Vitals and the main savings opportunities (never the multi-MB raw report). Use to judge page speed or SEO quality. Takes 20-60 seconds; scores vary slightly between runs, so a retry is safe.',
+ idempotent: true,
+ },
+ props: {
+ url: browserlessProps.url({ required: true, description: 'The page to audit, for example https://example.com' }),
+ device: Property.StaticDropdown({
+ displayName: 'Device',
+ description: 'Audit as a desktop (default) or mobile visitor.',
+ required: false,
+ options: {
+ options: [
+ { label: 'Desktop', value: 'desktop' },
+ { label: 'Mobile', value: 'mobile' },
+ ],
+ },
+ }),
+ categories: Property.StaticMultiSelectDropdown({
+ displayName: 'Categories',
+ description: 'Which scores to compute (all four when empty).',
+ required: false,
+ options: {
+ options: [
+ { label: 'Performance', value: 'performance' },
+ { label: 'Accessibility', value: 'accessibility' },
+ { label: 'Best Practices', value: 'best-practices' },
+ { label: 'SEO', value: 'seo' },
+ ],
+ },
+ }),
+ timeout: Property.Number({
+ displayName: 'Page Load Timeout (ms)',
+ description: 'Max page load wait during the audit (default 45000).',
+ required: false,
+ }),
+ },
+ outputSchema: browserlessOutputSchemas.auditPagePerformance,
+ async run(context) {
+ const props = context.propsValue;
+ const url = browserlessBody.httpUrl({ value: props.url, label: 'URL' });
+ const device = props.device === 'mobile' ? 'mobile' : 'desktop';
+ const categories = browserlessBody.textList(props.categories);
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Page Load Timeout', min: 1000, max: 480_000 }) ?? 45_000;
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/performance',
+ body: {
+ url,
+ config: {
+ extends: 'lighthouse:default',
+ settings: browserlessLighthouse.buildSettings({
+ device,
+ onlyCategories: categories.length > 0 ? Array.from(new Set(categories)) : browserlessLighthouse.DEFAULT_CATEGORIES,
+ timeout,
+ }),
+ },
+ },
+ timeoutMs: timeout + 60_000,
+ operation: 'Audit Page Performance',
+ });
+
+ const report = browserlessLighthouse.summarize({ body: response.body, url, device });
+ const { scores, ...rest } = report.summary;
+ return {
+ ...rest,
+ scores: {
+ performance: scores.performance,
+ accessibility: scores.accessibility,
+ bestPractices: scores.bestPractices,
+ seo: scores.seo,
+ },
+ lighthouseVersion: report.lighthouseVersion,
+ };
+ },
+});
diff --git a/packages/pieces/community/browserless/src/lib/actions/ai/create-pdf.ts b/packages/pieces/community/browserless/src/lib/actions/ai/create-pdf.ts
new file mode 100644
index 00000000000..f9538b6aa2f
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/ai/create-pdf.ts
@@ -0,0 +1,95 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../../common/auth';
+import { browserlessApi } from '../../common/client';
+import { browserlessBody, browserlessProps } from '../../common/props';
+import { browserlessOutputSchemas } from '../../output-schemas';
+
+export const createPdf = createAction({
+ auth: browserlessAuth,
+ name: 'create_pdf',
+ classification: 'READ',
+ displayName: 'Create PDF (File Only)',
+ description: 'Render a web page or HTML to PDF and return only the stored file.',
+ audience: 'ai',
+ aiMetadata: {
+ description:
+ 'Renders a URL or raw HTML in a headless browser and stores it as a PDF, returning only the file link plus name, size and MIME type (no base64 in the output). Use to produce a printable document, for example from HTML you wrote; give exactly one of url or html. Each call renders again and stores a new file.',
+ idempotent: false,
+ },
+ props: {
+ url: browserlessProps.url({ required: false, description: 'The page to convert, for example https://example.com. Leave empty when you pass HTML.' }),
+ html: Property.LongText({
+ displayName: 'HTML',
+ description: 'Raw HTML to render instead of a URL.',
+ required: false,
+ }),
+ format: Property.StaticDropdown({
+ displayName: 'Paper Format',
+ description: 'Paper size (default A4).',
+ required: false,
+ options: {
+ options: ['A3', 'A4', 'A5', 'Letter', 'Legal', 'Tabloid'].map((value) => ({ label: value, value })),
+ },
+ }),
+ landscape: Property.Checkbox({
+ displayName: 'Landscape',
+ description: 'Use landscape orientation.',
+ required: false,
+ defaultValue: false,
+ }),
+ printBackground: Property.Checkbox({
+ displayName: 'Print Background',
+ description: 'Include background colours and images (default on).',
+ required: false,
+ defaultValue: true,
+ }),
+ margin: Property.ShortText({
+ displayName: 'Margin',
+ description: 'One margin for all sides, for example `10mm` or `0.5in`.',
+ required: false,
+ }),
+ waitForSelector: browserlessProps.waitForSelector(),
+ waitForTimeout: browserlessProps.waitForTimeout(),
+ timeout: browserlessProps.sessionTimeout(),
+ },
+ outputSchema: browserlessOutputSchemas.fileOnly,
+ async run(context) {
+ const props = context.propsValue;
+ const source = browserlessBody.pageSource({ url: props.url, html: props.html });
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1000, max: 540_000 });
+ const margin = browserlessBody.nonEmpty(props.margin) ? props.margin.trim() : undefined;
+
+ const response = await browserlessApi.request({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/pdf',
+ body: {
+ ...source,
+ options: {
+ format: browserlessBody.nonEmpty(props.format) ? props.format : 'A4',
+ landscape: props.landscape === true,
+ printBackground: props.printBackground !== false,
+ ...(margin !== undefined ? { margin: { top: margin, right: margin, bottom: margin, left: margin } } : {}),
+ },
+ ...browserlessBody.pageOptions({ waitForSelector: props.waitForSelector, waitForTimeout: props.waitForTimeout }),
+ },
+ query: { timeout },
+ responseType: 'arraybuffer',
+ timeoutMs: timeout === undefined ? undefined : timeout + 30_000,
+ operation: 'Create PDF',
+ });
+
+ const data = browserlessApi.toBuffer(response.body);
+ const fileName = 'document.pdf';
+ const file = await context.files.write({ fileName, data });
+ return {
+ file,
+ file_name: fileName,
+ size_bytes: data.length,
+ mime_type: 'application/pdf',
+ source: 'url' in source ? source.url : 'html',
+ site_status_code: browserlessApi.siteResponse(response.headers).site_status_code,
+ };
+ },
+});
diff --git a/packages/pieces/community/browserless/src/lib/actions/ai/take-screenshot.ts b/packages/pieces/community/browserless/src/lib/actions/ai/take-screenshot.ts
new file mode 100644
index 00000000000..826e393ba4f
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/ai/take-screenshot.ts
@@ -0,0 +1,115 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../../common/auth';
+import { browserlessApi } from '../../common/client';
+import { browserlessBody, browserlessProps } from '../../common/props';
+import { browserlessOutputSchemas } from '../../output-schemas';
+
+export const takeScreenshot = createAction({
+ auth: browserlessAuth,
+ name: 'take_screenshot',
+ classification: 'READ',
+ displayName: 'Take Screenshot (File Only)',
+ description: 'Screenshot a web page or HTML and return only the stored image file.',
+ audience: 'ai',
+ aiMetadata: {
+ description:
+ 'Renders a URL or raw HTML in a headless browser and stores a PNG, JPEG or WebP screenshot, returning only the file link plus name, size and MIME type (no base64 in the output). Use when you need an image of a page; to read page text use Smart Scrape. Give exactly one of url or html. Each call renders again and stores a new file.',
+ idempotent: false,
+ },
+ props: {
+ url: browserlessProps.url({ required: false, description: 'The page to capture, for example https://example.com. Leave empty when you pass HTML.' }),
+ html: Property.LongText({
+ displayName: 'HTML',
+ description: 'Raw HTML to render instead of a URL.',
+ required: false,
+ }),
+ imageType: Property.StaticDropdown({
+ displayName: 'Image Type',
+ description: 'Image format (default PNG).',
+ required: false,
+ options: {
+ options: [
+ { label: 'PNG', value: 'png' },
+ { label: 'JPEG', value: 'jpeg' },
+ { label: 'WebP', value: 'webp' },
+ ],
+ },
+ }),
+ fullPage: Property.Checkbox({
+ displayName: 'Full Page',
+ description: 'Capture the whole scrollable page instead of the visible viewport.',
+ required: false,
+ defaultValue: false,
+ }),
+ quality: Property.Number({
+ displayName: 'Quality',
+ description: 'JPEG/WebP quality 0-100.',
+ required: false,
+ }),
+ width: Property.Number({
+ displayName: 'Viewport Width',
+ description: 'Browser width in pixels (set together with height).',
+ required: false,
+ }),
+ height: Property.Number({
+ displayName: 'Viewport Height',
+ description: 'Browser height in pixels (set together with width).',
+ required: false,
+ }),
+ selector: Property.ShortText({
+ displayName: 'Element Selector',
+ description: 'Capture only the element matching this CSS selector.',
+ required: false,
+ }),
+ waitForSelector: browserlessProps.waitForSelector(),
+ waitForTimeout: browserlessProps.waitForTimeout(),
+ timeout: browserlessProps.sessionTimeout(),
+ },
+ outputSchema: browserlessOutputSchemas.fileOnly,
+ async run(context) {
+ const props = context.propsValue;
+ const source = browserlessBody.pageSource({ url: props.url, html: props.html });
+ const imageType = browserlessBody.nonEmpty(props.imageType) ? props.imageType : 'png';
+ const quality = browserlessBody.optionalNumber({ value: props.quality, label: 'Quality', min: 0, max: 100 });
+ const width = browserlessBody.optionalNumber({ value: props.width, label: 'Viewport Width', min: 1 });
+ const height = browserlessBody.optionalNumber({ value: props.height, label: 'Viewport Height', min: 1 });
+ if ((width === undefined) !== (height === undefined)) {
+ throw new Error('Set both Viewport Width and Viewport Height, or leave both empty.');
+ }
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1000, max: 540_000 });
+
+ const response = await browserlessApi.request({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/screenshot',
+ body: {
+ ...source,
+ options: {
+ type: imageType,
+ fullPage: props.fullPage === true,
+ ...(quality !== undefined && imageType !== 'png' ? { quality } : {}),
+ },
+ ...(width !== undefined && height !== undefined ? { viewport: { width, height } } : {}),
+ ...(browserlessBody.nonEmpty(props.selector) ? { selector: props.selector.trim() } : {}),
+ ...browserlessBody.pageOptions({ waitForSelector: props.waitForSelector, waitForTimeout: props.waitForTimeout }),
+ },
+ query: { timeout },
+ responseType: 'arraybuffer',
+ timeoutMs: timeout === undefined ? undefined : timeout + 30_000,
+ operation: 'Take Screenshot',
+ });
+
+ const data = browserlessApi.toBuffer(response.body);
+ const fileName = `screenshot.${imageType === 'jpeg' ? 'jpg' : imageType}`;
+ const file = await context.files.write({ fileName, data });
+ return {
+ file,
+ file_name: fileName,
+ size_bytes: data.length,
+ mime_type: `image/${imageType}`,
+ source: 'url' in source ? source.url : 'html',
+ site_status_code: browserlessApi.siteResponse(response.headers).site_status_code,
+ };
+ },
+});
diff --git a/packages/pieces/community/browserless/src/lib/actions/cancel-crawl.ts b/packages/pieces/community/browserless/src/lib/actions/cancel-crawl.ts
new file mode 100644
index 00000000000..79172d6cf34
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/cancel-crawl.ts
@@ -0,0 +1,55 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { BrowserlessApiError, browserlessApi } from '../common/client';
+import { browserlessCrawl } from '../common/crawl';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const cancelCrawl = createAction({
+ auth: browserlessAuth,
+ name: 'cancel_crawl',
+ classification: 'DESTRUCTIVE',
+ displayName: 'Cancel Crawl',
+ description: 'Stop a running crawl. Pages already scraped stay available.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Stops a running crawl by its ID so no more pages are scraped (and billed); pages already finished stay readable with Get Crawl Results. A cancelled crawl cannot be resumed. If the crawl already finished, it reports cancelled=false with the final status instead of failing, so a retry is safe.',
+ idempotent: true,
+ },
+ props: {
+ crawlId: browserlessCrawl.crawlIdProp(),
+ },
+ outputSchema: browserlessOutputSchemas.cancelCrawl,
+ async run(context) {
+ const crawlId = browserlessCrawl.parseCrawlId(context.propsValue.crawlId);
+ try {
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.DELETE,
+ path: `/crawl/${encodeURIComponent(crawlId)}`,
+ timeoutMs: 60_000,
+ operation: 'Cancel Crawl',
+ });
+ const body = browserlessValues.record(response.body);
+ return {
+ crawl_id: crawlId,
+ cancelled: true,
+ status: browserlessValues.stringOrNull(body['status']) ?? 'cancelled',
+ message: null,
+ };
+ } catch (error) {
+ if (error instanceof BrowserlessApiError && error.status === 409) {
+ const body = browserlessValues.record(error.responseBody);
+ return {
+ crawl_id: crawlId,
+ cancelled: false,
+ status: browserlessValues.stringOrNull(body['status']),
+ message: browserlessValues.stringOrNull(body['message']) ?? 'The crawl had already finished.',
+ };
+ }
+ throw error;
+ }
+ },
+});
diff --git a/packages/pieces/community/browserless/src/lib/actions/capture-screenshot.ts b/packages/pieces/community/browserless/src/lib/actions/capture-screenshot.ts
index d482cddc008..581b7ba2b17 100644
--- a/packages/pieces/community/browserless/src/lib/actions/capture-screenshot.ts
+++ b/packages/pieces/community/browserless/src/lib/actions/capture-screenshot.ts
@@ -1,15 +1,17 @@
import { createAction, Property } from '@activepieces/pieces-framework';
import { HttpMethod } from '@activepieces/pieces-common';
import { browserlessAuth } from '../common/auth';
-import { browserlessCommon, convertBinaryToBase64, isBinaryResponse } from '../common/client';
+import { browserlessApi } from '../common/client';
+import { browserlessBody } from '../common/props';
+import { browserlessOutputSchemas } from '../output-schemas';
export const captureScreenshot = createAction({
name: 'capture_screenshot',
classification: 'READ',
displayName: 'Capture Screenshot',
description: 'Take a screenshot of a web page',
- audience: 'both',
- aiMetadata: { description: 'Renders a web page in a headless browser and returns a PNG or JPEG screenshot of it. Use to capture the visual state of a public URL; the page URL is required, and options like full-page capture, viewport size, clipping region, and waiting for a CSS selector control what is rendered. Not idempotent: each call runs a fresh headless-browser render, so the returned image reflects the live page at the moment of the call.', idempotent: false },
+ audience: 'human',
+ aiMetadata: { description: 'Renders a web page in a headless browser and returns a PNG, JPEG or WebP screenshot as a file. Use to capture the visual state of a public URL; for page text use Get Page Content or Smart Scrape instead. The page URL is required, and options like full-page capture, viewport size, clipping region, and waiting for a CSS selector control what is rendered. Each call renders the live page again and stores a new file (about 1 Browserless unit per 30 s).', idempotent: false },
auth: browserlessAuth,
props: {
url: Property.ShortText({
@@ -25,13 +27,14 @@ export const captureScreenshot = createAction({
options: {
options: [
{ label: 'PNG', value: 'png' },
- { label: 'JPEG', value: 'jpeg' }
+ { label: 'JPEG', value: 'jpeg' },
+ { label: 'WebP', value: 'webp' },
]
}
}),
quality: Property.Number({
displayName: 'Quality',
- description: 'Image quality (0-100, only for JPEG)',
+ description: 'Image quality (0-100, only for JPEG and WebP)',
required: false,
}),
fullPage: Property.Checkbox({
@@ -87,91 +90,75 @@ export const captureScreenshot = createAction({
required: false,
}),
},
+ outputSchema: browserlessOutputSchemas.captureScreenshot,
async run(context) {
- const requestBody: any = {
- url: context.propsValue.url,
- options: {
- type: context.propsValue.imageType || 'png',
- fullPage: context.propsValue.fullPage || false,
- }
- };
-
- if (context.propsValue.quality && context.propsValue.imageType === 'jpeg') {
- requestBody.options.quality = context.propsValue.quality;
- }
-
- if (context.propsValue.width && context.propsValue.height) {
- requestBody.viewport = {
- width: context.propsValue.width,
- height: context.propsValue.height,
- };
+ const props = context.propsValue;
+ const imageType = props.imageType || 'png';
+ const clip = [props.clipX, props.clipY, props.clipWidth, props.clipHeight];
+ const clipGiven = clip.filter((value) => value !== undefined && value !== null).length;
+ if (clipGiven > 0 && clipGiven < 4) {
+ throw new Error('To clip the screenshot, fill in all four of Clip X, Clip Y, Clip Width and Clip Height.');
}
-
- if (context.propsValue.waitForSelector) {
- requestBody.waitForSelector = {
- selector: context.propsValue.waitForSelector,
- };
+ const width = browserlessBody.optionalNumber({ value: props.width, label: 'Viewport Width', min: 1 });
+ const height = browserlessBody.optionalNumber({ value: props.height, label: 'Viewport Height', min: 1 });
+ if ((width === undefined) !== (height === undefined)) {
+ throw new Error('Set both Viewport Width and Viewport Height, or leave both empty.');
}
+ const quality = browserlessBody.optionalNumber({ value: props.quality, label: 'Quality', min: 0, max: 100 });
+ const delay = browserlessBody.optionalNumber({ value: props.delay, label: 'Delay', min: 0 });
- if (context.propsValue.delay) {
- requestBody.waitForTimeout = context.propsValue.delay;
- }
-
- if (context.propsValue.omitBackground) {
- requestBody.options.omitBackground = context.propsValue.omitBackground;
- }
-
- if (context.propsValue.clipX !== undefined &&
- context.propsValue.clipY !== undefined &&
- context.propsValue.clipWidth !== undefined &&
- context.propsValue.clipHeight !== undefined) {
- requestBody.options.clip = {
- x: context.propsValue.clipX,
- y: context.propsValue.clipY,
- width: context.propsValue.clipWidth,
- height: context.propsValue.clipHeight,
- };
- }
+ const requestBody = {
+ url: props.url,
+ options: {
+ type: imageType,
+ fullPage: props.fullPage === true,
+ ...(quality !== undefined && imageType !== 'png' ? { quality } : {}),
+ ...(props.omitBackground === true ? { omitBackground: true } : {}),
+ ...(clipGiven === 4
+ ? {
+ clip: {
+ x: Number(props.clipX),
+ y: Number(props.clipY),
+ width: Number(props.clipWidth),
+ height: Number(props.clipHeight),
+ },
+ }
+ : {}),
+ },
+ ...(width !== undefined && height !== undefined ? { viewport: { width, height } } : {}),
+ ...(browserlessBody.nonEmpty(props.waitForSelector) ? { waitForSelector: { selector: props.waitForSelector.trim() } } : {}),
+ ...(delay !== undefined ? { waitForTimeout: delay } : {}),
+ };
- const response = await browserlessCommon.apiCall({
+ const response = await browserlessApi.request({
auth: context.auth.props,
method: HttpMethod.POST,
- resourceUri: '/screenshot',
+ path: '/screenshot',
body: requestBody,
+ responseType: 'arraybuffer',
+ operation: 'Capture Screenshot',
});
- const imageType = context.propsValue.imageType || 'png';
- const fileName = `screenshot.${imageType}`;
-
- let fileData: Buffer;
-
- if (response.body instanceof ArrayBuffer) {
- fileData = Buffer.from(response.body);
- } else if (Buffer.isBuffer(response.body)) {
- fileData = response.body;
- } else if (typeof response.body === 'string') {
- fileData = Buffer.from(response.body, 'latin1');
- } else {
- fileData = Buffer.from(String(response.body), 'latin1');
- }
-
- const file = await context.files.write({
- data: fileData,
- fileName: fileName,
- });
+ const fileData = browserlessApi.toBuffer(response.body);
+ const fileName = `screenshot.${imageType === 'jpeg' ? 'jpg' : imageType}`;
+ const file = await context.files.write({ data: fileData, fileName });
+ const site = browserlessApi.siteResponse(response.headers);
return {
success: true,
- file: file,
- screenshotBase64: convertBinaryToBase64(fileData),
+ file,
+ screenshotBase64: fileData.toString('base64'),
metadata: {
- url: context.propsValue.url,
+ url: props.url,
type: imageType,
- fullPage: context.propsValue.fullPage || false,
+ fullPage: props.fullPage === true,
timestamp: new Date().toISOString(),
- contentType: response.headers?.['content-type'] || `image/${imageType}`,
- fileName: fileName,
- }
+ contentType: browserlessApi.headerValue({ headers: response.headers, name: 'content-type' }) ?? `image/${imageType}`,
+ fileName,
+ sizeBytes: fileData.length,
+ siteStatusCode: site.site_status_code,
+ finalUrl: site.final_url,
+ },
};
},
});
diff --git a/packages/pieces/community/browserless/src/lib/actions/generate-pdf.ts b/packages/pieces/community/browserless/src/lib/actions/generate-pdf.ts
index 4f0d5cc40c4..12fda462475 100644
--- a/packages/pieces/community/browserless/src/lib/actions/generate-pdf.ts
+++ b/packages/pieces/community/browserless/src/lib/actions/generate-pdf.ts
@@ -1,14 +1,16 @@
import { createAction, Property } from '@activepieces/pieces-framework';
import { HttpMethod } from '@activepieces/pieces-common';
import { browserlessAuth } from '../common/auth';
-import { browserlessCommon, convertBinaryToBase64 } from '../common/client';
+import { browserlessApi } from '../common/client';
+import { browserlessBody } from '../common/props';
+import { browserlessOutputSchemas } from '../output-schemas';
export const generatePdf = createAction({
name: 'generate_pdf',
classification: 'READ',
displayName: 'Generate PDF',
description: 'Convert a web page to PDF',
- audience: 'both',
+ audience: 'human',
aiMetadata: { description: 'Renders content in a headless browser and returns it as a PDF file. Source the content either from a page URL or from a raw HTML string (provide exactly one, not both). Use to produce a printable PDF of a page or supplied markup, with control over paper format, orientation, margins, and headers/footers. Not idempotent: each call runs a fresh headless-browser render, so repeating it produces a new PDF of the page or markup as it renders at that moment.', idempotent: false },
auth: browserlessAuth,
props: {
@@ -39,7 +41,7 @@ export const generatePdf = createAction({
{ label: 'Letter', value: 'Letter' },
{ label: 'Legal', value: 'Legal' },
{ label: 'Ledger', value: 'Ledger' },
- { label: 'Tabloid', value: 'Tabloid' }
+ { label: 'Tabloid', value: 'Tabloid' },
]
}
}),
@@ -159,7 +161,7 @@ export const generatePdf = createAction({
}),
timeout: Property.Number({
displayName: 'Timeout (ms)',
- description: 'Maximum time to wait for the page to load',
+ description: 'Maximum time in milliseconds to wait for the page to load',
required: false,
}),
waitForFunction: Property.LongText({
@@ -194,172 +196,110 @@ export const generatePdf = createAction({
defaultValue: false,
}),
},
+ outputSchema: browserlessOutputSchemas.generatePdf,
async run(context) {
- if (!context.propsValue.url && !context.propsValue.html) {
+ const props = context.propsValue;
+ const hasUrl = browserlessBody.nonEmpty(props.url);
+ const hasHtml = browserlessBody.nonEmpty(props.html);
+ if (!hasUrl && !hasHtml) {
throw new Error('Either URL or HTML content must be provided');
}
-
- if (context.propsValue.url && context.propsValue.html) {
+ if (hasUrl && hasHtml) {
throw new Error('Cannot provide both URL and HTML content. Choose one.');
}
-
- const requestBody: any = {
- options: {
- format: context.propsValue.format || 'A4',
- landscape: context.propsValue.landscape || false,
- printBackground: context.propsValue.printBackground !== false,
- displayHeaderFooter: context.propsValue.displayHeaderFooter || false,
- }
+ const scale = browserlessBody.optionalNumber({ value: props.scale, label: 'Scale', min: 0.1, max: 2 });
+ const navigationTimeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 0 });
+ const waitForTimeout = browserlessBody.optionalNumber({ value: props.waitForTimeout, label: 'Wait Timeout', min: 0 });
+ const selectorTimeout = browserlessBody.optionalNumber({ value: props.waitForSelectorTimeout, label: 'Wait for Selector Timeout', min: 0 });
+ const functionTimeout = browserlessBody.optionalNumber({ value: props.waitForFunctionTimeout, label: 'Wait for Function Timeout', min: 0 });
+
+ const margin = {
+ ...(browserlessBody.nonEmpty(props.marginTop) ? { top: props.marginTop } : {}),
+ ...(browserlessBody.nonEmpty(props.marginRight) ? { right: props.marginRight } : {}),
+ ...(browserlessBody.nonEmpty(props.marginBottom) ? { bottom: props.marginBottom } : {}),
+ ...(browserlessBody.nonEmpty(props.marginLeft) ? { left: props.marginLeft } : {}),
};
- if (context.propsValue.url) {
- requestBody.url = context.propsValue.url;
- } else if (context.propsValue.html) {
- requestBody.html = context.propsValue.html;
- }
-
- if (context.propsValue.userAgent) {
- requestBody.userAgent = context.propsValue.userAgent;
- }
-
- if (context.propsValue.waitForTimeout) {
- requestBody.waitForTimeout = context.propsValue.waitForTimeout;
- }
-
- if (context.propsValue.bestAttempt) {
- requestBody.bestAttempt = context.propsValue.bestAttempt;
- }
-
- const margin: any = {};
- if (context.propsValue.marginTop) margin.top = context.propsValue.marginTop;
- if (context.propsValue.marginRight) margin.right = context.propsValue.marginRight;
- if (context.propsValue.marginBottom) margin.bottom = context.propsValue.marginBottom;
- if (context.propsValue.marginLeft) margin.left = context.propsValue.marginLeft;
-
- if (Object.keys(margin).length > 0) {
- requestBody.options.margin = margin;
- }
-
- if (context.propsValue.headerTemplate) {
- requestBody.options.headerTemplate = context.propsValue.headerTemplate;
- }
-
- if (context.propsValue.footerTemplate) {
- requestBody.options.footerTemplate = context.propsValue.footerTemplate;
- }
-
- if (context.propsValue.scale) {
- requestBody.options.scale = Math.max(0.1, Math.min(2.0, context.propsValue.scale));
- }
-
- if (context.propsValue.waitForSelector) {
- const waitForSelectorObj: any = {
- selector: context.propsValue.waitForSelector,
- };
-
- if (context.propsValue.waitForSelectorTimeout !== undefined) {
- waitForSelectorObj.timeout = context.propsValue.waitForSelectorTimeout;
- }
-
- if (context.propsValue.waitForSelectorVisible !== undefined) {
- waitForSelectorObj.visible = context.propsValue.waitForSelectorVisible;
- }
-
- if (context.propsValue.waitForSelectorHidden !== undefined) {
- waitForSelectorObj.hidden = context.propsValue.waitForSelectorHidden;
- }
-
- requestBody.options.waitForSelector = waitForSelectorObj;
- }
-
- if (context.propsValue.preferCSSPageSize) {
- requestBody.options.preferCSSPageSize = context.propsValue.preferCSSPageSize;
- }
-
- if (context.propsValue.pageRanges) {
- requestBody.options.pageRanges = context.propsValue.pageRanges;
- }
-
- if (context.propsValue.width) {
- requestBody.options.width = context.propsValue.width;
- }
-
- if (context.propsValue.height) {
- requestBody.options.height = context.propsValue.height;
- }
-
- if (context.propsValue.omitBackground) {
- requestBody.options.omitBackground = context.propsValue.omitBackground;
- }
-
- if (context.propsValue.tagged) {
- requestBody.options.tagged = context.propsValue.tagged;
- }
-
- if (context.propsValue.outline) {
- requestBody.options.outline = context.propsValue.outline;
- }
-
- if (context.propsValue.timeout) {
- requestBody.options.timeout = context.propsValue.timeout;
- }
-
- if (context.propsValue.waitForFunction) {
- const waitForFunctionObj: any = {
- fn: context.propsValue.waitForFunction,
- };
-
- if (context.propsValue.waitForFunctionPolling !== undefined) {
- waitForFunctionObj.polling = context.propsValue.waitForFunctionPolling;
- }
-
- if (context.propsValue.waitForFunctionTimeout !== undefined) {
- waitForFunctionObj.timeout = context.propsValue.waitForFunctionTimeout;
- }
+ const options = {
+ format: props.format || 'A4',
+ landscape: props.landscape === true,
+ printBackground: props.printBackground !== false,
+ displayHeaderFooter: props.displayHeaderFooter === true,
+ ...(Object.keys(margin).length > 0 ? { margin } : {}),
+ ...(browserlessBody.nonEmpty(props.headerTemplate) ? { headerTemplate: props.headerTemplate } : {}),
+ ...(browserlessBody.nonEmpty(props.footerTemplate) ? { footerTemplate: props.footerTemplate } : {}),
+ ...(scale !== undefined ? { scale } : {}),
+ ...(props.preferCSSPageSize === true ? { preferCSSPageSize: true } : {}),
+ ...(browserlessBody.nonEmpty(props.pageRanges) ? { pageRanges: props.pageRanges } : {}),
+ ...(browserlessBody.nonEmpty(props.width) ? { width: props.width } : {}),
+ ...(browserlessBody.nonEmpty(props.height) ? { height: props.height } : {}),
+ ...(props.omitBackground === true ? { omitBackground: true } : {}),
+ ...(props.tagged === true ? { tagged: true } : {}),
+ ...(props.outline === true ? { outline: true } : {}),
+ };
- requestBody.options.waitForFunction = waitForFunctionObj;
- }
+ const waitForSelector = browserlessBody.nonEmpty(props.waitForSelector)
+ ? {
+ selector: props.waitForSelector.trim(),
+ ...(selectorTimeout !== undefined ? { timeout: selectorTimeout } : {}),
+ ...(props.waitForSelectorHidden === true ? { hidden: true } : props.waitForSelectorVisible === true ? { visible: true } : {}),
+ }
+ : undefined;
+
+ const waitForFunction = browserlessBody.nonEmpty(props.waitForFunction)
+ ? {
+ fn: props.waitForFunction,
+ ...(browserlessBody.nonEmpty(props.waitForFunctionPolling) ? { polling: pollingValue(props.waitForFunctionPolling) } : {}),
+ ...(functionTimeout !== undefined ? { timeout: functionTimeout } : {}),
+ }
+ : undefined;
+
+ const requestBody = {
+ ...(hasUrl ? { url: props.url } : { html: props.html }),
+ options,
+ ...(navigationTimeout !== undefined ? { gotoOptions: { timeout: navigationTimeout } } : {}),
+ ...(waitForSelector !== undefined ? { waitForSelector } : {}),
+ ...(waitForFunction !== undefined ? { waitForFunction } : {}),
+ ...(waitForTimeout !== undefined ? { waitForTimeout } : {}),
+ ...(browserlessBody.nonEmpty(props.userAgent) ? { userAgent: { userAgent: props.userAgent.trim() } } : {}),
+ ...(props.bestAttempt === true ? { bestAttempt: true } : {}),
+ };
- const response = await browserlessCommon.apiCall({
+ const response = await browserlessApi.request({
auth: context.auth.props,
method: HttpMethod.POST,
- resourceUri: '/pdf',
+ path: '/pdf',
body: requestBody,
+ responseType: 'arraybuffer',
+ operation: 'Generate PDF',
});
+ const fileData = browserlessApi.toBuffer(response.body);
const fileName = 'document.pdf';
-
- let fileData: Buffer;
-
- if (response.body instanceof ArrayBuffer) {
- fileData = Buffer.from(response.body);
- } else if (Buffer.isBuffer(response.body)) {
- fileData = response.body;
- } else if (typeof response.body === 'string') {
- fileData = Buffer.from(response.body, 'latin1');
- } else {
- fileData = Buffer.from(String(response.body), 'latin1');
- }
-
- const file = await context.files.write({
- data: fileData,
- fileName: fileName,
- });
+ const file = await context.files.write({ data: fileData, fileName });
+ const site = browserlessApi.siteResponse(response.headers);
return {
success: true,
- file: file,
- pdfBase64: convertBinaryToBase64(fileData),
+ file,
+ pdfBase64: fileData.toString('base64'),
metadata: {
- source: context.propsValue.url ? 'url' : 'html',
- url: context.propsValue.url || null,
- hasHtml: !!context.propsValue.html,
- format: context.propsValue.format || 'A4',
- landscape: context.propsValue.landscape || false,
+ source: hasUrl ? 'url' : 'html',
+ url: hasUrl ? props.url : null,
+ hasHtml,
+ format: props.format || 'A4',
+ landscape: props.landscape === true,
timestamp: new Date().toISOString(),
- fileName: fileName,
+ fileName,
contentType: 'application/pdf',
- }
+ sizeBytes: fileData.length,
+ siteStatusCode: site.site_status_code,
+ },
};
},
});
+
+function pollingValue(value: string): string | number {
+ const trimmed = value.trim();
+ return /^\d+$/.test(trimmed) ? Number(trimmed) : trimmed;
+}
diff --git a/packages/pieces/community/browserless/src/lib/actions/get-crawl.ts b/packages/pieces/community/browserless/src/lib/actions/get-crawl.ts
new file mode 100644
index 00000000000..96500be5900
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/get-crawl.ts
@@ -0,0 +1,70 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { browserlessApi } from '../common/client';
+import { browserlessCrawl } from '../common/crawl';
+import { browserlessBody } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const getCrawl = createAction({
+ auth: browserlessAuth,
+ name: 'get_crawl',
+ classification: 'READ',
+ displayName: 'Get Crawl Results',
+ description: 'Get the status of a crawl and one page of its scraped pages.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Returns the status and progress counters of a crawl started with Start Crawl, plus one batch of crawled pages (URL, title, HTTP status and a temporary link to the page content). Call again while status is in-progress; when has_more is true pass next_skip as Skip to read the next batch. Read-only and safe to retry.',
+ idempotent: true,
+ },
+ props: {
+ crawlId: browserlessCrawl.crawlIdProp(),
+ skip: Property.Number({
+ displayName: 'Skip',
+ description: 'Pages to skip; use Next Skip from the previous run.',
+ required: false,
+ }),
+ },
+ outputSchema: browserlessOutputSchemas.getCrawl,
+ async run(context) {
+ const crawlId = browserlessCrawl.parseCrawlId(context.propsValue.crawlId);
+ const skip = browserlessBody.optionalNumber({ value: context.propsValue.skip, label: 'Skip', min: 0 });
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.GET,
+ path: `/crawl/${encodeURIComponent(crawlId)}`,
+ query: { skip: skip === undefined ? undefined : Math.floor(skip) },
+ timeoutMs: 60_000,
+ operation: 'Get Crawl Results',
+ });
+
+ const body = browserlessValues.record(response.body);
+ const pages = Array.isArray(body['data']) ? body['data'].map(browserlessCrawl.toPage) : [];
+ const next = browserlessValues.stringOrNull(body['next']);
+ const nextSkip = readSkip(next);
+ return {
+ crawl_id: crawlId,
+ status: browserlessValues.stringOrNull(body['status']),
+ total: browserlessValues.numberOrNull(body['total']),
+ completed: browserlessValues.numberOrNull(body['completed']),
+ failed: browserlessValues.numberOrNull(body['failed']),
+ expires_at: browserlessValues.stringOrNull(body['expiresAt']),
+ skip: skip === undefined ? 0 : Math.floor(skip),
+ page_count: pages.length,
+ has_more: nextSkip !== null,
+ next_skip: nextSkip,
+ pages,
+ };
+ },
+});
+
+function readSkip(next: string | null): number | null {
+ if (next === null || next === '') {
+ return null;
+ }
+ const match = /[?&]skip=(\d+)/.exec(next);
+ return match ? Number(match[1]) : null;
+}
diff --git a/packages/pieces/community/browserless/src/lib/actions/get-page-content.ts b/packages/pieces/community/browserless/src/lib/actions/get-page-content.ts
new file mode 100644
index 00000000000..2bb59cbfe70
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/get-page-content.ts
@@ -0,0 +1,78 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { browserlessApi } from '../common/client';
+import { browserlessBody, browserlessProps } from '../common/props';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const getPageContent = createAction({
+ auth: browserlessAuth,
+ name: 'get_page_content',
+ classification: 'READ',
+ displayName: 'Get Page Content',
+ description: 'Load a page in a real browser and return its fully rendered HTML, including content added by JavaScript.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Loads a URL in a headless browser and returns the final rendered HTML after JavaScript runs, plus the HTTP status the site answered with. Use only when you need the raw HTML of a JavaScript-heavy page; to read or summarise a page prefer Smart Scrape (markdown, much shorter), and use Scrape URL for specific CSS selectors. HTML is cut to Maximum Characters (default 100000) and truncated=true says so. Read-only and safe to retry.',
+ idempotent: true,
+ },
+ props: {
+ url: browserlessProps.url({ required: true, description: 'The page to load, for example https://example.com/pricing' }),
+ waitUntil: browserlessProps.waitUntil(),
+ navigationTimeout: browserlessProps.navigationTimeout(),
+ waitForSelector: browserlessProps.waitForSelector(),
+ waitForTimeout: browserlessProps.waitForTimeout(),
+ rejectResourceTypes: Property.StaticMultiSelectDropdown({
+ displayName: 'Skip Resource Types',
+ description: 'Resource types the browser should not download, which makes the page load faster.',
+ required: false,
+ options: {
+ options: [
+ { label: 'Images', value: 'image' },
+ { label: 'Stylesheets', value: 'stylesheet' },
+ { label: 'Fonts', value: 'font' },
+ { label: 'Media (video/audio)', value: 'media' },
+ ],
+ },
+ }),
+ bestAttempt: browserlessProps.bestAttempt(),
+ blockAds: browserlessProps.blockAds(),
+ maxCharacters: browserlessProps.maxCharacters(),
+ timeout: browserlessProps.sessionTimeout(),
+ },
+ outputSchema: browserlessOutputSchemas.getPageContent,
+ async run(context) {
+ const props = context.propsValue;
+ const url = browserlessBody.httpUrl({ value: props.url, label: 'URL' });
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1000, max: 540_000 });
+ const rejectResourceTypes = browserlessBody.textList(props.rejectResourceTypes);
+ const maxCharacters = browserlessBody.maxCharacters(props.maxCharacters);
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/content',
+ body: {
+ url,
+ ...browserlessBody.pageOptions(props),
+ ...(rejectResourceTypes.length > 0 ? { rejectResourceTypes } : {}),
+ },
+ query: { blockAds: props.blockAds === true ? true : undefined, timeout },
+ responseType: 'text',
+ timeoutMs: timeout === undefined ? undefined : timeout + 30_000,
+ operation: 'Get Page Content',
+ });
+
+ const html = typeof response.body === 'string' ? response.body : JSON.stringify(response.body ?? '');
+ const capped = browserlessBody.cap({ text: html, max: maxCharacters });
+ return {
+ url,
+ ...browserlessApi.siteResponse(response.headers),
+ html: capped.text,
+ html_length: html.length,
+ truncated: capped.truncated,
+ };
+ },
+});
+
diff --git a/packages/pieces/community/browserless/src/lib/actions/get-website-performance.ts b/packages/pieces/community/browserless/src/lib/actions/get-website-performance.ts
index 28d62ed7e16..fd64f540207 100644
--- a/packages/pieces/community/browserless/src/lib/actions/get-website-performance.ts
+++ b/packages/pieces/community/browserless/src/lib/actions/get-website-performance.ts
@@ -1,15 +1,19 @@
import { createAction, Property } from '@activepieces/pieces-framework';
import { HttpMethod } from '@activepieces/pieces-common';
import { browserlessAuth } from '../common/auth';
-import { browserlessCommon } from '../common/client';
+import { browserlessApi } from '../common/client';
+import { browserlessBody } from '../common/props';
+import { browserlessLighthouse } from '../common/lighthouse';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
export const getWebsitePerformance = createAction({
name: 'get_website_performance',
classification: 'READ',
displayName: 'Get Website Performance',
description: 'Analyze website performance metrics using Lighthouse',
- audience: 'both',
- aiMetadata: { description: 'Runs a Lighthouse audit on a public URL in a headless browser and returns performance, accessibility, best-practices, SEO, and PWA scores plus key metrics. Use to measure a page\'s quality; the URL is required, and you can scope the audit to specific categories and simulate a desktop or mobile device with network throttling. Read-only analysis: re-running with the same input re-audits the page without side effects (scores may vary slightly run to run).', idempotent: true },
+ audience: 'human',
+ aiMetadata: { description: 'Runs a Lighthouse audit on a public URL in a headless browser and returns performance, accessibility, best-practices and SEO scores (0-100) plus Core Web Vitals. Use to measure a page\'s quality; the URL is required, and you can scope the audit to specific categories and simulate a desktop or mobile device with network throttling. Read-only analysis: re-running with the same input re-audits the page without side effects (scores may vary slightly run to run).', idempotent: true },
auth: browserlessAuth,
props: {
url: Property.ShortText({
@@ -19,7 +23,7 @@ export const getWebsitePerformance = createAction({
}),
categories: Property.Array({
displayName: 'Performance Categories',
- description: 'Select which performance categories to analyze (performance, accessibility, best-practices, seo, pwa)',
+ description: 'Categories to audit (Lighthouse 12+ has no PWA category).',
required: false,
properties: {
category: Property.StaticDropdown({
@@ -82,18 +86,18 @@ export const getWebsitePerformance = createAction({
}),
timeout: Property.Number({
displayName: 'Timeout (ms)',
- description: 'Maximum time to wait for the analysis to complete',
+ description: 'Max page load wait in ms during the audit (under 480000).',
required: false,
defaultValue: 60000,
}),
waitForSelector: Property.ShortText({
displayName: 'Wait for Selector',
- description: 'CSS selector to wait for before running performance analysis',
+ description: 'Not supported by Lighthouse audits; this setting is ignored.',
required: false,
}),
emulateMediaType: Property.StaticDropdown({
displayName: 'Emulate Media Type',
- description: 'Emulate CSS media type',
+ description: 'Not supported by Lighthouse audits; this setting is ignored.',
required: false,
options: {
options: [
@@ -143,180 +147,68 @@ export const getWebsitePerformance = createAction({
required: false,
defaultValue: false,
}),
+ includeFullReport: Property.Checkbox({
+ displayName: 'Include Full Report',
+ description: 'Also return the full Lighthouse report (often several MB).',
+ required: false,
+ defaultValue: true,
+ }),
},
+ outputSchema: browserlessOutputSchemas.getWebsitePerformance,
async run(context) {
- const requestBody: any = {
- url: context.propsValue.url,
- };
-
- const lighthouseConfig: any = {
- extends: 'lighthouse:default',
- settings: {}
- };
-
- if (context.propsValue.categories && context.propsValue.categories.length > 0) {
- lighthouseConfig.settings.onlyCategories = context.propsValue.categories.map((cat: any) => cat.category);
- }
-
- if (context.propsValue.locale) {
- lighthouseConfig.settings.locale = context.propsValue.locale;
- }
-
- if (context.propsValue.device) {
- lighthouseConfig.settings.formFactor = context.propsValue.device;
- }
-
- if (context.propsValue.throttling && context.propsValue.throttling !== 'none') {
- lighthouseConfig.settings.throttling = { rttMs: 150, throughputKbps: 1638.4, cpuSlowdownMultiplier: 4 };
-
- switch (context.propsValue.throttling) {
- case 'mobileSlow4G':
- lighthouseConfig.settings.throttling = { rttMs: 150, throughputKbps: 1638.4, cpuSlowdownMultiplier: 4 };
- break;
- case 'mobileRegular4G':
- lighthouseConfig.settings.throttling = { rttMs: 100, throughputKbps: 2048, cpuSlowdownMultiplier: 3 };
- break;
- case 'mobileFast4G':
- lighthouseConfig.settings.throttling = { rttMs: 50, throughputKbps: 4096, cpuSlowdownMultiplier: 2 };
- break;
- }
- }
-
- if (context.propsValue.userAgent) {
- lighthouseConfig.settings.userAgent = context.propsValue.userAgent;
- }
-
- if (context.propsValue.timeout) {
- lighthouseConfig.settings.timeout = context.propsValue.timeout;
- }
-
- if (context.propsValue.waitForSelector) {
- lighthouseConfig.settings.waitForSelector = context.propsValue.waitForSelector;
- }
-
- if (context.propsValue.emulateMediaType) {
- lighthouseConfig.settings.emulatedFormFactor = context.propsValue.emulateMediaType;
- }
-
- if (context.propsValue.onlyCategories && !lighthouseConfig.settings.onlyCategories) {
- lighthouseConfig.settings.onlyCategories = ['performance', 'accessibility', 'best-practices', 'seo', 'pwa'];
- }
-
- requestBody.config = lighthouseConfig;
+ const props = context.propsValue;
+ const device = props.device === 'mobile' ? 'mobile' : 'desktop';
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1 });
+ const selectedCategories = (props.categories ?? [])
+ .map(browserlessValues.record)
+ .map((entry) => entry['category'])
+ .filter((category): category is string => typeof category === 'string' && category !== '');
+ const onlyCategories =
+ selectedCategories.length > 0 ? Array.from(new Set(selectedCategories)) : props.onlyCategories === true ? browserlessLighthouse.DEFAULT_CATEGORIES : undefined;
+ const settings = browserlessLighthouse.buildSettings({
+ device,
+ onlyCategories,
+ throttling: props.throttling,
+ locale: props.locale,
+ userAgent: props.userAgent,
+ timeout,
+ });
- if (context.propsValue.budgets && context.propsValue.budgets.length > 0) {
- requestBody.budgets = context.propsValue.budgets.map((budget: any) => ({
- resourceType: budget.resourceType,
- budget: budget.budget * 1024
+ const budgets = (props.budgets ?? [])
+ .map(browserlessValues.record)
+ .filter((budget) => browserlessBody.nonEmpty(String(budget['resourceType'] ?? '')))
+ .map((budget) => ({
+ resourceType: String(budget['resourceType']),
+ budget: (browserlessBody.optionalNumber({ value: budget['budget'], label: 'Budget Size', min: 0 }) ?? 0) * 1024,
}));
- }
-
- let resourceUri = '/performance';
- const queryParams: string[] = [];
-
- if (context.propsValue.stealth) {
- queryParams.push('stealth=true');
- }
- if (context.propsValue.blockAds) {
- queryParams.push('blockAds=true');
- }
-
- if (queryParams.length > 0) {
- resourceUri += `?${queryParams.join('&')}`;
- }
-
- const response = await browserlessCommon.apiCall({
+ const response = await browserlessApi.request<unknown>({
auth: context.auth.props,
method: HttpMethod.POST,
- resourceUri,
- body: requestBody,
+ path: '/performance',
+ body: {
+ url: props.url,
+ config: { extends: 'lighthouse:default', settings },
+ ...(budgets.length > 0 ? { budgets } : {}),
+ },
+ query: {
+ stealth: props.stealth === true ? true : undefined,
+ blockAds: props.blockAds === true ? true : undefined,
+ },
+ timeoutMs: timeout === undefined ? undefined : timeout + 60_000,
+ operation: 'Get Website Performance',
});
- const performanceData = response.body;
-
- const summary: any = {
- url: context.propsValue.url,
- formFactor: context.propsValue.device || 'desktop',
- timestamp: new Date().toISOString(),
- };
-
- if (performanceData.lhr && performanceData.lhr.categories) {
- const categories = performanceData.lhr.categories;
- summary.scores = {
- performance: categories.performance?.score ? Math.round(categories.performance.score * 100) : null,
- accessibility: categories.accessibility?.score ? Math.round(categories.accessibility.score * 100) : null,
- bestPractices: categories['best-practices']?.score ? Math.round(categories['best-practices'].score * 100) : null,
- seo: categories.seo?.score ? Math.round(categories.seo.score * 100) : null,
- pwa: categories.pwa?.score ? Math.round(categories.pwa.score * 100) : null,
- };
- }
-
- if (performanceData.lhr && performanceData.lhr.audits) {
- const audits = performanceData.lhr.audits;
- summary.metrics = {
- firstContentfulPaint: {
- value: audits['first-contentful-paint']?.displayValue || null,
- score: audits['first-contentful-paint']?.score ? Math.round(audits['first-contentful-paint'].score * 100) : null
- },
- largestContentfulPaint: {
- value: audits['largest-contentful-paint']?.displayValue || null,
- score: audits['largest-contentful-paint']?.score ? Math.round(audits['largest-contentful-paint'].score * 100) : null
- },
- firstMeaningfulPaint: {
- value: audits['first-meaningful-paint']?.displayValue || null,
- score: audits['first-meaningful-paint']?.score ? Math.round(audits['first-meaningful-paint'].score * 100) : null
- },
- speedIndex: {
- value: audits['speed-index']?.displayValue || null,
- score: audits['speed-index']?.score ? Math.round(audits['speed-index'].score * 100) : null
- },
- timeToInteractive: {
- value: audits['interactive']?.displayValue || null,
- score: audits['interactive']?.score ? Math.round(audits['interactive'].score * 100) : null
- },
- totalBlockingTime: {
- value: audits['total-blocking-time']?.displayValue || null,
- score: audits['total-blocking-time']?.score ? Math.round(audits['total-blocking-time'].score * 100) : null
- },
- cumulativeLayoutShift: {
- value: audits['cumulative-layout-shift']?.displayValue || null,
- score: audits['cumulative-layout-shift']?.score ? Math.round(audits['cumulative-layout-shift'].score * 100) : null
- },
- };
-
- summary.opportunities = [];
- if (audits['unused-css-rules']?.details?.items?.length > 0) {
- summary.opportunities.push({
- type: 'unused-css',
- title: 'Remove unused CSS',
- potentialSavings: audits['unused-css-rules'].displayValue || 'Unknown'
- });
- }
- if (audits['unused-javascript']?.details?.items?.length > 0) {
- summary.opportunities.push({
- type: 'unused-javascript',
- title: 'Remove unused JavaScript',
- potentialSavings: audits['unused-javascript'].displayValue || 'Unknown'
- });
- }
- if (audits['render-blocking-resources']?.details?.items?.length > 0) {
- summary.opportunities.push({
- type: 'render-blocking',
- title: 'Eliminate render-blocking resources',
- potentialSavings: audits['render-blocking-resources'].displayValue || 'Unknown'
- });
- }
- }
-
+ const report = browserlessLighthouse.summarize({ body: response.body, url: props.url, device });
return {
success: true,
- summary,
- fullReport: performanceData,
+ summary: report.summary,
+ ...(props.includeFullReport === false ? {} : { fullReport: response.body }),
metadata: {
- analysisTime: response.headers?.['x-response-time'] || 'unknown',
- lighthouseVersion: performanceData.lhr?.lighthouseVersion || 'unknown',
- }
+ analysisTime: browserlessApi.headerValue({ headers: response.headers, name: 'x-response-time' }) ?? 'unknown',
+ lighthouseVersion: report.lighthouseVersion,
+ },
};
},
});
+
diff --git a/packages/pieces/community/browserless/src/lib/actions/list-crawls.ts b/packages/pieces/community/browserless/src/lib/actions/list-crawls.ts
new file mode 100644
index 00000000000..107990d81c0
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/list-crawls.ts
@@ -0,0 +1,74 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { browserlessApi } from '../common/client';
+import { browserlessCrawl } from '../common/crawl';
+import { browserlessBody } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const listCrawls = createAction({
+ auth: browserlessAuth,
+ name: 'list_crawls',
+ classification: 'SEARCH',
+ displayName: 'List Crawls',
+ description: 'List your recent crawls with their status and progress.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Lists crawl jobs on this Browserless account, newest first, with ID, start URL, status and page counts; filter by status and page with the cursor. Use to find a crawl ID when you no longer have it; use Get Crawl Results for the pages of one crawl. Read-only and safe to retry.',
+ idempotent: true,
+ },
+ props: {
+ status: Property.StaticDropdown({
+ displayName: 'Status',
+ description: 'Only list crawls with this status.',
+ required: false,
+ options: {
+ options: [
+ { label: 'In Progress', value: 'in-progress' },
+ { label: 'Completed', value: 'completed' },
+ { label: 'Failed', value: 'failed' },
+ { label: 'Cancelled', value: 'cancelled' },
+ ],
+ },
+ }),
+ limit: Property.Number({
+ displayName: 'Limit',
+ description: 'How many crawls to return in this batch.',
+ required: false,
+ }),
+ cursor: Property.ShortText({
+ displayName: 'Cursor',
+ description: 'Next Cursor from a previous run, to read the next batch.',
+ required: false,
+ }),
+ },
+ outputSchema: browserlessOutputSchemas.listCrawls,
+ async run(context) {
+ const props = context.propsValue;
+ const limit = browserlessBody.optionalNumber({ value: props.limit, label: 'Limit', min: 1, max: 1000 });
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.GET,
+ path: '/crawl',
+ query: {
+ status: browserlessBody.nonEmpty(props.status) ? props.status : undefined,
+ limit: limit === undefined ? undefined : Math.floor(limit),
+ cursor: browserlessBody.nonEmpty(props.cursor) ? props.cursor.trim() : undefined,
+ },
+ timeoutMs: 60_000,
+ operation: 'List Crawls',
+ });
+
+ const body = browserlessValues.record(response.body);
+ const crawls = Array.isArray(body['crawls']) ? body['crawls'].map(browserlessCrawl.toSummary) : [];
+ const nextCursor = browserlessValues.stringOrNull(body['nextCursor']);
+ return {
+ count: crawls.length,
+ has_more: nextCursor !== null && nextCursor !== '',
+ next_cursor: nextCursor === '' ? null : nextCursor,
+ crawls,
+ };
+ },
+});
diff --git a/packages/pieces/community/browserless/src/lib/actions/map-website.ts b/packages/pieces/community/browserless/src/lib/actions/map-website.ts
new file mode 100644
index 00000000000..7c3766b6d57
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/map-website.ts
@@ -0,0 +1,127 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { BrowserlessApiError, browserlessApi } from '../common/client';
+import { browserlessBody, browserlessProps } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const mapWebsite = createAction({
+ auth: browserlessAuth,
+ name: 'map_website',
+ classification: 'SEARCH',
+ displayName: 'Map Website URLs',
+ description: 'List the URLs of a website from its sitemap and links, optionally ordered by relevance to a search term.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Discovers the URLs of a website from its sitemap plus links found on the page and returns them as a list (500 by default, up to 5000), optionally ranked by a search term. Use to find which pages exist before scraping or crawling them; it does not return page content (use Smart Scrape or Start Crawl for that). Read-only and safe to retry.',
+ idempotent: true,
+ },
+ props: {
+ url: browserlessProps.url({ required: true, description: 'The website to map, for example https://example.com' }),
+ search: Property.ShortText({
+ displayName: 'Search Term',
+ description: 'Order the URLs by relevance to this term, for example `pricing`.',
+ required: false,
+ }),
+ limit: Property.Number({
+ displayName: 'Maximum URLs',
+ description: 'How many URLs to return (1-5000, default 500).',
+ required: false,
+ }),
+ sitemap: Property.StaticDropdown({
+ displayName: 'Sitemap',
+ description: 'How to use the site\'s sitemap.',
+ required: false,
+ options: {
+ options: [
+ { label: 'Use sitemap and page links (default)', value: 'include' },
+ { label: 'Only the sitemap', value: 'only' },
+ { label: 'Ignore the sitemap', value: 'skip' },
+ ],
+ },
+ }),
+ includeSubdomains: Property.Checkbox({
+ displayName: 'Include Subdomains',
+ description: 'Also return URLs on subdomains such as blog.example.com.',
+ required: false,
+ defaultValue: true,
+ }),
+ ignoreQueryParameters: Property.Checkbox({
+ displayName: 'Ignore Query Parameters',
+ description: 'Skip URLs that only differ by their ?query part.',
+ required: false,
+ defaultValue: true,
+ }),
+ proxy: browserlessProps.proxy({
+ description: 'Proxy network for fetching the site. Browserless uses Residential when empty; Datacenter is cheaper.',
+ }),
+ timeout: browserlessProps.sessionTimeout(),
+ },
+ outputSchema: browserlessOutputSchemas.mapWebsite,
+ async run(context) {
+ const props = context.propsValue;
+ const url = browserlessBody.httpUrl({ value: props.url, label: 'URL' });
+ const limit = browserlessBody.optionalNumber({ value: props.limit, label: 'Maximum URLs', min: 1, max: 5000 });
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1000, max: 540_000 });
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/map',
+ body: {
+ url,
+ ...(browserlessBody.nonEmpty(props.search) ? { search: props.search.trim() } : {}),
+ limit: limit === undefined ? DEFAULT_LIMIT : Math.floor(limit),
+ ...(browserlessBody.nonEmpty(props.sitemap) ? { sitemap: props.sitemap } : {}),
+ includeSubdomains: props.includeSubdomains !== false,
+ ignoreQueryParameters: props.ignoreQueryParameters !== false,
+ ...(browserlessBody.nonEmpty(props.proxy) ? { proxy: props.proxy } : {}),
+ ...(timeout !== undefined ? { timeout } : {}),
+ },
+ timeoutMs: timeout === undefined ? undefined : timeout + 30_000,
+ operation: 'Map Website URLs',
+ });
+
+ const links = toLinks(response.body);
+ if (links === null) {
+ const body = browserlessValues.record(response.body);
+ throw new BrowserlessApiError({
+ message: `Map Website URLs failed: ${browserlessValues.stringOrNull(body['error']) ?? browserlessValues.stringOrNull(body['message']) ?? 'unexpected response from Browserless'}`,
+ status: response.status,
+ responseBody: body,
+ });
+ }
+ return {
+ url,
+ count: links.length,
+ links,
+ };
+ },
+});
+
+function toLinks(body: unknown): MappedLink[] | null {
+ const record = browserlessValues.record(body);
+ const raw = Array.isArray(body) ? body : record['success'] === false ? null : Array.isArray(record['links']) ? record['links'] : null;
+ if (raw === null) {
+ return null;
+ }
+ return raw
+ .map((entry: unknown) => {
+ if (typeof entry === 'string') {
+ return { url: entry, title: null, description: null };
+ }
+ const item = browserlessValues.record(entry);
+ return {
+ url: browserlessValues.stringOrNull(item['url']) ?? '',
+ title: browserlessValues.stringOrNull(item['title']),
+ description: browserlessValues.stringOrNull(item['description']),
+ };
+ })
+ .filter((link) => link.url !== '');
+}
+
+type MappedLink = { url: string; title: string | null; description: string | null };
+
+const DEFAULT_LIMIT = 500;
diff --git a/packages/pieces/community/browserless/src/lib/actions/run-bql-query.ts b/packages/pieces/community/browserless/src/lib/actions/run-bql-query.ts
index 7833569e3e2..0078ba100d5 100644
--- a/packages/pieces/community/browserless/src/lib/actions/run-bql-query.ts
+++ b/packages/pieces/community/browserless/src/lib/actions/run-bql-query.ts
@@ -1,7 +1,12 @@
import { createAction, Property } from '@activepieces/pieces-framework';
import { HttpMethod } from '@activepieces/pieces-common';
import { browserlessAuth } from '../common/auth';
-import { browserlessCommon } from '../common/client';
+import { BrowserlessApiError, browserlessApi } from '../common/client';
+import { browserlessBody } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+const DEFAULT_TIMEOUT_MS = 30_000;
export const runBqlQuery = createAction({
name: 'run_bql_query',
@@ -9,7 +14,7 @@ export const runBqlQuery = createAction({
displayName: 'Run BQL Query',
description: 'Execute Browser Query Language (BQL) GraphQL-based queries for advanced browser automation',
audience: 'both',
- aiMetadata: { description: 'Runs a raw Browser Query Language (BQL) GraphQL query against a headless Chromium browser for advanced automation such as navigation, clicking, typing, and multi-step scripted browser flows. Use when the simpler screenshot/scrape/PDF actions cannot express the interaction; requires a valid BQL query string and supports session options like stealth, proxy, and cookies. Not idempotent: the query can perform stateful browser actions and side-effecting navigation, so repeating it may not yield the same result.', idempotent: false },
+ aiMetadata: { description: 'Runs a raw Browser Query Language (BQL) GraphQL query against a headless Chromium browser for advanced automation such as navigation, clicking, typing, and multi-step scripted browser flows. Use when the simpler screenshot/scrape/PDF actions cannot express the interaction; requires a valid BQL query string and supports session options like stealth, proxy, and cookies. It fails when the query returns errors and no data; partial results keep their errors in the output. Not idempotent: the query can click, type and submit forms, so a retry can repeat those effects.', idempotent: false },
auth: browserlessAuth,
props: {
query: Property.LongText({
@@ -31,7 +36,7 @@ export const runBqlQuery = createAction({
displayName: 'Timeout (ms)',
description: 'Maximum execution time in milliseconds',
required: false,
- defaultValue: 30000,
+ defaultValue: DEFAULT_TIMEOUT_MS,
}),
stealth: Property.Checkbox({
@@ -176,121 +181,117 @@ export const runBqlQuery = createAction({
}
}),
},
+ outputSchema: browserlessOutputSchemas.runBqlQuery,
async run(context) {
- const requestBody: any = {
- query: context.propsValue.query,
- };
-
- if (context.propsValue.variables) {
- requestBody.variables = context.propsValue.variables;
- }
-
- if (context.propsValue.operationName) {
- requestBody.operationName = context.propsValue.operationName;
- }
-
- let resourceUri = '/chromium/bql';
-
- const queryParams: string[] = [];
-
- if (context.propsValue.timeout) {
- queryParams.push(`timeout=${context.propsValue.timeout}`);
- }
-
- if (context.propsValue.stealth !== undefined) {
- queryParams.push(`stealth=${context.propsValue.stealth}`);
- }
-
- if (context.propsValue.headless !== undefined) {
- queryParams.push(`headless=${context.propsValue.headless}`);
- }
-
- if (context.propsValue.humanlike) {
- queryParams.push(`humanlike=true`);
- }
-
- if (context.propsValue.proxy && context.propsValue.proxy !== 'none') {
- queryParams.push(`proxy=${context.propsValue.proxy}`);
- if (context.propsValue.proxyCountry) {
- queryParams.push(`proxyCountry=${context.propsValue.proxyCountry}`);
- }
- if (context.propsValue.proxySticky) {
- queryParams.push(`proxySticky=true`);
- }
- }
-
- if (context.propsValue.blockAds) {
- queryParams.push(`blockAds=true`);
+ const props = context.propsValue;
+ if (!browserlessBody.nonEmpty(props.query)) {
+ throw new Error('Enter a BQL query.');
}
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1 }) ?? DEFAULT_TIMEOUT_MS;
+ const slowMo = browserlessBody.optionalNumber({ value: props.slowMo, label: 'Slow Motion', min: 0 });
+ const viewportWidth = browserlessBody.optionalNumber({ value: props.viewportWidth, label: 'Viewport Width', min: 1 });
+ const viewportHeight = browserlessBody.optionalNumber({ value: props.viewportHeight, label: 'Viewport Height', min: 1 });
+ const useProxy = browserlessBody.nonEmpty(props.proxy) && props.proxy !== 'none';
+ const cookies = (props.cookies ?? [])
+ .map(browserlessValues.record)
+ .filter((cookie) => browserlessBody.nonEmpty(String(cookie['name'] ?? '')))
+ .map((cookie) => ({
+ name: String(cookie['name']),
+ value: String(cookie['value'] ?? ''),
+ ...(browserlessBody.nonEmpty(stringOrUndefined(cookie['url'])) ? { url: cookie['url'] } : {}),
+ ...(browserlessBody.nonEmpty(stringOrUndefined(cookie['domain'])) ? { domain: cookie['domain'] } : {}),
+ ...(browserlessBody.nonEmpty(stringOrUndefined(cookie['path'])) ? { path: cookie['path'] } : {}),
+ ...(typeof cookie['secure'] === 'boolean' ? { secure: cookie['secure'] } : {}),
+ ...(typeof cookie['httpOnly'] === 'boolean' ? { httpOnly: cookie['httpOnly'] } : {}),
+ ...(browserlessBody.nonEmpty(stringOrUndefined(cookie['sameSite'])) ? { sameSite: cookie['sameSite'] } : {}),
+ ...(cookie['expires'] !== undefined && cookie['expires'] !== null && cookie['expires'] !== '' ? { expires: Number(cookie['expires']) } : {}),
+ }));
- if (context.propsValue.blockConsentModals) {
- queryParams.push(`blockConsentModals=true`);
- }
-
- if (context.propsValue.record) {
- queryParams.push(`record=true`);
- }
-
- if (context.propsValue.slowMo) {
- queryParams.push(`slowMo=${context.propsValue.slowMo}`);
- }
-
- if (context.propsValue.ignoreHTTPSErrors) {
- queryParams.push(`ignoreHTTPSErrors=true`);
- }
-
- if (context.propsValue.userAgent) {
- queryParams.push(`userAgent=${encodeURIComponent(context.propsValue.userAgent)}`);
- }
-
- if (context.propsValue.viewportWidth && context.propsValue.viewportHeight) {
- queryParams.push(`viewport=${context.propsValue.viewportWidth}x${context.propsValue.viewportHeight}`);
- }
-
- if (context.propsValue.cookies && context.propsValue.cookies.length > 0) {
- const cookiesJson = JSON.stringify(context.propsValue.cookies.map((cookie: any) => ({
- name: cookie.name,
- value: cookie.value,
- ...(cookie.url && { url: cookie.url }),
- ...(cookie.domain && { domain: cookie.domain }),
- ...(cookie.path && { path: cookie.path }),
- ...(cookie.secure !== undefined && { secure: cookie.secure }),
- ...(cookie.httpOnly !== undefined && { httpOnly: cookie.httpOnly }),
- ...(cookie.sameSite && { sameSite: cookie.sameSite }),
- ...(cookie.expires !== undefined && { expires: cookie.expires })
- })));
- queryParams.push(`cookies=${encodeURIComponent(cookiesJson)}`);
- }
-
- if (queryParams.length > 0) {
- resourceUri += `?${queryParams.join('&')}`;
- }
+ const query = {
+ timeout,
+ stealth: props.stealth !== false,
+ headless: props.headless !== false,
+ humanlike: props.humanlike === true ? true : undefined,
+ proxy: useProxy ? props.proxy : undefined,
+ proxyCountry: useProxy && browserlessBody.nonEmpty(props.proxyCountry) ? props.proxyCountry.trim() : undefined,
+ proxySticky: useProxy && props.proxySticky === true ? true : undefined,
+ blockAds: props.blockAds === true ? true : undefined,
+ blockConsentModals: props.blockConsentModals === true ? true : undefined,
+ record: props.record === true ? true : undefined,
+ slowMo,
+ ignoreHTTPSErrors: props.ignoreHTTPSErrors === true ? true : undefined,
+ userAgent: browserlessBody.nonEmpty(props.userAgent) ? props.userAgent.trim() : undefined,
+ viewport: viewportWidth !== undefined && viewportHeight !== undefined ? `${viewportWidth}x${viewportHeight}` : undefined,
+ cookies: cookies.length > 0 ? JSON.stringify(cookies) : undefined,
+ };
- const response = await browserlessCommon.apiCall({
+ const response = await browserlessApi.request<unknown>({
auth: context.auth.props,
method: HttpMethod.POST,
- resourceUri,
- body: requestBody,
+ path: '/chromium/bql',
+ body: {
+ query: props.query,
+ ...(props.variables && Object.keys(props.variables).length > 0 ? { variables: props.variables } : {}),
+ ...(browserlessBody.nonEmpty(props.operationName) ? { operationName: props.operationName.trim() } : {}),
+ },
+ query,
+ timeoutMs: timeout + 30_000,
+ operation: 'Run BQL Query',
});
- let parsedResult;
- try {
- parsedResult = typeof response.body === 'string' ? JSON.parse(response.body) : response.body;
- } catch (error) {
- parsedResult = response.body;
+ const parsed = parseBody(response.body);
+ const data = readField({ value: parsed, key: 'data' });
+ const errors = readField({ value: parsed, key: 'errors' });
+ const hasData = data !== null && typeof data === 'object' && Object.keys(data).length > 0;
+ if (!hasData && Array.isArray(errors) && errors.length > 0) {
+ throw new BrowserlessApiError({
+ message: `Run BQL Query failed: ${errors.map(errorText).join('; ').slice(0, 1000)}`,
+ status: response.status,
+ responseBody: parsed,
+ });
}
return {
success: true,
- data: parsedResult?.data || null,
- errors: parsedResult?.errors || null,
- result: parsedResult,
+ data: hasData ? data : null,
+ errors: Array.isArray(errors) && errors.length > 0 ? errors : null,
+ result: parsed,
metadata: {
browserType: 'chromium',
- executionTime: response.headers?.['x-response-time'] || 'unknown',
+ executionTime: browserlessApi.headerValue({ headers: response.headers, name: 'x-response-time' }) ?? 'unknown',
timestamp: new Date().toISOString(),
- stealth: context.propsValue.stealth || false,
- }
+ stealth: props.stealth === true,
+ },
};
},
});
+
+function parseBody(body: unknown): unknown {
+ if (typeof body !== 'string') {
+ return body;
+ }
+ try {
+ return JSON.parse(body);
+ } catch {
+ return body;
+ }
+}
+
+function readField({ value, key }: { value: unknown; key: string }): unknown {
+ if (typeof value !== 'object' || value === null) {
+ return null;
+ }
+ const field: unknown = Reflect.get(value, key);
+ return field ?? null;
+}
+
+function errorText(error: unknown): string {
+ if (typeof error === 'object' && error !== null && 'message' in error && typeof error.message === 'string') {
+ return error.message;
+ }
+ return JSON.stringify(error);
+}
+
+function stringOrUndefined(value: unknown): string | undefined {
+ return typeof value === 'string' ? value : undefined;
+}
diff --git a/packages/pieces/community/browserless/src/lib/actions/run-function.ts b/packages/pieces/community/browserless/src/lib/actions/run-function.ts
new file mode 100644
index 00000000000..ca5a9ea7ad7
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/run-function.ts
@@ -0,0 +1,150 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { browserlessApi } from '../common/client';
+import { browserlessBody, browserlessProps } from '../common/props';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const runFunction = createAction({
+ auth: browserlessAuth,
+ name: 'run_function',
+ classification: 'WRITE',
+ displayName: 'Run Puppeteer Function',
+ description: 'Run your own Puppeteer code in a Browserless browser and return what it returns.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Runs caller-supplied Puppeteer code (an ES module whose default export receives { page, context }) in a fresh Browserless browser and returns the `data` it returns: JSON or text, or a stored file when `type` is binary (for example application/pdf). Use only when Smart Scrape, Scrape URL, Capture Screenshot or BQL cannot do the job. The code can click, submit forms and change remote state, so it is not safe to retry blindly.',
+ idempotent: false,
+ },
+ props: {
+ instructions: Property.MarkDown({
+ value: 'Write an ES module whose default export is an async function. It receives `{ page, context }` and should return `{ data, type }`, for example:\n\n```js\nexport default async function ({ page, context }) {\n await page.goto(context.url);\n return { data: { title: await page.title() }, type: "application/json" };\n}\n```',
+ }),
+ code: Property.LongText({
+ displayName: 'Code',
+ description: 'The Puppeteer module to run.',
+ required: true,
+ }),
+ context: Property.Object({
+ displayName: 'Context',
+ description: 'Values your function receives as `context`.',
+ required: false,
+ }),
+ timeout: browserlessProps.sessionTimeout(),
+ },
+ outputSchema: browserlessOutputSchemas.runFunction,
+ async run(context) {
+ const props = context.propsValue;
+ if (!browserlessBody.nonEmpty(props.code)) {
+ throw new Error('Enter the code to run.');
+ }
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1000, max: 540_000 });
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/function',
+ body: {
+ code: props.code,
+ ...(props.context && Object.keys(props.context).length > 0 ? { context: props.context } : {}),
+ },
+ query: { timeout },
+ responseType: 'arraybuffer',
+ timeoutMs: timeout === undefined ? undefined : timeout + 30_000,
+ operation: 'Run Puppeteer Function',
+ });
+
+ const headerType = (browserlessApi.headerValue({ headers: response.headers, name: 'content-type' }) ?? 'application/octet-stream').toLowerCase();
+ const buffer = browserlessApi.toBuffer(response.body);
+ if (!isTextual(headerType)) {
+ return writeFileResult({ files: context.files, contentType: headerType, bytes: buffer });
+ }
+ const text = buffer.toString('utf8');
+ const parsed = headerType.includes('json') ? parseJson(text) : text;
+ const envelope = readEnvelope(parsed);
+ if (envelope === null) {
+ return { content_type: headerType, result: parsed, file: null, size_bytes: buffer.length };
+ }
+ if (!isTextual(envelope.type)) {
+ const bytes = toBytes(envelope.data);
+ if (bytes !== null) {
+ return writeFileResult({ files: context.files, contentType: envelope.type, bytes });
+ }
+ }
+ return { content_type: envelope.type, result: envelope.data, file: null, size_bytes: buffer.length };
+ },
+});
+
+async function writeFileResult({ files, contentType, bytes }: { files: { write: (params: { fileName: string; data: Buffer }) => Promise<string> }; contentType: string; bytes: Buffer }) {
+ const file = await files.write({ fileName: `function-result.${extensionFor(contentType)}`, data: bytes });
+ return { content_type: contentType, result: null, file, size_bytes: bytes.length };
+}
+
+function readEnvelope(value: unknown): { data: unknown; type: string } | null {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ return null;
+ }
+ const keys = Object.keys(value);
+ const type: unknown = Reflect.get(value, 'type');
+ if (!keys.includes('data') || typeof type !== 'string' || keys.some((key) => key !== 'data' && key !== 'type')) {
+ return null;
+ }
+ return { data: Reflect.get(value, 'data'), type: type.toLowerCase() };
+}
+
+function toBytes(value: unknown): Buffer | null {
+ if (typeof value === 'string') {
+ return Buffer.from(value, 'base64');
+ }
+ if (Array.isArray(value)) {
+ return value.every(isByte) ? Buffer.from(value) : null;
+ }
+ if (typeof value !== 'object' || value === null) {
+ return null;
+ }
+ const nodeBuffer: unknown = Reflect.get(value, 'data');
+ if (Reflect.get(value, 'type') === 'Buffer' && Array.isArray(nodeBuffer)) {
+ return nodeBuffer.every(isByte) ? Buffer.from(nodeBuffer) : null;
+ }
+ const entries = Object.entries(value);
+ if (entries.length === 0 || !entries.every(([key, byte], index) => key === String(index) && isByte(byte))) {
+ return null;
+ }
+ return Buffer.from(entries.map(([, byte]) => Number(byte)));
+}
+
+function isByte(value: unknown): boolean {
+ return typeof value === 'number' && Number.isInteger(value) && value >= 0 && value <= 255;
+}
+
+function isTextual(contentType: string): boolean {
+ return contentType.startsWith('text/') || contentType.includes('json') || contentType.includes('xml') || contentType.includes('javascript');
+}
+
+function parseJson(text: string): unknown {
+ try {
+ return JSON.parse(text);
+ } catch {
+ return text;
+ }
+}
+
+function extensionFor(contentType: string): string {
+ if (contentType.includes('pdf')) {
+ return 'pdf';
+ }
+ if (contentType.includes('png')) {
+ return 'png';
+ }
+ if (contentType.includes('jpeg') || contentType.includes('jpg')) {
+ return 'jpg';
+ }
+ if (contentType.includes('webp')) {
+ return 'webp';
+ }
+ if (contentType.includes('zip')) {
+ return 'zip';
+ }
+ return 'bin';
+}
diff --git a/packages/pieces/community/browserless/src/lib/actions/scrape-url.ts b/packages/pieces/community/browserless/src/lib/actions/scrape-url.ts
index 7ab9b6ac8e5..5af46db39cb 100644
--- a/packages/pieces/community/browserless/src/lib/actions/scrape-url.ts
+++ b/packages/pieces/community/browserless/src/lib/actions/scrape-url.ts
@@ -1,7 +1,14 @@
import { createAction, Property } from '@activepieces/pieces-framework';
import { HttpMethod } from '@activepieces/pieces-common';
import { browserlessAuth } from '../common/auth';
-import { browserlessCommon } from '../common/client';
+import { browserlessApi } from '../common/client';
+import { browserlessBody } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+const DEFAULT_WAIT_UNTIL = 'networkidle2';
+const DEFAULT_VIEWPORT = { width: 1920, height: 1080 };
export const scrapeUrl = createAction({
name: 'scrape_url',
@@ -9,7 +16,7 @@ export const scrapeUrl = createAction({
displayName: 'Scrape URL',
description: 'Extract content from a web page',
audience: 'both',
- aiMetadata: { description: 'Loads a web page in a headless browser and extracts the elements matched by the CSS selectors you supply. Use to pull specific text or attributes from a page rather than capturing an image; both the page URL and at least one element selector are required. Read-only: re-running with the same input re-scrapes the page and returns equivalent data without side effects.', idempotent: true },
+ aiMetadata: { description: 'Loads a web page in a headless browser and extracts the elements matched by the CSS selectors you supply. Use when you know which CSS selectors hold the data; for a whole page as markdown or text use Smart Scrape, for raw HTML use Get Page Content. The page URL and at least one selector are required. Read-only and safe to retry.', idempotent: true },
auth: browserlessAuth,
props: {
url: Property.ShortText({
@@ -123,13 +130,13 @@ export const scrapeUrl = createAction({
displayName: 'Timeout (ms)',
description: 'Maximum time to wait for the page to load',
required: false,
- defaultValue: 30000,
+ defaultValue: DEFAULT_TIMEOUT_MS,
}),
waitUntil: Property.StaticDropdown({
displayName: 'Wait Until',
description: 'When to consider navigation complete',
required: false,
- defaultValue: 'networkidle2',
+ defaultValue: DEFAULT_WAIT_UNTIL,
options: {
options: [
{ label: 'Load Event', value: 'load' },
@@ -148,145 +155,121 @@ export const scrapeUrl = createAction({
displayName: 'Viewport Width',
description: 'Browser viewport width in pixels',
required: false,
- defaultValue: 1920,
+ defaultValue: DEFAULT_VIEWPORT.width,
}),
viewportHeight: Property.Number({
displayName: 'Viewport Height',
description: 'Browser viewport height in pixels',
required: false,
- defaultValue: 1080,
+ defaultValue: DEFAULT_VIEWPORT.height,
}),
},
+ outputSchema: browserlessOutputSchemas.scrapeUrl,
async run(context) {
- const requestBody: any = {
- url: context.propsValue.url,
- elements: (context.propsValue.elements || []).map((element: any) => {
- let selector = element.selector;
- if (typeof selector === 'string') {
- try {
- const parsed = JSON.parse(selector);
- if (parsed.selector) {
- selector = parsed.selector;
- }
- } catch (e) {
- console.error('Error parsing selector:', e);
- }
- } else if (typeof selector === 'object' && selector.selector) {
- selector = selector.selector;
- }
-
- const elementConfig: any = {
- selector: selector,
- };
-
- if (element.timeout !== undefined) {
- elementConfig.timeout = element.timeout;
- }
-
- return elementConfig;
- })
- };
-
- if (context.propsValue.timeout || context.propsValue.waitUntil) {
- requestBody.gotoOptions = {};
- if (context.propsValue.timeout) {
- requestBody.gotoOptions.timeout = context.propsValue.timeout;
- }
- if (context.propsValue.waitUntil) {
- requestBody.gotoOptions.waitUntil = context.propsValue.waitUntil;
- }
- }
-
- if (context.propsValue.waitForSelector) {
- const waitForSelectorObj: any = {
- selector: context.propsValue.waitForSelector,
- };
-
- if (context.propsValue.waitForSelectorTimeout !== undefined) {
- waitForSelectorObj.timeout = context.propsValue.waitForSelectorTimeout;
+ const props = context.propsValue;
+ const elements = (props.elements ?? []).map(browserlessValues.record).map((element, index) => {
+ const selector = readSelector(element['selector']);
+ if (selector === '') {
+ throw new Error(`Element ${index + 1} needs a CSS selector.`);
}
-
- if (context.propsValue.waitForSelectorVisible !== undefined) {
- waitForSelectorObj.visible = context.propsValue.waitForSelectorVisible;
- }
-
- if (context.propsValue.waitForSelectorHidden !== undefined) {
- waitForSelectorObj.hidden = context.propsValue.waitForSelectorHidden;
- }
-
- requestBody.waitForSelector = waitForSelectorObj;
- }
-
- if (context.propsValue.waitForTimeout) {
- requestBody.waitForTimeout = context.propsValue.waitForTimeout;
- }
-
- if (context.propsValue.waitForEvent) {
- const waitForEventObj: any = {
- event: context.propsValue.waitForEvent,
- };
-
- if (context.propsValue.waitForEventTimeout !== undefined) {
- waitForEventObj.timeout = context.propsValue.waitForEventTimeout;
- }
-
- requestBody.waitForEvent = waitForEventObj;
- }
-
- const debugOpts: any = {};
- if (context.propsValue.debugConsole) debugOpts.console = true;
- if (context.propsValue.debugCookies) debugOpts.cookies = true;
- if (context.propsValue.debugNetwork) debugOpts.network = true;
-
- if (Object.keys(debugOpts).length > 0) {
- requestBody.debugOpts = debugOpts;
- }
-
- if (context.propsValue.bestAttempt) {
- requestBody.bestAttempt = context.propsValue.bestAttempt;
- }
-
- if (context.propsValue.waitForFunction) {
- requestBody.waitForFunction = {
- fn: context.propsValue.waitForFunction
- };
- }
-
- if (context.propsValue.userAgent) {
- requestBody.userAgent = context.propsValue.userAgent;
+ const timeout = browserlessBody.optionalNumber({ value: element['timeout'], label: `Element ${index + 1} timeout`, min: 0 });
+ return { selector, ...(timeout !== undefined ? { timeout } : {}) };
+ });
+ if (elements.length === 0) {
+ throw new Error('Add at least one CSS selector under Elements to Extract.');
}
- if (context.propsValue.viewportWidth && context.propsValue.viewportHeight) {
- requestBody.viewport = {
- width: context.propsValue.viewportWidth,
- height: context.propsValue.viewportHeight,
- };
- }
+ const navigationTimeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 0 }) ?? DEFAULT_TIMEOUT_MS;
+ const selectorTimeout = browserlessBody.optionalNumber({ value: props.waitForSelectorTimeout, label: 'Wait for Selector Timeout', min: 0 });
+ const eventTimeout = browserlessBody.optionalNumber({ value: props.waitForEventTimeout, label: 'Wait for Event Timeout', min: 0 });
+ const waitForTimeout = browserlessBody.optionalNumber({ value: props.waitForTimeout, label: 'Wait Timeout', min: 0 });
+ const viewportWidth = browserlessBody.optionalNumber({ value: props.viewportWidth, label: 'Viewport Width', min: 1 }) ?? DEFAULT_VIEWPORT.width;
+ const viewportHeight = browserlessBody.optionalNumber({ value: props.viewportHeight, label: 'Viewport Height', min: 1 }) ?? DEFAULT_VIEWPORT.height;
- if (context.propsValue.cookies && context.propsValue.cookies.length > 0) {
- requestBody.cookies = context.propsValue.cookies.map((cookie: any) => ({
+ const gotoOptions = {
+ timeout: navigationTimeout,
+ waitUntil: browserlessBody.nonEmpty(props.waitUntil) ? props.waitUntil : DEFAULT_WAIT_UNTIL,
+ };
+ const debugOpts = {
+ ...(props.debugConsole === true ? { console: true } : {}),
+ ...(props.debugCookies === true ? { cookies: true } : {}),
+ ...(props.debugNetwork === true ? { network: true } : {}),
+ };
+ const cookies = (props.cookies ?? [])
+ .map(browserlessValues.record)
+ .map((cookie) => ({
+ name: String(cookie['name'] ?? '').trim(),
+ value: String(cookie['value'] ?? ''),
+ domain: String(cookie['domain'] ?? '').trim(),
+ }))
+ .filter((cookie) => cookie.name !== '')
+ .map((cookie) => ({
name: cookie.name,
value: cookie.value,
- ...(cookie.domain && { domain: cookie.domain })
+ ...(cookie.domain !== '' ? { domain: cookie.domain } : { url: props.url }),
}));
- }
+ const requestBody = {
+ url: props.url,
+ elements,
+ gotoOptions,
+ ...(browserlessBody.nonEmpty(props.waitForSelector)
+ ? {
+ waitForSelector: {
+ selector: props.waitForSelector.trim(),
+ ...(selectorTimeout !== undefined ? { timeout: selectorTimeout } : {}),
+ ...(props.waitForSelectorHidden === true ? { hidden: true } : props.waitForSelectorVisible !== false ? { visible: true } : {}),
+ },
+ }
+ : {}),
+ ...(waitForTimeout !== undefined ? { waitForTimeout } : {}),
+ ...(browserlessBody.nonEmpty(props.waitForEvent)
+ ? { waitForEvent: { event: props.waitForEvent.trim(), ...(eventTimeout !== undefined ? { timeout: eventTimeout } : {}) } }
+ : {}),
+ ...(Object.keys(debugOpts).length > 0 ? { debugOpts } : {}),
+ ...(props.bestAttempt === true ? { bestAttempt: true } : {}),
+ ...(browserlessBody.nonEmpty(props.waitForFunction) ? { waitForFunction: { fn: props.waitForFunction } } : {}),
+ ...(browserlessBody.nonEmpty(props.userAgent) ? { userAgent: { userAgent: props.userAgent.trim() } } : {}),
+ viewport: { width: viewportWidth, height: viewportHeight },
+ ...(cookies.length > 0 ? { cookies } : {}),
+ };
- const response = await browserlessCommon.apiCall({
+ const response = await browserlessApi.request({
auth: context.auth.props,
method: HttpMethod.POST,
- resourceUri: '/scrape',
+ path: '/scrape',
body: requestBody,
+ operation: 'Scrape URL',
});
return {
success: true,
data: response.body,
metadata: {
- url: context.propsValue.url,
- elementsCount: (context.propsValue.elements || []).length,
+ url: props.url,
+ elementsCount: elements.length,
timestamp: new Date().toISOString(),
- }
+ ...browserlessApi.siteResponse(response.headers),
+ },
};
},
});
+
+function readSelector(value: unknown): string {
+ if (typeof value === 'string') {
+ const trimmed = value.trim();
+ return trimmed.startsWith('{') ? readSelector(parseJsonObject(trimmed)) || trimmed : trimmed;
+ }
+ if (typeof value === 'object' && value !== null && 'selector' in value && typeof value.selector === 'string') {
+ return value.selector.trim();
+ }
+ return '';
+}
+
+function parseJsonObject(text: string): unknown {
+ try {
+ return JSON.parse(text);
+ } catch {
+ return null;
+ }
+}
diff --git a/packages/pieces/community/browserless/src/lib/actions/search-web.ts b/packages/pieces/community/browserless/src/lib/actions/search-web.ts
new file mode 100644
index 00000000000..6cfd3ad5a90
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/search-web.ts
@@ -0,0 +1,191 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { BrowserlessApiError, browserlessApi } from '../common/client';
+import { browserlessBody, browserlessProps } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const searchWeb = createAction({
+ auth: browserlessAuth,
+ name: 'search_web',
+ classification: 'SEARCH',
+ displayName: 'Search the Web',
+ description: 'Search the web (pages, news or images) and optionally read each result page as markdown.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Runs a web search and returns ranked results (title, URL, snippet) from web, news and/or image sources, optionally scraping each result page into markdown (each page cut to Maximum Characters, default 100000). Use to find pages about a topic before reading them; to read one known URL use Smart Scrape. The result count per source is capped by the Browserless plan (Free 3, Starter 10, Scale 20). Read-only and safe to retry.',
+ idempotent: true,
+ },
+ props: {
+ query: Property.ShortText({
+ displayName: 'Search Query',
+ description: 'What to search for, for example `best headless browser 2026`.',
+ required: true,
+ }),
+ sources: Property.StaticMultiSelectDropdown({
+ displayName: 'Sources',
+ description: 'Where to search. Web is used when empty.',
+ required: false,
+ defaultValue: ['web'],
+ options: {
+ options: [
+ { label: 'Web', value: 'web' },
+ { label: 'News', value: 'news' },
+ { label: 'Images', value: 'images' },
+ ],
+ },
+ }),
+ limit: Property.Number({
+ displayName: 'Results per Source',
+ description: 'Results per source (default 10, capped by your plan).',
+ required: false,
+ }),
+ tbs: Property.StaticDropdown({
+ displayName: 'Time Range',
+ description: 'Only return results from this period.',
+ required: false,
+ options: {
+ options: [
+ { label: 'Past Day', value: 'day' },
+ { label: 'Past Week', value: 'week' },
+ { label: 'Past Month', value: 'month' },
+ { label: 'Past Year', value: 'year' },
+ ],
+ },
+ }),
+ categories: Property.StaticMultiSelectDropdown({
+ displayName: 'Categories',
+ description: 'Narrow results to GitHub repositories, research papers or PDF documents.',
+ required: false,
+ options: {
+ options: [
+ { label: 'GitHub', value: 'github' },
+ { label: 'Research', value: 'research' },
+ { label: 'PDF', value: 'pdf' },
+ ],
+ },
+ }),
+ lang: Property.ShortText({
+ displayName: 'Language',
+ description: 'Two-letter language code for results, for example `en` or `de` (default `en`).',
+ required: false,
+ }),
+ country: Property.ShortText({
+ displayName: 'Country',
+ description: 'Two-letter country code for local results, for example `us` or `gb`.',
+ required: false,
+ }),
+ location: Property.ShortText({
+ displayName: 'Location',
+ description: 'City or region to narrow results, used with Country.',
+ required: false,
+ }),
+ scrapeFormats: Property.StaticMultiSelectDropdown({
+ displayName: 'Read Result Pages As',
+ description: 'Also read each result page in these formats (slower, costs more).',
+ required: false,
+ options: {
+ options: [
+ { label: 'Markdown', value: 'markdown' },
+ { label: 'HTML', value: 'html' },
+ { label: 'Links', value: 'links' },
+ ],
+ },
+ }),
+ onlyMainContent: Property.Checkbox({
+ displayName: 'Only Main Content',
+ description: 'When reading result pages, drop navigation, headers and footers.',
+ required: false,
+ defaultValue: false,
+ }),
+ proxy: browserlessProps.proxy({
+ description: 'Proxy network used to read result pages. Browserless uses Residential when empty; Datacenter is cheaper.',
+ }),
+ maxCharacters: browserlessProps.maxCharacters(),
+ timeout: browserlessProps.sessionTimeout(),
+ },
+ outputSchema: browserlessOutputSchemas.searchWeb,
+ async run(context) {
+ const props = context.propsValue;
+ const query = (props.query ?? '').trim();
+ if (query === '') {
+ throw new Error('Enter a search query.');
+ }
+ const limit = browserlessBody.optionalNumber({ value: props.limit, label: 'Results per Source', min: 1, max: 100 });
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1000, max: 540_000 });
+ const sources = browserlessBody.textList(props.sources);
+ const categories = browserlessBody.textList(props.categories);
+ const scrapeFormats = browserlessBody.textList(props.scrapeFormats);
+ const maxCharacters = browserlessBody.maxCharacters(props.maxCharacters);
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/search',
+ body: {
+ query,
+ ...(limit !== undefined ? { limit: Math.floor(limit) } : {}),
+ ...(sources.length > 0 ? { sources } : {}),
+ ...(categories.length > 0 ? { categories } : {}),
+ ...(browserlessBody.nonEmpty(props.tbs) ? { tbs: props.tbs } : {}),
+ ...(browserlessBody.nonEmpty(props.lang) ? { lang: props.lang.trim().toLowerCase() } : {}),
+ ...(browserlessBody.nonEmpty(props.country) ? { country: props.country.trim().toLowerCase() } : {}),
+ ...(browserlessBody.nonEmpty(props.location) ? { location: props.location.trim() } : {}),
+ ...(browserlessBody.nonEmpty(props.proxy) ? { proxy: props.proxy } : {}),
+ ...(scrapeFormats.length > 0
+ ? { scrapeOptions: { formats: scrapeFormats, ...(props.onlyMainContent === true ? { onlyMainContent: true } : {}) } }
+ : {}),
+ ...(timeout !== undefined ? { timeout } : {}),
+ },
+ timeoutMs: timeout === undefined ? undefined : timeout + 30_000,
+ operation: 'Search the Web',
+ });
+
+ const body = browserlessValues.record(response.body);
+ const errorCode = browserlessValues.stringOrNull(body['errorCode']);
+ if (body['success'] !== true && errorCode !== 'no_results') {
+ throw new BrowserlessApiError({
+ message: `Search the Web failed: ${browserlessValues.stringOrNull(body['error']) ?? errorCode ?? 'Browserless returned no results'}`,
+ status: response.status,
+ responseBody: { success: body['success'], error: body['error'], errorCode },
+ });
+ }
+ const data = browserlessValues.record(body['data']);
+ const results = SOURCES.flatMap((source) =>
+ (Array.isArray(data[source]) ? data[source] : []).map((item: unknown) => toResult({ source, item: browserlessValues.record(item), maxCharacters })),
+ );
+
+ return {
+ query,
+ total_results: browserlessValues.numberOrNull(body['totalResults']) ?? results.length,
+ result_count: results.length,
+ no_results: errorCode === 'no_results' || results.length === 0,
+ results,
+ };
+ },
+});
+
+function toResult({ source, item, maxCharacters }: { source: string; item: Record<string, unknown>; maxCharacters: number }) {
+ const metadata = browserlessValues.record(item['metadata']);
+ const markdown = browserlessBody.cap({ text: browserlessValues.stringOrNull(item['markdown']), max: maxCharacters });
+ const html = browserlessBody.cap({ text: browserlessValues.stringOrNull(item['html']), max: maxCharacters });
+ return {
+ source,
+ position: browserlessValues.numberOrNull(item['position']),
+ title: browserlessValues.stringOrNull(item['title']),
+ url: browserlessValues.stringOrNull(item['url']),
+ description: browserlessValues.stringOrNull(item['description']),
+ date: browserlessValues.stringOrNull(item['date']),
+ image_url: browserlessValues.stringOrNull(item['imageUrl']),
+ markdown: markdown.text,
+ html: html.text,
+ truncated: markdown.truncated || html.truncated,
+ links: Array.isArray(item['links']) ? browserlessValues.stringList(item['links']) : null,
+ page_status_code: browserlessValues.numberOrNull(metadata['statusCode']),
+ page_error: browserlessValues.stringOrNull(metadata['error']),
+ };
+}
+
+const SOURCES = ['web', 'news', 'images'];
diff --git a/packages/pieces/community/browserless/src/lib/actions/smart-scrape.ts b/packages/pieces/community/browserless/src/lib/actions/smart-scrape.ts
new file mode 100644
index 00000000000..beb2c9158cb
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/smart-scrape.ts
@@ -0,0 +1,159 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { BrowserlessApiError, browserlessApi } from '../common/client';
+import { browserlessBody, browserlessProps } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const smartScrape = createAction({
+ auth: browserlessAuth,
+ name: 'smart_scrape',
+ classification: 'READ',
+ displayName: 'Smart Scrape',
+ description: 'Get a page as markdown, text, HTML or links. Browserless tries a fast fetch first and falls back to a real browser, proxy or CAPTCHA solving when the site blocks it.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Reads one web page and returns it as LLM-ready markdown (default), plain text, HTML and/or the list of links, with title and description, escalating from a plain fetch to a stealth browser when the site blocks bots. Use this to read or summarise a page; use Scrape URL for specific CSS selectors and Capture Screenshot for images. Text fields are cut to Maximum Characters (default 100000) and truncated=true says so. Fails when every strategy fails. Asking for a screenshot or PDF stores a new file on each call.',
+ idempotent: false,
+ },
+ props: {
+ url: browserlessProps.url({ required: true, description: 'The page to read, for example https://example.com/blog/post' }),
+ formats: Property.StaticMultiSelectDropdown({
+ displayName: 'Formats',
+ description: 'What to return. Markdown is the best choice for AI steps.',
+ required: false,
+ defaultValue: ['markdown'],
+ options: {
+ options: [
+ { label: 'Markdown', value: 'markdown' },
+ { label: 'Plain Text', value: 'rawText' },
+ { label: 'HTML', value: 'html' },
+ { label: 'Links', value: 'links' },
+ { label: 'Screenshot (file)', value: 'screenshot' },
+ { label: 'PDF (file)', value: 'pdf' },
+ ],
+ },
+ }),
+ onlyMainContent: Property.Checkbox({
+ displayName: 'Only Main Content',
+ description: 'Drop navigation, headers, footers and sidebars.',
+ required: false,
+ defaultValue: false,
+ }),
+ includeTags: Property.Array({
+ displayName: 'Keep Only These Selectors',
+ description: 'CSS selectors to keep; cannot be combined with Remove or Only Main Content.',
+ required: false,
+ }),
+ excludeTags: Property.Array({
+ displayName: 'Remove These Selectors',
+ description: 'CSS selectors to remove (for example `.cookie-banner`).',
+ required: false,
+ }),
+ waitFor: Property.Number({
+ displayName: 'Extra Wait (ms)',
+ description: 'Wait after load (0-30000 ms); above 0 forces a real browser.',
+ required: false,
+ }),
+ proxy: browserlessProps.proxy({
+ description: 'Proxy network used when a plain fetch is blocked. Browserless uses Residential when empty; Datacenter is cheaper.',
+ }),
+ maxCharacters: browserlessProps.maxCharacters(),
+ timeout: browserlessProps.sessionTimeout(),
+ },
+ outputSchema: browserlessOutputSchemas.smartScrape,
+ async run(context) {
+ const props = context.propsValue;
+ const url = browserlessBody.httpUrl({ value: props.url, label: 'URL' });
+ const formats = browserlessBody.textList(props.formats);
+ const maxCharacters = browserlessBody.maxCharacters(props.maxCharacters);
+ const includeTags = browserlessBody.textList(props.includeTags);
+ const excludeTags = browserlessBody.textList(props.excludeTags);
+ if (includeTags.length > 0 && (excludeTags.length > 0 || props.onlyMainContent === true)) {
+ throw new Error('"Keep Only These Selectors" cannot be combined with "Remove These Selectors" or "Only Main Content".');
+ }
+ if (includeTags.length > 100 || excludeTags.length > 100) {
+ throw new Error('Use at most 100 selectors.');
+ }
+ const waitFor = browserlessBody.optionalNumber({ value: props.waitFor, label: 'Extra Wait', min: 0, max: 30_000 });
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1000, max: 540_000 });
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/smart-scrape',
+ body: {
+ url,
+ formats: formats.length > 0 ? formats : ['markdown'],
+ ...(props.onlyMainContent === true ? { onlyMainContent: true } : {}),
+ ...(includeTags.length > 0 ? { includeTags } : {}),
+ ...(excludeTags.length > 0 ? { excludeTags } : {}),
+ ...(waitFor !== undefined ? { waitFor } : {}),
+ ...(browserlessBody.nonEmpty(props.proxy) ? { proxy: props.proxy } : {}),
+ },
+ query: { timeout },
+ timeoutMs: timeout === undefined ? undefined : timeout + 30_000,
+ operation: 'Smart Scrape',
+ });
+
+ const body = browserlessValues.record(response.body);
+ if (body['ok'] !== true) {
+ const message = typeof body['message'] === 'string' && body['message'] !== '' ? body['message'] : 'every scraping strategy failed';
+ throw new BrowserlessApiError({
+ message: `Smart Scrape failed for ${url}: ${message}`,
+ status: typeof body['statusCode'] === 'number' ? body['statusCode'] : null,
+ responseBody: { ok: body['ok'], message: body['message'], attempted: body['attempted'] },
+ });
+ }
+
+ const metadata = browserlessValues.record(body['metadata']);
+ const content = body['content'];
+ const isHtml = (browserlessValues.stringOrNull(body['contentType']) ?? 'text/html').toLowerCase().includes('html');
+ const screenshot = await browserlessValues.writeBase64File({ files: context.files, value: body['screenshot'], fileName: 'smart-scrape.png' });
+ const pdf = await browserlessValues.writeBase64File({ files: context.files, value: body['pdf'], fileName: 'smart-scrape.pdf' });
+ const markdown = browserlessBody.cap({ text: browserlessValues.stringOrNull(body['markdown']), max: maxCharacters });
+ const rawText = browserlessBody.cap({ text: browserlessValues.stringOrNull(body['rawText']), max: maxCharacters });
+ const html = browserlessBody.cap({ text: isHtml && formats.includes('html') && typeof content === 'string' ? content : null, max: maxCharacters });
+ const otherContent = isHtml ? { value: null, truncated: false } : capContent({ content, max: maxCharacters });
+ const links = Array.isArray(body['links']) ? browserlessValues.stringList(body['links']) : null;
+
+ return {
+ url,
+ status_code: browserlessValues.numberOrNull(body['statusCode']),
+ content_type: browserlessValues.stringOrNull(body['contentType']),
+ strategy: browserlessValues.stringOrNull(body['strategy']),
+ attempted_strategies: browserlessValues.stringList(body['attempted']),
+ title: browserlessValues.stringOrNull(metadata['title']),
+ description: browserlessValues.stringOrNull(metadata['description']),
+ language: browserlessValues.stringOrNull(metadata['language']),
+ source_url: browserlessValues.stringOrNull(metadata['sourceURL']),
+ markdown: markdown.text,
+ raw_text: rawText.text,
+ html: html.text,
+ content: otherContent.value,
+ truncated: markdown.truncated || rawText.truncated || html.truncated || otherContent.truncated,
+ links,
+ links_count: links === null ? 0 : links.length,
+ screenshot_file: screenshot,
+ pdf_file: pdf,
+ };
+ },
+});
+
+
+function capContent({ content, max }: { content: unknown; max: number }): { value: unknown; truncated: boolean } {
+ if (content === undefined || content === null) {
+ return { value: null, truncated: false };
+ }
+ if (typeof content === 'string') {
+ const capped = browserlessBody.cap({ text: content, max });
+ return { value: capped.text, truncated: capped.truncated };
+ }
+ const serialized = JSON.stringify(content);
+ if (max === 0 || serialized === undefined || serialized.length <= max) {
+ return { value: content, truncated: false };
+ }
+ return { value: serialized.slice(0, max), truncated: true };
+}
diff --git a/packages/pieces/community/browserless/src/lib/actions/start-crawl.ts b/packages/pieces/community/browserless/src/lib/actions/start-crawl.ts
new file mode 100644
index 00000000000..4e4da381f89
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/start-crawl.ts
@@ -0,0 +1,141 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { BrowserlessApiError, browserlessApi } from '../common/client';
+import { browserlessBody, browserlessProps } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+const DEFAULT_LIMIT = 10;
+
+export const startCrawl = createAction({
+ auth: browserlessAuth,
+ name: 'start_crawl',
+ classification: 'SEARCH',
+ displayName: 'Start Crawl',
+ description: 'Start crawling a website in the background. Browserless follows links and scrapes every page it finds; use Get Crawl Results to read them.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Starts an asynchronous crawl that follows links from a start URL and scrapes each page (markdown by default), returning a crawl ID right away. Use for many pages of one site; for a single page use Smart Scrape, for just the URL list use Map Website URLs. Poll Get Crawl Results with the ID until status is completed. Each call starts a new billed crawl, so do not retry blindly.',
+ idempotent: false,
+ },
+ props: {
+ url: browserlessProps.url({ required: true, description: 'Where the crawl starts, for example https://example.com/docs' }),
+ limit: Property.Number({
+ displayName: 'Maximum Pages',
+ description: 'Stop after this many pages (capped by your plan); each page costs units.',
+ required: false,
+ defaultValue: DEFAULT_LIMIT,
+ }),
+ maxDepth: Property.Number({
+ displayName: 'Maximum Link Depth',
+ description: 'How many links away from the start URL to follow (0-20, default 5).',
+ required: false,
+ }),
+ includePaths: Property.Array({
+ displayName: 'Only Paths Matching',
+ description: 'Regexes; only matching paths are crawled, e.g. `^/docs/`.',
+ required: false,
+ }),
+ excludePaths: Property.Array({
+ displayName: 'Skip Paths Matching',
+ description: 'Regular expressions for paths to skip, for example `^/blog/`.',
+ required: false,
+ }),
+ allowSubdomains: Property.Checkbox({
+ displayName: 'Follow Subdomains',
+ description: 'Also crawl subdomains such as blog.example.com.',
+ required: false,
+ defaultValue: false,
+ }),
+ allowExternalLinks: Property.Checkbox({
+ displayName: 'Follow External Links',
+ description: 'Also follow links to other websites.',
+ required: false,
+ defaultValue: false,
+ }),
+ sitemap: Property.StaticDropdown({
+ displayName: 'Sitemap',
+ description: 'How to use the sitemap to find pages.',
+ required: false,
+ options: {
+ options: [
+ { label: 'Auto (default)', value: 'auto' },
+ { label: 'Only the sitemap', value: 'force' },
+ { label: 'Ignore the sitemap', value: 'skip' },
+ ],
+ },
+ }),
+ formats: Property.StaticMultiSelectDropdown({
+ displayName: 'Page Formats',
+ description: 'What to store for each page (default markdown).',
+ required: false,
+ options: {
+ options: [
+ { label: 'Markdown', value: 'markdown' },
+ { label: 'HTML', value: 'html' },
+ { label: 'Plain Text', value: 'rawText' },
+ ],
+ },
+ }),
+ onlyMainContent: Property.Checkbox({
+ displayName: 'Only Main Content',
+ description: 'Drop navigation, headers and footers from each page (Browserless default is on).',
+ required: false,
+ defaultValue: true,
+ }),
+ proxy: browserlessProps.proxy({
+ description: 'Proxy network for fetching pages. Browserless uses Residential when empty; Datacenter is cheaper.',
+ }),
+ },
+ outputSchema: browserlessOutputSchemas.startCrawl,
+ async run(context) {
+ const props = context.propsValue;
+ const url = browserlessBody.httpUrl({ value: props.url, label: 'URL' });
+ const limit = browserlessBody.optionalNumber({ value: props.limit, label: 'Maximum Pages', min: 1 }) ?? DEFAULT_LIMIT;
+ const maxDepth = browserlessBody.optionalNumber({ value: props.maxDepth, label: 'Maximum Link Depth', min: 0, max: 20 });
+ const includePaths = browserlessBody.textList(props.includePaths);
+ const excludePaths = browserlessBody.textList(props.excludePaths);
+ const formats = browserlessBody.textList(props.formats);
+ const scrapeOptions = {
+ ...(formats.length > 0 ? { formats } : {}),
+ onlyMainContent: props.onlyMainContent !== false,
+ ...(browserlessBody.nonEmpty(props.proxy) ? { proxy: props.proxy } : {}),
+ };
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/crawl',
+ body: {
+ url,
+ limit: Math.floor(limit),
+ ...(maxDepth !== undefined ? { maxDepth: Math.floor(maxDepth) } : {}),
+ ...(includePaths.length > 0 ? { includePaths } : {}),
+ ...(excludePaths.length > 0 ? { excludePaths } : {}),
+ ...(props.allowSubdomains === true ? { allowSubdomains: true } : {}),
+ ...(props.allowExternalLinks === true ? { allowExternalLinks: true } : {}),
+ ...(browserlessBody.nonEmpty(props.sitemap) ? { sitemap: props.sitemap } : {}),
+ ...(Object.keys(scrapeOptions).length > 0 ? { scrapeOptions } : {}),
+ },
+ timeoutMs: 60_000,
+ operation: 'Start Crawl',
+ });
+
+ const body = browserlessValues.record(response.body);
+ const crawlId = browserlessValues.stringOrNull(body['id']);
+ if (crawlId === null) {
+ throw new BrowserlessApiError({
+ message: `Start Crawl failed: ${browserlessValues.stringOrNull(body['message']) ?? browserlessValues.stringOrNull(body['error']) ?? 'Browserless returned no crawl ID'}`,
+ status: response.status,
+ responseBody: body,
+ });
+ }
+ return {
+ crawl_id: crawlId,
+ url,
+ status: 'in-progress',
+ };
+ },
+});
diff --git a/packages/pieces/community/browserless/src/lib/actions/unblock-page.ts b/packages/pieces/community/browserless/src/lib/actions/unblock-page.ts
new file mode 100644
index 00000000000..d315539c16c
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/actions/unblock-page.ts
@@ -0,0 +1,126 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { Property, createAction } from '@activepieces/pieces-framework';
+import { browserlessAuth } from '../common/auth';
+import { browserlessApi } from '../common/client';
+import { browserlessBody, browserlessProps } from '../common/props';
+import { browserlessValues } from '../common/values';
+import { browserlessOutputSchemas } from '../output-schemas';
+
+export const unblockPage = createAction({
+ auth: browserlessAuth,
+ name: 'unblock_page',
+ classification: 'READ',
+ displayName: 'Unblock Page',
+ description: 'Open a page that blocks bots (Cloudflare, CAPTCHAs and similar) and return its HTML, cookies and a screenshot.',
+ audience: 'both',
+ aiMetadata: {
+ description:
+ 'Opens a bot-protected URL in a stealth browser, solving CAPTCHAs when enabled, and returns the unblocked HTML (cut to Maximum Characters, default 100000), the session cookies and optionally a screenshot file. Use only when Get Page Content or Smart Scrape is blocked, because it is slower and costs more (residential proxy 6 units/MB, 10 units per solved CAPTCHA). Each call stores a new screenshot file.',
+ idempotent: false,
+ },
+ props: {
+ url: browserlessProps.url({ required: true, description: 'The blocked page to open, for example https://example.com/products' }),
+ returnContent: Property.Checkbox({
+ displayName: 'Return HTML',
+ description: 'Return the page HTML.',
+ required: false,
+ defaultValue: true,
+ }),
+ returnCookies: Property.Checkbox({
+ displayName: 'Return Cookies',
+ description: 'Return the cookies the site set, so later requests can reuse the unblocked session.',
+ required: false,
+ defaultValue: true,
+ }),
+ returnScreenshot: Property.Checkbox({
+ displayName: 'Return Screenshot',
+ description: 'Return a full-page screenshot as a file.',
+ required: false,
+ defaultValue: false,
+ }),
+ solveCaptchas: Property.Checkbox({
+ displayName: 'Solve CAPTCHAs',
+ description: 'Let Browserless solve CAPTCHAs it meets (10 units per successful solve).',
+ required: false,
+ defaultValue: false,
+ }),
+ proxy: browserlessProps.proxy({
+ description: 'Proxy network to route the browser through. Residential works best against bot detection.',
+ }),
+ proxyCountry: Property.ShortText({
+ displayName: 'Proxy Country',
+ description: 'Two-letter country code for the proxy exit, for example `us` or `de`.',
+ required: false,
+ }),
+ waitUntil: browserlessProps.waitUntil(),
+ navigationTimeout: browserlessProps.navigationTimeout(),
+ waitForSelector: browserlessProps.waitForSelector(),
+ waitForTimeout: browserlessProps.waitForTimeout(),
+ bestAttempt: browserlessProps.bestAttempt(),
+ maxCharacters: browserlessProps.maxCharacters(),
+ timeout: browserlessProps.sessionTimeout(),
+ },
+ outputSchema: browserlessOutputSchemas.unblockPage,
+ async run(context) {
+ const props = context.propsValue;
+ const url = browserlessBody.httpUrl({ value: props.url, label: 'URL' });
+ const timeout = browserlessBody.optionalNumber({ value: props.timeout, label: 'Timeout', min: 1000, max: 540_000 });
+ const useProxy = browserlessBody.nonEmpty(props.proxy);
+ const returnContent = props.returnContent !== false;
+ const returnCookies = props.returnCookies !== false;
+ const returnScreenshot = props.returnScreenshot === true;
+
+ const response = await browserlessApi.request<unknown>({
+ auth: context.auth.props,
+ method: HttpMethod.POST,
+ path: '/unblock',
+ body: {
+ url,
+ content: returnContent,
+ cookies: returnCookies,
+ screenshot: returnScreenshot,
+ browserWSEndpoint: false,
+ ...browserlessBody.pageOptions(props),
+ },
+ query: {
+ proxy: useProxy ? props.proxy : undefined,
+ proxyCountry: useProxy && browserlessBody.nonEmpty(props.proxyCountry) ? props.proxyCountry.trim().toLowerCase() : undefined,
+ solveCaptchas: props.solveCaptchas === true ? true : undefined,
+ timeout,
+ },
+ timeoutMs: timeout === undefined ? undefined : timeout + 30_000,
+ operation: 'Unblock Page',
+ });
+
+ const body = browserlessValues.record(response.body);
+ const fullHtml = browserlessValues.stringOrNull(body['content']);
+ const html = browserlessBody.cap({ text: fullHtml, max: browserlessBody.maxCharacters(props.maxCharacters) });
+ const cookies = Array.isArray(body['cookies']) ? body['cookies'].map(toCookie) : [];
+ const screenshotFile = await browserlessValues.writeBase64File({ files: context.files, value: body['screenshot'], fileName: 'unblocked.jpg' });
+ return {
+ url,
+ solved: browserlessValues.booleanOrNull(body['solved']),
+ html: html.text,
+ html_length: fullHtml === null ? 0 : fullHtml.length,
+ truncated: html.truncated,
+ cookie_count: cookies.length,
+ cookies,
+ cookie_header: cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; '),
+ screenshot_file: screenshotFile,
+ };
+ },
+});
+
+function toCookie(entry: unknown) {
+ const item = browserlessValues.record(entry);
+ return {
+ name: browserlessValues.stringOrNull(item['name']) ?? '',
+ value: browserlessValues.stringOrNull(item['value']) ?? '',
+ domain: browserlessValues.stringOrNull(item['domain']),
+ path: browserlessValues.stringOrNull(item['path']),
+ expires: browserlessValues.numberOrNull(item['expires']),
+ http_only: browserlessValues.booleanOrNull(item['httpOnly']),
+ secure: browserlessValues.booleanOrNull(item['secure']),
+ same_site: browserlessValues.stringOrNull(item['sameSite']),
+ };
+}
diff --git a/packages/pieces/community/browserless/src/lib/common/auth.ts b/packages/pieces/community/browserless/src/lib/common/auth.ts
index 846d23e8332..907c86e4e12 100644
--- a/packages/pieces/community/browserless/src/lib/common/auth.ts
+++ b/packages/pieces/community/browserless/src/lib/common/auth.ts
@@ -1,4 +1,6 @@
+import { HttpMethod } from '@activepieces/pieces-common';
import { PieceAuth, Property } from '@activepieces/pieces-framework';
+import { browserlessApi } from './client';
export const browserlessAuth = PieceAuth.CustomAuth({
description: `
@@ -49,9 +51,36 @@ export const browserlessAuth = PieceAuth.CustomAuth({
}),
customBaseUrl: Property.ShortText({
displayName: 'Custom Base URL',
- description: 'Enter your custom Browserless endpoint URL',
+ description: 'Only for "Custom Endpoint": the address of your dedicated or self-hosted Browserless, for example https://chrome.browserless.io',
required: false,
}),
},
required: true,
+ validate: async ({ auth }) => {
+ try {
+ await browserlessApi.request({
+ auth,
+ method: HttpMethod.GET,
+ path: '/meta',
+ timeoutMs: 20_000,
+ operation: 'Checking the connection',
+ });
+ return { valid: true };
+ } catch (error) {
+ const status = typeof error === 'object' && error !== null && 'status' in error ? error.status : null;
+ if (status === 401 || status === 403) {
+ return {
+ valid: false,
+ error: 'Invalid API token for this endpoint. Copy the token from your Browserless dashboard and pick the region your account uses (private-fleet tokens need the Custom Endpoint).',
+ };
+ }
+ if (typeof status === 'number' && status !== 429 && status < 500) {
+ return { valid: true };
+ }
+ return {
+ valid: false,
+ error: error instanceof Error ? error.message : 'Could not reach Browserless with these settings.',
+ };
+ }
+ },
});
diff --git a/packages/pieces/community/browserless/src/lib/common/client.ts b/packages/pieces/community/browserless/src/lib/common/client.ts
index d03a85058a0..ee1aa24749d 100644
--- a/packages/pieces/community/browserless/src/lib/common/client.ts
+++ b/packages/pieces/community/browserless/src/lib/common/client.ts
@@ -1,72 +1,309 @@
-import { HttpMethod, httpClient } from '@activepieces/pieces-common';
+import { HttpMethod, QueryParams, httpClient } from '@activepieces/pieces-common';
-export interface BrowserlessAuth {
- apiToken: string;
- region: string;
- customBaseUrl?: string;
+export const browserlessApi = {
+ request,
+ resolveBaseUrl,
+ normalizeCustomBaseUrl,
+ isSameTarget,
+ clampTimeout,
+ toError,
+ toBuffer,
+ headerValue,
+ siteResponse,
+};
+
+async function request<T = unknown>({
+ auth,
+ method,
+ path,
+ body,
+ query,
+ responseType = 'json',
+ timeoutMs,
+ operation,
+}: {
+ auth: BrowserlessAuth;
+ method: HttpMethod;
+ path: string;
+ body?: unknown;
+ query?: Record<string, QueryValue>;
+ responseType?: 'json' | 'arraybuffer' | 'text';
+ timeoutMs?: number;
+ operation: string;
+}): Promise<BrowserlessResponse<T>> {
+ if (!path.startsWith('/') || path.startsWith('//')) {
+ throw new Error(`Invalid Browserless path "${path}".`);
+ }
+ const token = (auth.apiToken ?? '').trim();
+ if (token === '') {
+ throw new Error('The Browserless API token is empty. Reconnect with a valid token.');
+ }
+ const baseUrl = resolveBaseUrl(auth);
+ try {
+ const response = await httpClient.sendRequest<T>({
+ method,
+ url: `${baseUrl}${path}`,
+ headers: {
+ ...(body === undefined ? {} : { 'Content-Type': 'application/json' }),
+ Accept: responseType === 'json' ? 'application/json' : '*/*',
+ },
+ queryParams: { ...cleanQuery(query), token },
+ body,
+ responseType,
+ timeout: clampTimeout(timeoutMs),
+ followRedirects: false,
+ });
+ if (response.status < 200 || response.status >= 300) {
+ throw new BrowserlessApiError({
+ message: `${operation} failed: Browserless answered with status ${response.status} instead of a result. Redirects are not followed, so check the Custom Base URL points straight at the Browserless API.`,
+ status: response.status,
+ responseBody: response.body,
+ });
+ }
+ return {
+ status: response.status,
+ body: response.body,
+ headers: response.headers ?? {},
+ };
+ } catch (error) {
+ throw toError({ error, operation });
+ }
}
-export function convertBinaryToBase64(binaryData: any): string {
- if (typeof binaryData === 'string') {
- return Buffer.from(binaryData, 'binary').toString('base64');
- } else if (binaryData instanceof ArrayBuffer) {
- return Buffer.from(binaryData).toString('base64');
- } else if (Buffer.isBuffer(binaryData)) {
- return binaryData.toString('base64');
- } else {
- return Buffer.from(String(binaryData), 'binary').toString('base64');
- }
-}
-
-export function isBinaryResponse(headers: any): boolean {
- const contentType = headers?.get?.('content-type') || headers?.['content-type'] || '';
- return contentType.includes('image/') ||
- contentType.includes('application/pdf') ||
- contentType.includes('application/octet-stream');
-}
-
-export const browserlessCommon = {
- async apiCall({
- auth,
- method,
- resourceUri,
- body,
- headers = {},
- }: {
- auth: BrowserlessAuth;
- method: HttpMethod;
- resourceUri: string;
- body?: any;
- headers?: Record<string, string>;
- }) {
- const baseUrl = auth.region === 'custom' ? auth.customBaseUrl : auth.region;
-
- if (!baseUrl) {
- throw new Error('Base URL is required. Please configure your Browserless endpoint.');
+function resolveBaseUrl(auth: BrowserlessAuth): string {
+ if (auth.region !== 'custom') {
+ if (!BROWSERLESS_REGIONS.includes(auth.region)) {
+ throw new Error('Unknown Browserless region. Reconnect and pick a region from the list.');
}
+ return auth.region;
+ }
+ const raw = (auth.customBaseUrl ?? '').trim();
+ if (raw === '') {
+ throw new Error('Custom Base URL is required when the region is "Custom Endpoint".');
+ }
+ return normalizeCustomBaseUrl(raw);
+}
- const url = `${baseUrl}${resourceUri}`;
+function normalizeCustomBaseUrl(raw: string): string {
+ const parsed = parseUrl(raw.trim());
+ if (parsed === null) {
+ throw new Error(`Custom Base URL "${raw}" is not a valid URL. Use a full address such as https://chrome.browserless.io`);
+ }
+ if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
+ throw new Error('Custom Base URL must start with https:// or http://');
+ }
+ if (parsed.username !== '' || parsed.password !== '') {
+ throw new Error('Custom Base URL must not contain a username or password.');
+ }
+ if (parsed.search !== '' || parsed.hash !== '') {
+ throw new Error('Custom Base URL must not contain a query string or fragment. The API token is added for you.');
+ }
+ const path = parsed.pathname.replace(/\/+$/, '');
+ return `${parsed.origin}${path}`;
+}
- const requestHeaders = {
- 'Content-Type': 'application/json',
- 'Accept': '*/*',
- ...headers,
- };
+function isSameTarget({ baseUrl, url }: { baseUrl: string; url: string }): boolean {
+ const target = parseUrl(url);
+ const base = parseUrl(baseUrl);
+ if (target === null || base === null) {
+ return false;
+ }
+ if (target.origin !== base.origin || target.username !== '' || target.password !== '') {
+ return false;
+ }
+ if (target.pathname.split('/').includes('..') || /%2e%2e/i.test(url)) {
+ return false;
+ }
+ const basePath = base.pathname.replace(/\/+$/, '');
+ if (basePath === '') {
+ return true;
+ }
+ return target.pathname === basePath || target.pathname.startsWith(`${basePath}/`);
+}
+
+function clampTimeout(value: number | undefined | null): number {
+ if (value === undefined || value === null || !Number.isFinite(value) || value <= 0) {
+ return DEFAULT_TIMEOUT_MS;
+ }
+ return Math.min(Math.floor(value), MAX_TIMEOUT_MS);
+}
+
+function toError({ error, operation }: { error: unknown; operation: string }): Error {
+ if (error instanceof BrowserlessApiError) {
+ return error;
+ }
+ const response = httpErrorResponse(error);
+ if (response !== null) {
+ const detail = vendorMessage(response.body);
+ return new BrowserlessApiError({
+ message: `${operation} failed: ${explainStatus(response.status)}${detail ? `: ${detail}` : ''}`,
+ status: response.status,
+ responseBody: response.body,
+ });
+ }
+ const message = error instanceof Error ? error.message : String(error);
+ if (/abort/i.test(message)) {
+ return new BrowserlessApiError({
+ message: `${operation} failed: Browserless did not answer before the step deadline. Lower the page wait times or the timeout.`,
+ status: null,
+ responseBody: null,
+ });
+ }
+ return new BrowserlessApiError({ message: `${operation} failed: ${message}`, status: null, responseBody: null });
+}
+
+function toBuffer(body: unknown): Buffer {
+ if (Buffer.isBuffer(body)) {
+ return body;
+ }
+ if (body instanceof ArrayBuffer) {
+ return Buffer.from(body);
+ }
+ if (ArrayBuffer.isView(body)) {
+ return Buffer.from(body.buffer, body.byteOffset, body.byteLength);
+ }
+ if (typeof body === 'string') {
+ return Buffer.from(body, 'latin1');
+ }
+ throw new BrowserlessApiError({ message: 'Browserless returned an unexpected non-binary response.', status: null, responseBody: null });
+}
+
+function headerValue({ headers, name }: { headers: ResponseHeaders; name: string }): string | null {
+ const value = headers[name.toLowerCase()] ?? headers[name];
+ if (Array.isArray(value)) {
+ return value[0] ?? null;
+ }
+ return value ?? null;
+}
+
+function siteResponse(headers: ResponseHeaders): SiteResponse {
+ const code = headerValue({ headers, name: 'x-response-code' });
+ const parsed = code === null ? NaN : Number(code);
+ return {
+ site_status_code: Number.isFinite(parsed) ? parsed : null,
+ site_status_text: headerValue({ headers, name: 'x-response-status' }),
+ final_url: headerValue({ headers, name: 'x-response-url' }),
+ };
+}
+
+function parseUrl(value: string): URL | null {
+ try {
+ return new URL(value);
+ } catch {
+ return null;
+ }
+}
+
+function httpErrorResponse(error: unknown): { status: number; body: unknown } | null {
+ if (typeof error !== 'object' || error === null || !('response' in error)) {
+ return null;
+ }
+ const response = error.response;
+ if (typeof response !== 'object' || response === null || !('status' in response)) {
+ return null;
+ }
+ const status = response.status;
+ if (typeof status !== 'number') {
+ return null;
+ }
+ return { status, body: 'body' in response ? response.body : null };
+}
- let responseType: 'json' | 'arraybuffer' | 'text' = 'json';
- if (resourceUri.includes('/screenshot') || resourceUri.includes('/pdf')) {
- responseType = 'arraybuffer';
+function vendorMessage(body: unknown): string | null {
+ if (body === null || body === undefined) {
+ return null;
+ }
+ if (typeof body === 'string') {
+ return truncate(body);
+ }
+ if (Buffer.isBuffer(body)) {
+ return truncate(body.toString('utf8'));
+ }
+ if (typeof body === 'object') {
+ for (const key of ['message', 'error', 'errors']) {
+ const value: unknown = Reflect.get(body, key);
+ if (typeof value === 'string' && value.trim() !== '') {
+ return truncate(value);
+ }
}
+ return truncate(JSON.stringify(body));
+ }
+ return truncate(String(body));
+}
- const requestConfig = {
- method,
- url,
- headers: requestHeaders,
- body: body ? JSON.stringify(body) : undefined,
- queryParams: { token: auth.apiToken },
- responseType,
- };
+function truncate(text: string): string | null {
+ const trimmed = text.trim();
+ return trimmed === '' ? null : trimmed.slice(0, 500);
+}
+
+function explainStatus(status: number): string {
+ switch (status) {
+ case 400:
+ return 'Browserless rejected the request (400)';
+ case 401:
+ case 403:
+ return `Browserless refused the API token (${status}). Check the token and that it belongs to the selected region or endpoint`;
+ case 404:
+ return 'Browserless could not find that resource (404)';
+ case 408:
+ return 'The page did not finish within the timeout (408). Increase the timeout or enable Best Attempt';
+ case 429:
+ return 'Browserless is rate limiting or your concurrency limit is reached (429). Try again shortly';
+ default:
+ return status >= 500 ? `Browserless had an internal error (${status})` : `Browserless request failed (${status})`;
+ }
+}
+
+function cleanQuery(query: Record<string, QueryValue> | undefined): QueryParams {
+ const out: QueryParams = {};
+ for (const [key, value] of Object.entries(query ?? {})) {
+ if (value === undefined || value === null || value === '') {
+ continue;
+ }
+ out[key] = String(value);
+ }
+ return out;
+}
+
+const DEFAULT_TIMEOUT_MS = 300_000;
+const MAX_TIMEOUT_MS = 540_000;
+
+const BROWSERLESS_REGIONS = [
+ 'https://production-sfo.browserless.io',
+ 'https://production-lon.browserless.io',
+ 'https://production-ams.browserless.io',
+];
+
+export class BrowserlessApiError extends Error {
+ readonly status: number | null;
+ readonly responseBody: unknown;
+
+ constructor({ message, status, responseBody }: { message: string; status: number | null; responseBody: unknown }) {
+ super(message);
+ this.name = 'BrowserlessApiError';
+ this.status = status;
+ this.responseBody = responseBody;
+ }
+}
+
+export interface BrowserlessAuth {
+ apiToken: string;
+ region: string;
+ customBaseUrl?: string;
+}
+
+export type QueryValue = string | number | boolean | undefined | null;
+
+export type ResponseHeaders = Record<string, string | string[] | undefined>;
+
+export type BrowserlessResponse<T> = {
+ status: number;
+ body: T;
+ headers: ResponseHeaders;
+};
- return await httpClient.sendRequest(requestConfig);
- },
+export type SiteResponse = {
+ site_status_code: number | null;
+ site_status_text: string | null;
+ final_url: string | null;
};
diff --git a/packages/pieces/community/browserless/src/lib/common/crawl.ts b/packages/pieces/community/browserless/src/lib/common/crawl.ts
new file mode 100644
index 00000000000..b62b5068b29
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/common/crawl.ts
@@ -0,0 +1,54 @@
+import { Property } from '@activepieces/pieces-framework';
+import { browserlessValues } from './values';
+
+export const browserlessCrawl = {
+ crawlIdProp,
+ parseCrawlId,
+ toPage,
+ toSummary,
+};
+
+function crawlIdProp() {
+ return Property.ShortText({
+ displayName: 'Crawl ID',
+ description: 'The ID returned by Start Crawl (looks like `crawl_abc123`). List Crawls shows recent IDs.',
+ required: true,
+ });
+}
+
+function parseCrawlId(value: unknown): string {
+ const id = typeof value === 'string' ? value.trim() : '';
+ if (!/^[A-Za-z0-9_-]{1,200}$/.test(id)) {
+ throw new Error('Crawl ID must look like `crawl_abc123` (letters, digits, - and _ only).');
+ }
+ return id;
+}
+
+function toPage(entry: unknown) {
+ const item = browserlessValues.record(entry);
+ const metadata = browserlessValues.record(item['metadata']);
+ return {
+ status: browserlessValues.stringOrNull(item['status']),
+ url: browserlessValues.stringOrNull(metadata['sourceURL']),
+ title: browserlessValues.stringOrNull(metadata['title']),
+ description: browserlessValues.stringOrNull(metadata['description']),
+ language: browserlessValues.stringOrNull(metadata['language']),
+ status_code: browserlessValues.numberOrNull(metadata['statusCode']),
+ scraped_at: browserlessValues.stringOrNull(metadata['scrapedAt']),
+ error: browserlessValues.stringOrNull(metadata['error']),
+ content_url: browserlessValues.stringOrNull(item['contentUrl']),
+ };
+}
+
+function toSummary(entry: unknown) {
+ const item = browserlessValues.record(entry);
+ return {
+ id: browserlessValues.stringOrNull(item['id']),
+ url: browserlessValues.stringOrNull(item['url']),
+ status: browserlessValues.stringOrNull(item['status']),
+ total: browserlessValues.numberOrNull(item['total']),
+ completed: browserlessValues.numberOrNull(item['completed']),
+ created_at: browserlessValues.stringOrNull(item['createdAt']),
+ completed_at: browserlessValues.stringOrNull(item['completedAt']),
+ };
+}
diff --git a/packages/pieces/community/browserless/src/lib/common/lighthouse.ts b/packages/pieces/community/browserless/src/lib/common/lighthouse.ts
new file mode 100644
index 00000000000..365bb42821b
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/common/lighthouse.ts
@@ -0,0 +1,123 @@
+import { browserlessValues } from './values';
+
+export const browserlessLighthouse = {
+ buildSettings,
+ summarize,
+ DEFAULT_CATEGORIES: ['performance', 'accessibility', 'best-practices', 'seo'],
+};
+
+function buildSettings({
+ device,
+ onlyCategories,
+ throttling,
+ locale,
+ userAgent,
+ timeout,
+}: {
+ device: 'desktop' | 'mobile';
+ onlyCategories?: string[];
+ throttling?: string | null;
+ locale?: string | null;
+ userAgent?: string | null;
+ timeout?: number;
+}): Record<string, unknown> {
+ const preset = throttling ? THROTTLING[throttling] : undefined;
+ return {
+ formFactor: device,
+ screenEmulation: SCREEN_EMULATION[device],
+ ...(onlyCategories ? { onlyCategories } : {}),
+ ...(typeof locale === 'string' && locale.trim() !== '' ? { locale: locale.trim() } : {}),
+ ...(throttling === 'none' ? { throttlingMethod: 'provided' } : {}),
+ ...(preset ? { throttlingMethod: 'simulate', throttling: preset } : {}),
+ ...(typeof userAgent === 'string' && userAgent.trim() !== '' ? { emulatedUserAgent: userAgent.trim() } : {}),
+ ...(timeout !== undefined ? { maxWaitForLoad: timeout } : {}),
+ };
+}
+
+function summarize({ body, url, device }: { body: unknown; url: string; device: string }) {
+ const lhr = readLighthouseResult(body);
+ const categories = browserlessValues.record(lhr['categories']);
+ const audits = browserlessValues.record(lhr['audits']);
+ const runtimeError = browserlessValues.record(lhr['runtimeError']);
+ return {
+ lighthouseVersion: stringOr({ value: lhr['lighthouseVersion'], fallback: 'unknown' }),
+ summary: {
+ url,
+ finalUrl: typeof lhr['finalDisplayedUrl'] === 'string' ? lhr['finalDisplayedUrl'] : typeof lhr['finalUrl'] === 'string' ? lhr['finalUrl'] : null,
+ formFactor: device,
+ timestamp: new Date().toISOString(),
+ scores: {
+ performance: categoryScore({ categories, id: 'performance' }),
+ accessibility: categoryScore({ categories, id: 'accessibility' }),
+ bestPractices: categoryScore({ categories, id: 'best-practices' }),
+ seo: categoryScore({ categories, id: 'seo' }),
+ pwa: categoryScore({ categories, id: 'pwa' }),
+ },
+ metrics: {
+ firstContentfulPaint: auditMetric({ audits, id: 'first-contentful-paint' }),
+ largestContentfulPaint: auditMetric({ audits, id: 'largest-contentful-paint' }),
+ firstMeaningfulPaint: auditMetric({ audits, id: 'first-meaningful-paint' }),
+ speedIndex: auditMetric({ audits, id: 'speed-index' }),
+ timeToInteractive: auditMetric({ audits, id: 'interactive' }),
+ totalBlockingTime: auditMetric({ audits, id: 'total-blocking-time' }),
+ cumulativeLayoutShift: auditMetric({ audits, id: 'cumulative-layout-shift' }),
+ },
+ opportunities: OPPORTUNITIES.filter((item) => hasItems(browserlessValues.record(audits[item.audit]))).map((item) => ({
+ type: item.type,
+ title: item.title,
+ potentialSavings: stringOr({ value: browserlessValues.record(audits[item.audit])['displayValue'], fallback: 'Unknown' }),
+ })),
+ runtimeError: typeof runtimeError['message'] === 'string' ? runtimeError['message'] : null,
+ },
+ };
+}
+
+function readLighthouseResult(body: unknown): Record<string, unknown> {
+ const root = browserlessValues.record(body);
+ const data = browserlessValues.record(root['data']);
+ const candidates = [browserlessValues.record(root['lhr']), browserlessValues.record(data['lhr']), data, root];
+ return candidates.find((candidate) => 'categories' in candidate || 'audits' in candidate) ?? {};
+}
+
+function toPercent(score: unknown): number | null {
+ return typeof score === 'number' && Number.isFinite(score) ? Math.round(score * 100) : null;
+}
+
+function categoryScore({ categories, id }: { categories: Record<string, unknown>; id: string }): number | null {
+ return toPercent(browserlessValues.record(categories[id])['score']);
+}
+
+function auditMetric({ audits, id }: { audits: Record<string, unknown>; id: string }) {
+ const audit = browserlessValues.record(audits[id]);
+ return {
+ value: typeof audit['displayValue'] === 'string' ? audit['displayValue'] : null,
+ numericValue: typeof audit['numericValue'] === 'number' ? audit['numericValue'] : null,
+ score: toPercent(audit['score']),
+ };
+}
+
+function hasItems(audit: Record<string, unknown>): boolean {
+ const items = browserlessValues.record(audit['details'])['items'];
+ return Array.isArray(items) && items.length > 0;
+}
+
+function stringOr({ value, fallback }: { value: unknown; fallback: string }): string {
+ return typeof value === 'string' && value !== '' ? value : fallback;
+}
+
+const THROTTLING: Record<string, { rttMs: number; throughputKbps: number; cpuSlowdownMultiplier: number }> = {
+ mobileSlow4G: { rttMs: 150, throughputKbps: 1638.4, cpuSlowdownMultiplier: 4 },
+ mobileRegular4G: { rttMs: 100, throughputKbps: 2048, cpuSlowdownMultiplier: 3 },
+ mobileFast4G: { rttMs: 50, throughputKbps: 4096, cpuSlowdownMultiplier: 2 },
+};
+
+const SCREEN_EMULATION = {
+ desktop: { mobile: false, width: 1350, height: 940, deviceScaleFactor: 1, disabled: false },
+ mobile: { mobile: true, width: 412, height: 823, deviceScaleFactor: 1.75, disabled: false },
+};
+
+const OPPORTUNITIES = [
+ { audit: 'unused-css-rules', type: 'unused-css', title: 'Remove unused CSS' },
+ { audit: 'unused-javascript', type: 'unused-javascript', title: 'Remove unused JavaScript' },
+ { audit: 'render-blocking-resources', type: 'render-blocking', title: 'Eliminate render-blocking resources' },
+];
diff --git a/packages/pieces/community/browserless/src/lib/common/props.ts b/packages/pieces/community/browserless/src/lib/common/props.ts
new file mode 100644
index 00000000000..0795d5c8e69
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/common/props.ts
@@ -0,0 +1,180 @@
+import { Property } from '@activepieces/pieces-framework';
+
+export const browserlessProps = {
+ url: ({ required, description }: { required: boolean; description: string }) =>
+ Property.ShortText({
+ displayName: 'URL',
+ description,
+ required,
+ }),
+ waitUntil: () =>
+ Property.StaticDropdown({
+ displayName: 'Wait Until',
+ description: 'When the page counts as loaded. "Network Idle" waits longest and suits pages that load data with JavaScript.',
+ required: false,
+ options: {
+ options: [
+ { label: 'Load Event', value: 'load' },
+ { label: 'DOM Content Loaded', value: 'domcontentloaded' },
+ { label: 'Network Idle (0 connections)', value: 'networkidle0' },
+ { label: 'Network Idle (2 connections)', value: 'networkidle2' },
+ ],
+ },
+ }),
+ navigationTimeout: () =>
+ Property.Number({
+ displayName: 'Page Load Timeout (ms)',
+ description: 'How long to wait for the page to load before failing (Browserless default 30000).',
+ required: false,
+ }),
+ waitForSelector: () =>
+ Property.ShortText({
+ displayName: 'Wait for Selector',
+ description: 'A CSS selector (for example `#content` or `.price`) to wait for before reading the page.',
+ required: false,
+ }),
+ waitForTimeout: () =>
+ Property.Number({
+ displayName: 'Extra Wait (ms)',
+ description: 'Extra time to wait after the page loads, for animations or late content.',
+ required: false,
+ }),
+ bestAttempt: () =>
+ Property.Checkbox({
+ displayName: 'Best Attempt',
+ description: 'Continue with whatever has loaded when a wait or the page load times out, instead of failing.',
+ required: false,
+ defaultValue: false,
+ }),
+ blockAds: () =>
+ Property.Checkbox({
+ displayName: 'Block Ads',
+ description: 'Load the page with an ad blocker, which is usually faster.',
+ required: false,
+ defaultValue: false,
+ }),
+ proxy: ({ description }: { description: string }) =>
+ Property.StaticDropdown({
+ displayName: 'Proxy',
+ description,
+ required: false,
+ options: {
+ options: [
+ { label: 'Datacenter (2 units/MB)', value: 'datacenter' },
+ { label: 'Residential (6 units/MB)', value: 'residential' },
+ ],
+ },
+ }),
+ maxCharacters: () =>
+ Property.Number({
+ displayName: 'Maximum Characters',
+ description: 'Cut long text fields to this many characters (default 100000, 0 = no limit).',
+ required: false,
+ defaultValue: DEFAULT_MAX_CHARACTERS,
+ }),
+ sessionTimeout: () =>
+ Property.Number({
+ displayName: 'Timeout (ms)',
+ description: 'Maximum time the whole request may take on Browserless. Browser time is billed per 30 seconds, so a lower limit caps cost. Keep it under 540000.',
+ required: false,
+ }),
+};
+
+export const browserlessBody = {
+ pageOptions,
+ optionalNumber,
+ nonEmpty,
+ textList,
+ httpUrl,
+ maxCharacters,
+ cap,
+ pageSource,
+};
+
+function pageOptions({
+ waitUntil,
+ navigationTimeout,
+ waitForSelector,
+ waitForTimeout,
+ bestAttempt,
+}: {
+ waitUntil?: string | null;
+ navigationTimeout?: number | null;
+ waitForSelector?: string | null;
+ waitForTimeout?: number | null;
+ bestAttempt?: boolean | null;
+}): Record<string, unknown> {
+ const gotoOptions = {
+ ...(nonEmpty(waitUntil) ? { waitUntil } : {}),
+ ...(optionalNumber({ value: navigationTimeout, label: 'Page Load Timeout', min: 0 }) !== undefined ? { timeout: Number(navigationTimeout) } : {}),
+ };
+ const extraWait = optionalNumber({ value: waitForTimeout, label: 'Extra Wait', min: 0 });
+ return {
+ ...(Object.keys(gotoOptions).length > 0 ? { gotoOptions } : {}),
+ ...(nonEmpty(waitForSelector) ? { waitForSelector: { selector: waitForSelector.trim() } } : {}),
+ ...(extraWait !== undefined ? { waitForTimeout: extraWait } : {}),
+ ...(bestAttempt === true ? { bestAttempt: true } : {}),
+ };
+}
+
+function optionalNumber({ value, label, min, max }: { value: unknown; label: string; min?: number; max?: number }): number | undefined {
+ if (value === undefined || value === null || value === '') {
+ return undefined;
+ }
+ const parsed = typeof value === 'number' ? value : Number(value);
+ if (!Number.isFinite(parsed)) {
+ throw new Error(`${label} must be a number.`);
+ }
+ if (min !== undefined && parsed < min) {
+ throw new Error(`${label} must be at least ${min}.`);
+ }
+ if (max !== undefined && parsed > max) {
+ throw new Error(`${label} must be at most ${max}.`);
+ }
+ return parsed;
+}
+
+function maxCharacters(value: unknown): number {
+ const parsed = optionalNumber({ value, label: 'Maximum Characters', min: 0 });
+ return parsed === undefined ? DEFAULT_MAX_CHARACTERS : Math.floor(parsed);
+}
+
+function cap({ text, max }: { text: string | null; max: number }): { text: string | null; truncated: boolean } {
+ if (text === null || max === 0 || text.length <= max) {
+ return { text, truncated: false };
+ }
+ return { text: text.slice(0, max), truncated: true };
+}
+
+function pageSource({ url, html }: { url: unknown; html: unknown }): { url: string } | { html: string } {
+ const hasUrl = typeof url === 'string' && url.trim() !== '';
+ const hasHtml = typeof html === 'string' && html.trim() !== '';
+ if (hasUrl === hasHtml) {
+ throw new Error('Provide exactly one of URL or HTML.');
+ }
+ return hasUrl ? { url: httpUrl({ value: url, label: 'URL' }) } : { html: String(html) };
+}
+
+function httpUrl({ value, label }: { value: unknown; label: string }): string {
+ const trimmed = typeof value === 'string' ? value.trim() : '';
+ if (!/^https?:\/\/[^\s/]+\S*$/i.test(trimmed)) {
+ throw new Error(`${label} must be a full web address starting with http:// or https://`);
+ }
+ return trimmed;
+}
+
+function nonEmpty(value: string | null | undefined): value is string {
+ return typeof value === 'string' && value.trim() !== '';
+}
+
+function textList(value: unknown): string[] {
+ if (value === undefined || value === null) {
+ return [];
+ }
+ const items = Array.isArray(value) ? value : [value];
+ return items
+ .map((item) => (typeof item === 'string' ? item.trim() : typeof item === 'number' ? String(item) : ''))
+ .filter((item) => item !== '');
+}
+
+const DEFAULT_MAX_CHARACTERS = 100_000;
diff --git a/packages/pieces/community/browserless/src/lib/common/values.ts b/packages/pieces/community/browserless/src/lib/common/values.ts
new file mode 100644
index 00000000000..d6e134df8d3
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/common/values.ts
@@ -0,0 +1,41 @@
+export const browserlessValues = {
+ record,
+ stringOrNull,
+ numberOrNull,
+ booleanOrNull,
+ stringList,
+ writeBase64File,
+};
+
+function record(value: unknown): Record<string, unknown> {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ return {};
+ }
+ return Object.fromEntries(Object.entries(value));
+}
+
+function stringOrNull(value: unknown): string | null {
+ return typeof value === 'string' ? value : null;
+}
+
+function numberOrNull(value: unknown): number | null {
+ return typeof value === 'number' && Number.isFinite(value) ? value : null;
+}
+
+function booleanOrNull(value: unknown): boolean | null {
+ return typeof value === 'boolean' ? value : null;
+}
+
+function stringList(value: unknown): string[] {
+ return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : [];
+}
+
+async function writeBase64File({ files, value, fileName }: { files: FileWriter; value: unknown; fileName: string }): Promise<string | null> {
+ if (typeof value !== 'string' || value === '') {
+ return null;
+ }
+ const base64 = value.replace(/^data:[^;]+;base64,/, '');
+ return files.write({ fileName, data: Buffer.from(base64, 'base64') });
+}
+
+export type FileWriter = { write: (params: { fileName: string; data: Buffer }) => Promise<string> };
diff --git a/packages/pieces/community/browserless/src/lib/output-schemas.ts b/packages/pieces/community/browserless/src/lib/output-schemas.ts
new file mode 100644
index 00000000000..5ed3cc54c7b
--- /dev/null
+++ b/packages/pieces/community/browserless/src/lib/output-schemas.ts
@@ -0,0 +1,440 @@
+import { OutputSchema } from '@activepieces/pieces-framework';
+
+const siteFields: Fields = [
+ { key: 'site_status_code', label: 'Site Status Code', format: 'number', description: 'HTTP status the target site answered with (from X-Response-Code).' },
+ { key: 'site_status_text', label: 'Site Status Text' },
+ { key: 'final_url', label: 'Final URL', format: 'url', description: 'The page address after redirects.' },
+];
+
+const metricFields: Fields = [
+ { key: 'value', label: 'Value' },
+ { key: 'numericValue', label: 'Numeric Value', format: 'number' },
+ { key: 'score', label: 'Score (0-100)', format: 'number' },
+];
+
+const crawlPageFields: Fields = [
+ { key: 'url', label: 'Page URL', format: 'url' },
+ { key: 'title', label: 'Title' },
+ { key: 'description', label: 'Description' },
+ { key: 'status', label: 'Page Status' },
+ { key: 'status_code', label: 'HTTP Status', format: 'number' },
+ { key: 'language', label: 'Language' },
+ { key: 'scraped_at', label: 'Scraped At', format: 'datetime' },
+ { key: 'error', label: 'Error' },
+ { key: 'content_url', label: 'Content Link', format: 'url', description: 'Temporary link to the scraped content JSON; it expires with the crawl.' },
+];
+
+const crawlSummaryFields: Fields = [
+ { key: 'id', label: 'Crawl ID' },
+ { key: 'url', label: 'Start URL', format: 'url' },
+ { key: 'status', label: 'Status' },
+ { key: 'total', label: 'Pages Found', format: 'number' },
+ { key: 'completed', label: 'Pages Completed', format: 'number' },
+ { key: 'created_at', label: 'Created At', format: 'datetime' },
+ { key: 'completed_at', label: 'Completed At', format: 'datetime' },
+];
+
+const captureScreenshot: OutputSchema = {
+ fields: [
+ { key: 'file', label: 'Screenshot File', format: 'url', description: 'Pass this to steps that take a file.' },
+ { key: 'success', label: 'Success', format: 'boolean' },
+ {
+ key: 'metadata',
+ label: 'Details',
+ children: [
+ { key: 'url', label: 'Page URL', format: 'url' },
+ { key: 'type', label: 'Image Type' },
+ { key: 'fullPage', label: 'Full Page', format: 'boolean' },
+ { key: 'fileName', label: 'File Name' },
+ { key: 'contentType', label: 'Content Type' },
+ { key: 'sizeBytes', label: 'Size', format: 'filesize' },
+ { key: 'siteStatusCode', label: 'Site Status Code', format: 'number' },
+ { key: 'finalUrl', label: 'Final URL', format: 'url' },
+ { key: 'timestamp', label: 'Captured At', format: 'datetime' },
+ ],
+ },
+ { key: 'screenshotBase64', label: 'Screenshot (Base64)', description: 'The image as base64 text. Prefer the file for large images.' },
+ ],
+};
+
+const generatePdf: OutputSchema = {
+ fields: [
+ { key: 'file', label: 'PDF File', format: 'url', description: 'Pass this to steps that take a file.' },
+ { key: 'success', label: 'Success', format: 'boolean' },
+ {
+ key: 'metadata',
+ label: 'Details',
+ children: [
+ { key: 'source', label: 'Source (url or html)' },
+ { key: 'url', label: 'Page URL', format: 'url' },
+ { key: 'format', label: 'Paper Format' },
+ { key: 'landscape', label: 'Landscape', format: 'boolean' },
+ { key: 'fileName', label: 'File Name' },
+ { key: 'sizeBytes', label: 'Size', format: 'filesize' },
+ { key: 'siteStatusCode', label: 'Site Status Code', format: 'number' },
+ { key: 'timestamp', label: 'Generated At', format: 'datetime' },
+ ],
+ },
+ { key: 'pdfBase64', label: 'PDF (Base64)', description: 'The PDF as base64 text. Prefer the file for large documents.' },
+ ],
+};
+
+const scrapeUrl: OutputSchema = {
+ fields: [
+ {
+ key: 'elements',
+ label: 'Elements',
+ value: 'data.data',
+ labelKey: 'selector',
+ listItems: [
+ { key: 'selector', label: 'Selector' },
+ {
+ key: 'results',
+ label: 'Matches',
+ labelKey: 'text',
+ listItems: [
+ { key: 'text', label: 'Text' },
+ { key: 'html', label: 'HTML', format: 'html' },
+ {
+ key: 'attributes',
+ label: 'Attributes',
+ labelKey: 'name',
+ listItems: [
+ { key: 'name', label: 'Name' },
+ { key: 'value', label: 'Value' },
+ ],
+ },
+ { key: 'width', label: 'Width', format: 'number' },
+ { key: 'height', label: 'Height', format: 'number' },
+ { key: 'top', label: 'Top', format: 'number' },
+ { key: 'left', label: 'Left', format: 'number' },
+ ],
+ },
+ ],
+ },
+ {
+ key: 'metadata',
+ label: 'Details',
+ children: [
+ { key: 'url', label: 'Page URL', format: 'url' },
+ { key: 'elementsCount', label: 'Selectors Requested', format: 'number' },
+ ...siteFields,
+ { key: 'timestamp', label: 'Scraped At', format: 'datetime' },
+ ],
+ },
+ { key: 'success', label: 'Success', format: 'boolean' },
+ ],
+};
+
+const runBqlQuery: OutputSchema = {
+ fields: [
+ { key: 'data', label: 'Query Data', description: 'The data returned by the BQL query, shaped by your query fields.' },
+ {
+ key: 'errors',
+ label: 'Errors',
+ labelKey: 'message',
+ listItems: [{ key: 'message', label: 'Message' }],
+ },
+ {
+ key: 'metadata',
+ label: 'Details',
+ children: [
+ { key: 'executionTime', label: 'Execution Time' },
+ { key: 'stealth', label: 'Stealth', format: 'boolean' },
+ { key: 'timestamp', label: 'Ran At', format: 'datetime' },
+ ],
+ },
+ { key: 'success', label: 'Success', format: 'boolean' },
+ ],
+};
+
+const getWebsitePerformance: OutputSchema = {
+ fields: [
+ {
+ key: 'scores',
+ label: 'Scores (0-100)',
+ value: 'summary.scores',
+ children: [
+ { key: 'performance', label: 'Performance', format: 'number' },
+ { key: 'accessibility', label: 'Accessibility', format: 'number' },
+ { key: 'bestPractices', label: 'Best Practices', format: 'number' },
+ { key: 'seo', label: 'SEO', format: 'number' },
+ ],
+ },
+ {
+ key: 'metrics',
+ label: 'Metrics',
+ value: 'summary.metrics',
+ children: [
+ { key: 'firstContentfulPaint', label: 'First Contentful Paint', children: metricFields },
+ { key: 'largestContentfulPaint', label: 'Largest Contentful Paint', children: metricFields },
+ { key: 'speedIndex', label: 'Speed Index', children: metricFields },
+ { key: 'timeToInteractive', label: 'Time to Interactive', children: metricFields },
+ { key: 'totalBlockingTime', label: 'Total Blocking Time', children: metricFields },
+ { key: 'cumulativeLayoutShift', label: 'Cumulative Layout Shift', children: metricFields },
+ ],
+ },
+ {
+ key: 'opportunities',
+ label: 'Opportunities',
+ value: 'summary.opportunities',
+ labelKey: 'title',
+ listItems: [
+ { key: 'type', label: 'Type' },
+ { key: 'title', label: 'Title' },
+ { key: 'potentialSavings', label: 'Potential Savings' },
+ ],
+ },
+ { key: 'url', label: 'URL', value: 'summary.url', format: 'url' },
+ { key: 'finalUrl', label: 'Final URL', value: 'summary.finalUrl', format: 'url' },
+ { key: 'formFactor', label: 'Device', value: 'summary.formFactor' },
+ { key: 'runtimeError', label: 'Lighthouse Error', value: 'summary.runtimeError' },
+ { key: 'lighthouseVersion', label: 'Lighthouse Version', value: 'metadata.lighthouseVersion' },
+ { key: 'timestamp', label: 'Audited At', value: 'summary.timestamp', format: 'datetime' },
+ { key: 'success', label: 'Success', format: 'boolean' },
+ ],
+};
+
+const getPageContent: OutputSchema = {
+ fields: [
+ { key: 'html', label: 'HTML', format: 'html' },
+ { key: 'html_length', label: 'Full HTML Length (characters)', format: 'number' },
+ { key: 'truncated', label: 'Truncated', format: 'boolean', description: 'True when the HTML was cut to Maximum Characters.' },
+ { key: 'url', label: 'Requested URL', format: 'url' },
+ ...siteFields,
+ ],
+};
+
+const smartScrape: OutputSchema = {
+ fields: [
+ { key: 'markdown', label: 'Markdown' },
+ { key: 'raw_text', label: 'Plain Text' },
+ { key: 'html', label: 'HTML', format: 'html' },
+ { key: 'content', label: 'Other Content', description: 'Parsed JSON or extracted PDF text when the URL is not an HTML page. JSON longer than Maximum Characters comes back as cut JSON text with truncated=true.' },
+ { key: 'title', label: 'Page Title' },
+ { key: 'description', label: 'Page Description' },
+ { key: 'language', label: 'Language' },
+ { key: 'source_url', label: 'Source URL', format: 'url' },
+ { key: 'status_code', label: 'HTTP Status', format: 'number' },
+ { key: 'content_type', label: 'Content Type' },
+ { key: 'strategy', label: 'Strategy Used', description: 'http-fetch, http-proxy, browser or browser-captcha.' },
+ { key: 'attempted_strategies', label: 'Strategies Tried' },
+ { key: 'links', label: 'Links' },
+ { key: 'links_count', label: 'Links Count', format: 'number' },
+ { key: 'truncated', label: 'Truncated', format: 'boolean', description: 'True when a text field was cut to Maximum Characters.' },
+ { key: 'screenshot_file', label: 'Screenshot File', format: 'url' },
+ { key: 'pdf_file', label: 'PDF File', format: 'url' },
+ { key: 'url', label: 'Requested URL', format: 'url' },
+ ],
+};
+
+const searchWeb: OutputSchema = {
+ fields: [
+ {
+ key: 'results',
+ label: 'Results',
+ labelKey: 'title',
+ listItems: [
+ { key: 'title', label: 'Title' },
+ { key: 'url', label: 'URL', format: 'url' },
+ { key: 'description', label: 'Snippet' },
+ { key: 'source', label: 'Source (web, news, images)' },
+ { key: 'position', label: 'Position', format: 'number' },
+ { key: 'date', label: 'Date' },
+ { key: 'image_url', label: 'Image', format: 'image' },
+ { key: 'markdown', label: 'Page Markdown' },
+ { key: 'html', label: 'Page HTML', format: 'html' },
+ { key: 'links', label: 'Page Links' },
+ { key: 'page_status_code', label: 'Page HTTP Status', format: 'number' },
+ { key: 'page_error', label: 'Page Error' },
+ { key: 'truncated', label: 'Page Content Truncated', format: 'boolean' },
+ ],
+ },
+ { key: 'result_count', label: 'Results Returned', format: 'number' },
+ { key: 'total_results', label: 'Total Results', format: 'number' },
+ { key: 'no_results', label: 'No Results', format: 'boolean' },
+ { key: 'query', label: 'Query' },
+ ],
+};
+
+const mapWebsite: OutputSchema = {
+ fields: [
+ {
+ key: 'links',
+ label: 'URLs',
+ labelKey: 'url',
+ listItems: [
+ { key: 'url', label: 'URL', format: 'url' },
+ { key: 'title', label: 'Title' },
+ { key: 'description', label: 'Description' },
+ ],
+ },
+ { key: 'count', label: 'URL Count', format: 'number' },
+ { key: 'url', label: 'Website', format: 'url' },
+ ],
+};
+
+const startCrawl: OutputSchema = {
+ fields: [
+ { key: 'crawl_id', label: 'Crawl ID', description: 'Pass this to Get Crawl Results or Cancel Crawl.' },
+ { key: 'status', label: 'Status' },
+ { key: 'url', label: 'Start URL', format: 'url' },
+ ],
+};
+
+const getCrawl: OutputSchema = {
+ fields: [
+ { key: 'status', label: 'Status', description: 'in-progress, completed, failed or cancelled.' },
+ { key: 'total', label: 'Pages Found', format: 'number' },
+ { key: 'completed', label: 'Pages Completed', format: 'number' },
+ { key: 'failed', label: 'Pages Failed', format: 'number' },
+ {
+ key: 'pages',
+ label: 'Pages',
+ labelKey: 'url',
+ listItems: crawlPageFields,
+ },
+ { key: 'page_count', label: 'Pages in This Batch', format: 'number' },
+ { key: 'has_more', label: 'Has More', format: 'boolean' },
+ { key: 'next_skip', label: 'Next Skip', format: 'number', description: 'Pass as Skip to read the next batch.' },
+ { key: 'skip', label: 'Skipped Pages', format: 'number' },
+ { key: 'expires_at', label: 'Results Expire At', format: 'datetime' },
+ { key: 'crawl_id', label: 'Crawl ID' },
+ ],
+};
+
+const listCrawls: OutputSchema = {
+ fields: [
+ {
+ key: 'crawls',
+ label: 'Crawls',
+ labelKey: 'url',
+ listItems: crawlSummaryFields,
+ },
+ { key: 'count', label: 'Count', format: 'number' },
+ { key: 'has_more', label: 'Has More', format: 'boolean' },
+ { key: 'next_cursor', label: 'Next Cursor', description: 'Pass as Cursor to read the next batch.' },
+ ],
+};
+
+const cancelCrawl: OutputSchema = {
+ fields: [
+ { key: 'crawl_id', label: 'Crawl ID' },
+ { key: 'cancelled', label: 'Cancelled', format: 'boolean', description: 'False when the crawl had already finished.' },
+ { key: 'status', label: 'Status' },
+ { key: 'message', label: 'Message' },
+ ],
+};
+
+const unblockPage: OutputSchema = {
+ fields: [
+ { key: 'html', label: 'HTML', format: 'html' },
+ { key: 'html_length', label: 'Full HTML Length (characters)', format: 'number' },
+ { key: 'truncated', label: 'Truncated', format: 'boolean', description: 'True when the HTML was cut to Maximum Characters.' },
+ { key: 'solved', label: 'Challenge Solved', format: 'boolean' },
+ {
+ key: 'cookies',
+ label: 'Cookies',
+ labelKey: 'name',
+ listItems: [
+ { key: 'name', label: 'Name' },
+ { key: 'value', label: 'Value' },
+ { key: 'domain', label: 'Domain' },
+ { key: 'path', label: 'Path' },
+ { key: 'expires', label: 'Expires (Unix seconds)', format: 'number' },
+ { key: 'http_only', label: 'HTTP Only', format: 'boolean' },
+ { key: 'secure', label: 'Secure', format: 'boolean' },
+ { key: 'same_site', label: 'SameSite' },
+ ],
+ },
+ { key: 'cookie_count', label: 'Cookie Count', format: 'number' },
+ { key: 'cookie_header', label: 'Cookie Header', description: 'All cookies as one name=value; string for a Cookie header.' },
+ { key: 'screenshot_file', label: 'Screenshot File', format: 'url' },
+ { key: 'url', label: 'URL', format: 'url' },
+ ],
+};
+
+const auditPagePerformance: OutputSchema = {
+ fields: [
+ {
+ key: 'scores',
+ label: 'Scores (0-100)',
+ children: [
+ { key: 'performance', label: 'Performance', format: 'number' },
+ { key: 'accessibility', label: 'Accessibility', format: 'number' },
+ { key: 'bestPractices', label: 'Best Practices', format: 'number' },
+ { key: 'seo', label: 'SEO', format: 'number' },
+ ],
+ },
+ {
+ key: 'metrics',
+ label: 'Metrics',
+ children: [
+ { key: 'firstContentfulPaint', label: 'First Contentful Paint', children: metricFields },
+ { key: 'largestContentfulPaint', label: 'Largest Contentful Paint', children: metricFields },
+ { key: 'speedIndex', label: 'Speed Index', children: metricFields },
+ { key: 'timeToInteractive', label: 'Time to Interactive', children: metricFields },
+ { key: 'totalBlockingTime', label: 'Total Blocking Time', children: metricFields },
+ { key: 'cumulativeLayoutShift', label: 'Cumulative Layout Shift', children: metricFields },
+ ],
+ },
+ {
+ key: 'opportunities',
+ label: 'Opportunities',
+ labelKey: 'title',
+ listItems: [
+ { key: 'type', label: 'Type' },
+ { key: 'title', label: 'Title' },
+ { key: 'potentialSavings', label: 'Potential Savings' },
+ ],
+ },
+ { key: 'url', label: 'URL', format: 'url' },
+ { key: 'finalUrl', label: 'Final URL', format: 'url' },
+ { key: 'formFactor', label: 'Device' },
+ { key: 'runtimeError', label: 'Lighthouse Error' },
+ { key: 'lighthouseVersion', label: 'Lighthouse Version' },
+ { key: 'timestamp', label: 'Audited At', format: 'datetime' },
+ ],
+};
+
+const fileOnly: OutputSchema = {
+ fields: [
+ { key: 'file', label: 'File', format: 'url', description: 'Pass this to steps that take a file.' },
+ { key: 'file_name', label: 'File Name' },
+ { key: 'size_bytes', label: 'Size', format: 'filesize' },
+ { key: 'mime_type', label: 'MIME Type' },
+ { key: 'source', label: 'Source (URL or html)' },
+ { key: 'site_status_code', label: 'Site Status Code', format: 'number' },
+ ],
+};
+
+const runFunction: OutputSchema = {
+ fields: [
+ { key: 'result', label: 'Result', description: 'What your function returned (JSON or text).' },
+ { key: 'file', label: 'Result File', format: 'url', description: 'Set when the function returned binary data such as a PDF or image.' },
+ { key: 'content_type', label: 'Content Type' },
+ { key: 'size_bytes', label: 'Size', format: 'filesize' },
+ ],
+};
+
+export const browserlessOutputSchemas = {
+ captureScreenshot,
+ generatePdf,
+ scrapeUrl,
+ runBqlQuery,
+ getWebsitePerformance,
+ getPageContent,
+ smartScrape,
+ searchWeb,
+ mapWebsite,
+ startCrawl,
+ getCrawl,
+ listCrawls,
+ cancelCrawl,
+ unblockPage,
+ runFunction,
+ fileOnly,
+ auditPagePerformance,
+};
+
+type Fields = OutputSchema['fields'];
diff --git a/packages/pieces/community/browserless/test/ai-actions.test.ts b/packages/pieces/community/browserless/test/ai-actions.test.ts
new file mode 100644
index 00000000000..cc301616c48
--- /dev/null
+++ b/packages/pieces/community/browserless/test/ai-actions.test.ts
@@ -0,0 +1,84 @@
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { auditPagePerformance } from '../src/lib/actions/ai/audit-page-performance';
+import { createPdf } from '../src/lib/actions/ai/create-pdf';
+import { takeScreenshot } from '../src/lib/actions/ai/take-screenshot';
+import { browserlessOutputSchemas } from '../src/lib/output-schemas';
+import { missingSchemaPaths, replies, runAction, stubFetch } from './helpers';
+
+const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+describe('take_screenshot (ai)', () => {
+ test('url source: stores the image and returns only file metadata', async () => {
+ const seen = stubFetch(replies([{ binary: PNG, headers: { 'x-response-code': '200' } }]));
+ const { output, written } = await runAction({
+ action: takeScreenshot,
+ propsValue: { url: 'https://example.com', imageType: 'jpeg', quality: 70, fullPage: true, width: 1024, height: 768, selector: '#main', waitForTimeout: 200, timeout: 30000 },
+ });
+ expect(seen[0].path).toBe('/screenshot');
+ expect(seen[0].query.get('timeout')).toBe('30000');
+ expect(seen[0].body).toEqual({
+ url: 'https://example.com',
+ options: { type: 'jpeg', fullPage: true, quality: 70 },
+ viewport: { width: 1024, height: 768 },
+ selector: '#main',
+ waitForTimeout: 200,
+ });
+ expect(written[0]).toMatchObject({ fileName: 'screenshot.jpg' });
+ expect(output).toEqual({ file: 'https://files.example/screenshot.jpg', file_name: 'screenshot.jpg', size_bytes: 4, mime_type: 'image/jpeg', source: 'https://example.com', site_status_code: 200 });
+ expect(JSON.stringify(output)).not.toMatch(/base64/i);
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.fileOnly.fields })).toEqual([]);
+ });
+ test('html source and exactly-one validation', async () => {
+ const seen = stubFetch(replies([{ binary: PNG }]));
+ const { output } = await runAction({ action: takeScreenshot, propsValue: { html: '<h1>Hi</h1>' } });
+ expect(seen[0].body).toEqual({ html: '<h1>Hi</h1>', options: { type: 'png', fullPage: false } });
+ expect(output).toMatchObject({ source: 'html', mime_type: 'image/png' });
+ await expect(runAction({ action: takeScreenshot, propsValue: {} })).rejects.toThrow(/exactly one of URL or HTML/);
+ await expect(runAction({ action: takeScreenshot, propsValue: { url: 'https://a.com', html: '<p>' } })).rejects.toThrow(/exactly one of URL or HTML/);
+ await expect(runAction({ action: takeScreenshot, propsValue: { url: 'https://a.com', width: 100 } })).rejects.toThrow(/Viewport Width and Viewport Height/);
+ });
+});
+
+describe('create_pdf (ai)', () => {
+ test('renders html with one margin for all sides and returns only the file', async () => {
+ const seen = stubFetch(replies([{ binary: Buffer.from('%PDF-1.7') }]));
+ const { output, written } = await runAction({ action: createPdf, propsValue: { html: '<h1>Invoice</h1>', format: 'Letter', landscape: true, margin: '10mm' } });
+ expect(seen[0].path).toBe('/pdf');
+ expect(seen[0].body).toEqual({
+ html: '<h1>Invoice</h1>',
+ options: { format: 'Letter', landscape: true, printBackground: true, margin: { top: '10mm', right: '10mm', bottom: '10mm', left: '10mm' } },
+ });
+ expect(written[0].fileName).toBe('document.pdf');
+ expect(output).toEqual({ file: 'https://files.example/document.pdf', file_name: 'document.pdf', size_bytes: 8, mime_type: 'application/pdf', source: 'html', site_status_code: null });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.fileOnly.fields })).toEqual([]);
+ });
+});
+
+describe('audit_page_performance (ai)', () => {
+ test('returns the summary only, never the raw report', async () => {
+ const lhr = {
+ lighthouseVersion: '13.5.0',
+ finalDisplayedUrl: 'https://example.com/',
+ categories: { performance: { score: 1 }, seo: { score: 0.8 } },
+ audits: { 'first-contentful-paint': { displayValue: '0.2 s', numericValue: 245, score: 1 }, 'unused-css-rules': { displayValue: 'Est savings 5 KiB', details: { items: [{}] } } },
+ fullPageScreenshot: { data: 'x'.repeat(10_000) },
+ };
+ const seen = stubFetch(replies([{ body: { type: 'json', data: lhr } }]));
+ const { output } = await runAction({ action: auditPagePerformance, propsValue: { url: 'https://example.com', device: 'mobile', categories: ['performance', 'seo'] } });
+ expect(seen[0].path).toBe('/performance');
+ expect(seen[0].body).toMatchObject({ url: 'https://example.com', config: { settings: { formFactor: 'mobile', onlyCategories: ['performance', 'seo'], maxWaitForLoad: 45000 } } });
+ expect(output).toMatchObject({
+ scores: { performance: 100, seo: 80, accessibility: null, bestPractices: null },
+ metrics: { firstContentfulPaint: { value: '0.2 s', score: 100 } },
+ opportunities: [{ type: 'unused-css' }],
+ finalUrl: 'https://example.com/',
+ lighthouseVersion: '13.5.0',
+ });
+ expect(JSON.stringify(output).length).toBeLessThan(3000);
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.auditPagePerformance.fields })).toEqual([]);
+ });
+});
diff --git a/packages/pieces/community/browserless/test/auth.test.ts b/packages/pieces/community/browserless/test/auth.test.ts
new file mode 100644
index 00000000000..10ad39cce30
--- /dev/null
+++ b/packages/pieces/community/browserless/test/auth.test.ts
@@ -0,0 +1,47 @@
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { browserlessAuth } from '../src/lib/common/auth';
+import { TOKEN, replies, stubFetch } from './helpers';
+
+const props = { apiToken: TOKEN, region: 'https://production-lon.browserless.io', customBaseUrl: '' };
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+async function validate(auth: typeof props) {
+ if (!browserlessAuth.validate) {
+ throw new Error('validate missing');
+ }
+ return browserlessAuth.validate({ auth, server: { apiUrl: 'http://localhost:3000/api/', publicUrl: 'http://localhost:4200/' } });
+}
+
+describe('auth.validate', () => {
+ test('GET /meta on the selected region with the token', async () => {
+ const seen = stubFetch(replies([{ body: { version: '2.40.0' } }]));
+ expect(await validate(props)).toEqual({ valid: true });
+ expect(seen[0].method).toBe('GET');
+ expect(seen[0].url.startsWith('https://production-lon.browserless.io/meta?')).toBe(true);
+ expect(seen[0].query.get('token')).toBe(TOKEN);
+ });
+ test.each([401, 403])('%s is an invalid token', async (status) => {
+ stubFetch(replies([{ status, text: 'Invalid API key' }]));
+ const result = await validate(props);
+ expect(result.valid).toBe(false);
+ expect(result).toMatchObject({ error: expect.stringMatching(/Invalid API token/) });
+ });
+ test('a 404 means the token was accepted', async () => {
+ stubFetch(replies([{ status: 404, text: 'Not Found' }]));
+ expect(await validate(props)).toEqual({ valid: true });
+ });
+ test('network failure is reported, not treated as valid', async () => {
+ stubFetch(replies([{ throws: new Error('getaddrinfo ENOTFOUND') }]));
+ const result = await validate(props);
+ expect(result.valid).toBe(false);
+ });
+ test('custom endpoint without URL fails before any request', async () => {
+ const seen = stubFetch(replies([{ body: {} }]));
+ const result = await validate({ ...props, region: 'custom', customBaseUrl: '' });
+ expect(result).toMatchObject({ valid: false, error: expect.stringMatching(/Custom Base URL is required/) });
+ expect(seen).toHaveLength(0);
+ });
+});
diff --git a/packages/pieces/community/browserless/test/client.test.ts b/packages/pieces/community/browserless/test/client.test.ts
new file mode 100644
index 00000000000..a1e52082f1f
--- /dev/null
+++ b/packages/pieces/community/browserless/test/client.test.ts
@@ -0,0 +1,124 @@
+import { HttpMethod } from '@activepieces/pieces-common';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { BrowserlessApiError, browserlessApi } from '../src/lib/common/client';
+import { TOKEN, replies, stubFetch } from './helpers';
+
+const sfo = { apiToken: TOKEN, region: 'https://production-sfo.browserless.io' };
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+describe('base URL', () => {
+ test('uses the selected shared region', () => {
+ expect(browserlessApi.resolveBaseUrl(sfo)).toBe('https://production-sfo.browserless.io');
+ });
+ test('rejects an unknown region value', () => {
+ expect(() => browserlessApi.resolveBaseUrl({ apiToken: TOKEN, region: 'https://evil.example' })).toThrow(/Unknown Browserless region/);
+ });
+ test('normalizes a custom endpoint and keeps a path prefix', () => {
+ expect(browserlessApi.resolveBaseUrl({ apiToken: TOKEN, region: 'custom', customBaseUrl: ' https://chrome.browserless.io/ ' })).toBe('https://chrome.browserless.io');
+ expect(browserlessApi.resolveBaseUrl({ apiToken: TOKEN, region: 'custom', customBaseUrl: 'http://10.0.0.5:3000/bl/' })).toBe('http://10.0.0.5:3000/bl');
+ });
+ test.each([
+ ['', /Custom Base URL is required/],
+ ['chrome.browserless.io', /not a valid URL/],
+ ['ftp://chrome.browserless.io', /https:\/\/ or http:\/\//],
+ ['https://user:[email protected]', /username or password/],
+ ['https://chrome.browserless.io?token=abc', /query string/],
+ ])('rejects custom base URL %s', (customBaseUrl, message) => {
+ expect(() => browserlessApi.resolveBaseUrl({ apiToken: TOKEN, region: 'custom', customBaseUrl })).toThrow(message);
+ });
+});
+
+describe('same-target check for the custom API call', () => {
+ test.each([
+ ['https://production-sfo.browserless.io/meta', true],
+ ['https://production-sfo.browserless.io.evil.io/meta', false],
+ ['https://evil.io/production-sfo.browserless.io', false],
+ ['http://production-sfo.browserless.io/meta', false],
+ ['https://[email protected]/meta', false],
+ ['//evil.io/meta', false],
+ ])('%s -> %s', (url, expected) => {
+ expect(browserlessApi.isSameTarget({ baseUrl: 'https://production-sfo.browserless.io', url })).toBe(expected);
+ });
+ test('respects a custom base path prefix and refuses traversal', () => {
+ const baseUrl = 'https://bl.example.com/api';
+ expect(browserlessApi.isSameTarget({ baseUrl, url: 'https://bl.example.com/api/meta' })).toBe(true);
+ expect(browserlessApi.isSameTarget({ baseUrl, url: 'https://bl.example.com/apix/meta' })).toBe(false);
+ expect(browserlessApi.isSameTarget({ baseUrl, url: 'https://bl.example.com/api/%2e%2e/admin' })).toBe(false);
+ });
+});
+
+describe('request', () => {
+ test('sends the token as a query param to the region host, JSON body, no redirects', async () => {
+ const seen = stubFetch(replies([{ body: { ok: true } }]));
+ const response = await browserlessApi.request({ auth: sfo, method: HttpMethod.POST, path: '/content', body: { url: 'https://example.com' }, query: { blockAds: true, empty: '', none: undefined }, operation: 'Test' });
+ expect(response.body).toEqual({ ok: true });
+ expect(seen[0].origin).toBe('https://production-sfo.browserless.io');
+ expect(seen[0].path).toBe('/content');
+ expect(seen[0].query.get('token')).toBe(TOKEN);
+ expect(seen[0].query.get('blockAds')).toBe('true');
+ expect(seen[0].query.has('empty')).toBe(false);
+ expect(seen[0].query.has('none')).toBe(false);
+ expect(seen[0].headers.get('content-type')).toBe('application/json');
+ expect(seen[0].redirect).toBe('manual');
+ expect(seen[0].body).toEqual({ url: 'https://example.com' });
+ });
+ test('trims the token and refuses an empty one before any request', async () => {
+ const seen = stubFetch(replies([{ body: {} }]));
+ await browserlessApi.request({ auth: { ...sfo, apiToken: ` ${TOKEN} ` }, method: HttpMethod.GET, path: '/meta', operation: 'Test' });
+ expect(seen[0].query.get('token')).toBe(TOKEN);
+ await expect(browserlessApi.request({ auth: { ...sfo, apiToken: ' ' }, method: HttpMethod.GET, path: '/meta', operation: 'Test' })).rejects.toThrow(/token is empty/);
+ expect(seen).toHaveLength(1);
+ });
+ test('refuses absolute or protocol-relative paths', async () => {
+ await expect(browserlessApi.request({ auth: sfo, method: HttpMethod.GET, path: '//evil.io/x', operation: 'Test' })).rejects.toThrow(/Invalid Browserless path/);
+ await expect(browserlessApi.request({ auth: sfo, method: HttpMethod.GET, path: 'https://evil.io', operation: 'Test' })).rejects.toThrow(/Invalid Browserless path/);
+ });
+ test('401 becomes a clear error with responseBody and no body field', async () => {
+ stubFetch(replies([{ status: 401, text: 'Invalid API key. Please check your API key and try again.' }]));
+ const error = await browserlessApi.request({ auth: sfo, method: HttpMethod.GET, path: '/meta', operation: 'Check' }).catch((e: unknown) => e);
+ expect(error).toBeInstanceOf(BrowserlessApiError);
+ if (!(error instanceof BrowserlessApiError)) {
+ return;
+ }
+ expect(error.status).toBe(401);
+ expect(error.message).toMatch(/^Check failed: Browserless refused the API token \(401\).*Invalid API key/);
+ expect(error.responseBody).toContain('Invalid API key');
+ expect('body' in error).toBe(false);
+ });
+ test.each([
+ [400, /rejected the request \(400\)/],
+ [408, /did not finish within the timeout \(408\)/],
+ [429, /concurrency limit/],
+ [503, /internal error \(503\)/],
+ ])('status %s is explained', async (status, message) => {
+ stubFetch(replies([{ status, body: { message: 'vendor says no' } }]));
+ await expect(browserlessApi.request({ auth: sfo, method: HttpMethod.POST, path: '/pdf', body: {}, operation: 'PDF' })).rejects.toThrow(message);
+ });
+ test.each([301, 302, 307, 308])('status %s is an error, not a result', async (status) => {
+ stubFetch(replies([{ status, text: 'moved' }]));
+ const error = await browserlessApi.request({ auth: sfo, method: HttpMethod.POST, path: '/pdf', body: {}, responseType: 'arraybuffer', operation: 'PDF' }).catch((e: unknown) => e);
+ expect(error).toBeInstanceOf(BrowserlessApiError);
+ if (!(error instanceof BrowserlessApiError)) {
+ return;
+ }
+ expect(error.status).toBe(status);
+ expect(error.message).toMatch(new RegExp(`^PDF failed: Browserless answered with status ${status}.*Redirects are not followed`));
+ });
+ test('clamps the client deadline', () => {
+ expect(browserlessApi.clampTimeout(undefined)).toBe(300_000);
+ expect(browserlessApi.clampTimeout(10_000)).toBe(10_000);
+ expect(browserlessApi.clampTimeout(9_999_999)).toBe(540_000);
+ expect(browserlessApi.clampTimeout(-1)).toBe(300_000);
+ });
+ test('reads site status headers', () => {
+ expect(browserlessApi.siteResponse({ 'x-response-code': '404', 'x-response-status': 'Not Found', 'x-response-url': 'https://example.com/x' })).toEqual({
+ site_status_code: 404,
+ site_status_text: 'Not Found',
+ final_url: 'https://example.com/x',
+ });
+ expect(browserlessApi.siteResponse({})).toEqual({ site_status_code: null, site_status_text: null, final_url: null });
+ });
+});
diff --git a/packages/pieces/community/browserless/test/existing-actions.test.ts b/packages/pieces/community/browserless/test/existing-actions.test.ts
new file mode 100644
index 00000000000..4a1ece8292f
--- /dev/null
+++ b/packages/pieces/community/browserless/test/existing-actions.test.ts
@@ -0,0 +1,261 @@
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { captureScreenshot } from '../src/lib/actions/capture-screenshot';
+import { generatePdf } from '../src/lib/actions/generate-pdf';
+import { getWebsitePerformance } from '../src/lib/actions/get-website-performance';
+import { runBqlQuery } from '../src/lib/actions/run-bql-query';
+import { scrapeUrl } from '../src/lib/actions/scrape-url';
+import { browserlessOutputSchemas } from '../src/lib/output-schemas';
+import { TOKEN, actionContext, missingSchemaPaths, replies, runAction, stubFetch } from './helpers';
+
+const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0xff]);
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+describe('capture_screenshot', () => {
+ test('posts puppeteer options and stores the binary file', async () => {
+ const seen = stubFetch(replies([{ binary: PNG, headers: { 'content-type': 'image/png', 'x-response-code': '200', 'x-response-url': 'https://example.com/' } }]));
+ const { output, written } = await runAction({
+ action: captureScreenshot,
+ propsValue: { url: 'https://example.com', imageType: 'png', fullPage: true, quality: 80, width: 1280, height: 720, waitForSelector: ' h1 ', delay: 500 },
+ });
+ expect(seen[0].path).toBe('/screenshot');
+ expect(seen[0].query.get('token')).toBe(TOKEN);
+ expect(seen[0].body).toEqual({
+ url: 'https://example.com',
+ options: { type: 'png', fullPage: true },
+ viewport: { width: 1280, height: 720 },
+ waitForSelector: { selector: 'h1' },
+ waitForTimeout: 500,
+ });
+ expect(written[0].fileName).toBe('screenshot.png');
+ expect(written[0].data.equals(PNG)).toBe(true);
+ expect(output).toMatchObject({ success: true, file: 'https://files.example/screenshot.png', screenshotBase64: PNG.toString('base64') });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.captureScreenshot.fields })).toEqual([]);
+ });
+ test('jpeg/webp send quality and a full clip', async () => {
+ const seen = stubFetch(replies([{ binary: PNG }]));
+ await runAction({ action: captureScreenshot, propsValue: { url: 'https://example.com', imageType: 'webp', quality: 0, clipX: 0, clipY: 0, clipWidth: 100, clipHeight: 50 } });
+ expect(seen[0].body).toMatchObject({ options: { type: 'webp', quality: 0, clip: { x: 0, y: 0, width: 100, height: 50 } } });
+ });
+ test('partial clip or half a viewport fails before the request', async () => {
+ const seen = stubFetch(replies([{ binary: PNG }]));
+ await expect(runAction({ action: captureScreenshot, propsValue: { url: 'https://example.com', clipX: 0, clipY: 0 } })).rejects.toThrow(/all four/);
+ await expect(runAction({ action: captureScreenshot, propsValue: { url: 'https://example.com', width: 800 } })).rejects.toThrow(/Viewport Width and Viewport Height/);
+ expect(seen).toHaveLength(0);
+ });
+});
+
+describe('generate_pdf', () => {
+ test('wait options go top-level, userAgent is an object, options stay puppeteer-only', async () => {
+ const seen = stubFetch(replies([{ binary: Buffer.from('%PDF-1.7') }]));
+ const { output, written } = await runAction({
+ action: generatePdf,
+ propsValue: {
+ url: 'https://example.com',
+ format: 'Letter',
+ printBackground: true,
+ marginTop: '10mm',
+ scale: 1.2,
+ timeout: 20000,
+ waitForSelector: '#ready',
+ waitForSelectorTimeout: 5000,
+ waitForSelectorVisible: true,
+ waitForSelectorHidden: false,
+ waitForFunction: '() => window.done',
+ waitForFunctionPolling: '100',
+ userAgent: 'MyBot/1.0',
+ waitForTimeout: 250,
+ bestAttempt: true,
+ },
+ });
+ expect(seen[0].path).toBe('/pdf');
+ expect(seen[0].body).toEqual({
+ url: 'https://example.com',
+ options: { format: 'Letter', landscape: false, printBackground: true, displayHeaderFooter: false, margin: { top: '10mm' }, scale: 1.2 },
+ gotoOptions: { timeout: 20000 },
+ waitForSelector: { selector: '#ready', timeout: 5000, visible: true },
+ waitForFunction: { fn: '() => window.done', polling: 100 },
+ waitForTimeout: 250,
+ userAgent: { userAgent: 'MyBot/1.0' },
+ bestAttempt: true,
+ });
+ expect(written[0].fileName).toBe('document.pdf');
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.generatePdf.fields })).toEqual([]);
+ });
+ test('html source and exclusivity', async () => {
+ const seen = stubFetch(replies([{ binary: Buffer.from('%PDF') }]));
+ await runAction({ action: generatePdf, propsValue: { html: '<h1>Hi</h1>' } });
+ expect(seen[0].body).toMatchObject({ html: '<h1>Hi</h1>' });
+ await expect(runAction({ action: generatePdf, propsValue: {} })).rejects.toThrow(/Either URL or HTML/);
+ await expect(runAction({ action: generatePdf, propsValue: { url: 'https://a.com', html: '<p>' } })).rejects.toThrow(/Cannot provide both/);
+ await expect(runAction({ action: generatePdf, propsValue: { url: 'https://a.com', scale: 3 } })).rejects.toThrow(/Scale must be at most 2/);
+ });
+ test('a redirect from a custom endpoint fails instead of saving the redirect body as a PDF', async () => {
+ stubFetch(replies([{ status: 302, text: '<a href="/login">Found</a>', headers: { location: 'https://proxy.example/login' } }]));
+ const { context, written } = actionContext({ propsValue: { url: 'https://example.com' }, authProps: { region: 'custom', customBaseUrl: 'https://proxy.example' } });
+ await expect(generatePdf.run(context)).rejects.toThrow(/Generate PDF failed: Browserless answered with status 302.*Redirects are not followed/);
+ expect(written).toEqual([]);
+ });
+});
+
+describe('scrape_url', () => {
+ const scrapeResponse = {
+ data: [{ selector: 'h1', results: [{ text: 'Example Domain', html: 'Example Domain', attributes: [{ name: 'class', value: 'title' }], width: 600, height: 38, top: 133, left: 660 }] }],
+ debug: { console: [], cookies: [], html: null, network: { inbound: [], outbound: [] }, screenshot: null },
+ };
+ test('plain selectors, cookies get the page URL, userAgent object', async () => {
+ const seen = stubFetch(replies([{ body: scrapeResponse, headers: { 'x-response-code': '200' } }]));
+ const { output } = await runAction({
+ action: scrapeUrl,
+ propsValue: {
+ url: 'https://example.com',
+ elements: [{ selector: 'h1' }, { selector: '{"selector":"p"}', timeout: 1000 }],
+ cookies: [{ name: 'sid', value: 'x' }, { name: 'a', value: 'b', domain: '.example.com' }, { name: '', value: 'skip' }],
+ userAgent: 'Bot',
+ timeout: 30000,
+ waitUntil: 'networkidle2',
+ viewportWidth: 1920,
+ viewportHeight: 1080,
+ },
+ });
+ expect(seen[0].path).toBe('/scrape');
+ expect(seen[0].body).toEqual({
+ url: 'https://example.com',
+ elements: [{ selector: 'h1' }, { selector: 'p', timeout: 1000 }],
+ gotoOptions: { timeout: 30000, waitUntil: 'networkidle2' },
+ userAgent: { userAgent: 'Bot' },
+ viewport: { width: 1920, height: 1080 },
+ cookies: [
+ { name: 'sid', value: 'x', url: 'https://example.com' },
+ { name: 'a', value: 'b', domain: '.example.com' },
+ ],
+ });
+ expect(output).toMatchObject({ success: true, data: scrapeResponse, metadata: { elementsCount: 2, site_status_code: 200 } });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.scrapeUrl.fields })).toEqual([]);
+ });
+ test('advertised defaults are sent when an agent leaves them empty', async () => {
+ const seen = stubFetch(replies([{ body: scrapeResponse }]));
+ await runAction({ action: scrapeUrl, propsValue: { url: 'https://example.com', elements: [{ selector: 'h1' }], waitForSelector: '#main' } });
+ expect(seen[0].body).toEqual({
+ url: 'https://example.com',
+ elements: [{ selector: 'h1' }],
+ gotoOptions: { timeout: 30000, waitUntil: 'networkidle2' },
+ waitForSelector: { selector: '#main', visible: true },
+ viewport: { width: 1920, height: 1080 },
+ });
+ });
+ test('a blank selector is refused', async () => {
+ stubFetch(replies([{ body: scrapeResponse }]));
+ await expect(runAction({ action: scrapeUrl, propsValue: { url: 'https://example.com', elements: [{ selector: ' ' }] } })).rejects.toThrow(/needs a CSS selector/);
+ });
+});
+
+describe('run_bql_query', () => {
+ test('posts to /chromium/bql with launch query params', async () => {
+ const seen = stubFetch(replies([{ body: { data: { goto: { status: 200 } } } }]));
+ const { output } = await runAction({
+ action: runBqlQuery,
+ propsValue: { query: 'mutation { goto(url: "https://example.com") { status } }', variables: {}, timeout: 30000, stealth: true, headless: true, proxy: 'residential', proxyCountry: 'us', viewportWidth: 800, viewportHeight: 600 },
+ });
+ expect(seen[0].path).toBe('/chromium/bql');
+ expect(seen[0].query.get('token')).toBe(TOKEN);
+ expect(seen[0].query.get('timeout')).toBe('30000');
+ expect(seen[0].query.get('stealth')).toBe('true');
+ expect(seen[0].query.get('proxy')).toBe('residential');
+ expect(seen[0].query.get('proxyCountry')).toBe('us');
+ expect(seen[0].query.get('viewport')).toBe('800x600');
+ expect(seen[0].body).toEqual({ query: 'mutation { goto(url: "https://example.com") { status } }' });
+ expect(output).toMatchObject({ success: true, data: { goto: { status: 200 } }, errors: null });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.runBqlQuery.fields })).toEqual([]);
+ });
+ test('advertised defaults are sent when an agent leaves them empty', async () => {
+ const seen = stubFetch(replies([{ body: { data: { goto: { status: 200 } } } }]));
+ await runAction({ action: runBqlQuery, propsValue: { query: 'mutation { x }' } });
+ expect(seen[0].query.get('timeout')).toBe('30000');
+ expect(seen[0].query.get('stealth')).toBe('true');
+ expect(seen[0].query.get('headless')).toBe('true');
+ });
+ test('unticked stealth and headless are sent as false', async () => {
+ const seen = stubFetch(replies([{ body: { data: { goto: { status: 200 } } } }]));
+ await runAction({ action: runBqlQuery, propsValue: { query: 'mutation { x }', stealth: false, headless: false } });
+ expect(seen[0].query.get('stealth')).toBe('false');
+ expect(seen[0].query.get('headless')).toBe('false');
+ });
+ test('errors without data fail the step', async () => {
+ stubFetch(replies([{ body: { data: null, errors: [{ message: 'Timed out waiting for selector' }] } }]));
+ await expect(runAction({ action: runBqlQuery, propsValue: { query: 'mutation { x }' } })).rejects.toThrow(/Run BQL Query failed: Timed out waiting for selector/);
+ });
+ test('partial data keeps the errors in the output', async () => {
+ stubFetch(replies([{ body: { data: { goto: { status: 200 } }, errors: [{ message: 'click failed' }] } }]));
+ const { output } = await runAction({ action: runBqlQuery, propsValue: { query: 'mutation { x }' } });
+ expect(output).toMatchObject({ data: { goto: { status: 200 } }, errors: [{ message: 'click failed' }] });
+ });
+});
+
+describe('get_website_performance', () => {
+ const lhr = {
+ lighthouseVersion: '12.2.0',
+ finalDisplayedUrl: 'https://example.com/',
+ categories: { performance: { score: 0 }, accessibility: { score: 0.91 }, 'best-practices': { score: 1 }, seo: { score: 0.5 } },
+ audits: {
+ 'first-contentful-paint': { displayValue: '0.8 s', numericValue: 812, score: 0.98 },
+ 'largest-contentful-paint': { displayValue: '1.2 s', numericValue: 1200, score: 0 },
+ 'speed-index': { displayValue: '0.9 s', score: 1 },
+ interactive: { displayValue: '1.0 s', score: 1 },
+ 'total-blocking-time': { displayValue: '0 ms', score: 1 },
+ 'cumulative-layout-shift': { displayValue: '0', score: 1 },
+ 'unused-javascript': { displayValue: 'Est savings of 20 KiB', details: { items: [{}] } },
+ },
+ };
+ test('desktop audit sends matching screen emulation and keeps 0 scores', async () => {
+ const seen = stubFetch(replies([{ body: { lhr } }]));
+ const { output } = await runAction({
+ action: getWebsitePerformance,
+ propsValue: { url: 'https://example.com', device: 'desktop', throttling: 'mobileSlow4G', onlyCategories: true, locale: 'en-US', timeout: 60000, emulateMediaType: 'print', stealth: true },
+ });
+ expect(seen[0].path).toBe('/performance');
+ expect(seen[0].query.get('stealth')).toBe('true');
+ expect(seen[0].body).toEqual({
+ url: 'https://example.com',
+ config: {
+ extends: 'lighthouse:default',
+ settings: {
+ formFactor: 'desktop',
+ screenEmulation: { mobile: false, width: 1350, height: 940, deviceScaleFactor: 1, disabled: false },
+ onlyCategories: ['performance', 'accessibility', 'best-practices', 'seo'],
+ locale: 'en-US',
+ throttlingMethod: 'simulate',
+ throttling: { rttMs: 150, throughputKbps: 1638.4, cpuSlowdownMultiplier: 4 },
+ maxWaitForLoad: 60000,
+ },
+ },
+ });
+ expect(output).toMatchObject({
+ summary: {
+ scores: { performance: 0, accessibility: 91, bestPractices: 100, seo: 50, pwa: null },
+ metrics: { largestContentfulPaint: { value: '1.2 s', numericValue: 1200, score: 0 } },
+ opportunities: [{ type: 'unused-javascript', potentialSavings: 'Est savings of 20 KiB' }],
+ finalUrl: 'https://example.com/',
+ },
+ fullReport: { lhr },
+ metadata: { lighthouseVersion: '12.2.0' },
+ });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.getWebsitePerformance.fields })).toEqual([]);
+ });
+ test('reads the live response shape { type, data: lhr }', async () => {
+ stubFetch(replies([{ body: { type: 'json', data: lhr } }]));
+ const { output } = await runAction({ action: getWebsitePerformance, propsValue: { url: 'https://example.com', includeFullReport: false } });
+ expect(output).toMatchObject({ summary: { scores: { performance: 0, seo: 50 }, finalUrl: 'https://example.com/' }, metadata: { lighthouseVersion: '12.2.0' } });
+ });
+ test('mobile emulation, no throttling, categories deduped, full report optional', async () => {
+ const seen = stubFetch(replies([{ body: { lhr } }]));
+ const { output } = await runAction({
+ action: getWebsitePerformance,
+ propsValue: { url: 'https://example.com', device: 'mobile', throttling: 'none', categories: [{ category: 'seo' }, { category: 'seo' }], includeFullReport: false },
+ });
+ expect(seen[0].body).toMatchObject({ config: { settings: { formFactor: 'mobile', screenEmulation: { mobile: true }, onlyCategories: ['seo'], throttlingMethod: 'provided' } } });
+ expect(output).not.toHaveProperty('fullReport');
+ });
+});
diff --git a/packages/pieces/community/browserless/test/helpers.ts b/packages/pieces/community/browserless/test/helpers.ts
new file mode 100644
index 00000000000..e5ca097c8c2
--- /dev/null
+++ b/packages/pieces/community/browserless/test/helpers.ts
@@ -0,0 +1,131 @@
+import { createMockActionContext } from '@activepieces/pieces-framework';
+import { vi } from 'vitest';
+
+export function stubFetch(respond: (request: SeenRequest, index: number) => FakeReply): SeenRequest[] {
+ const seen: SeenRequest[] = [];
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (input: string | URL | Request, init?: RequestInit) => {
+ const url = input instanceof Request ? input.url : String(input);
+ const parsed = new URL(url);
+ const headers = new Headers(init?.headers);
+ const rawBody = typeof init?.body === 'string' ? init.body : null;
+ const request: SeenRequest = {
+ url,
+ origin: parsed.origin,
+ method: (init?.method ?? 'GET').toUpperCase(),
+ path: parsed.pathname,
+ query: parsed.searchParams,
+ headers,
+ redirect: init?.redirect ?? null,
+ body: rawBody ? JSON.parse(rawBody) : null,
+ };
+ seen.push(request);
+ const reply = respond(request, seen.length - 1);
+ if (reply.throws) {
+ throw reply.throws;
+ }
+ const payload = reply.binary ?? reply.text ?? JSON.stringify(reply.body ?? {});
+ const contentType = reply.binary ? 'application/octet-stream' : reply.text !== undefined ? 'text/html' : 'application/json';
+ return new Response(payload, {
+ status: reply.status ?? 200,
+ headers: { 'content-type': contentType, ...(reply.headers ?? {}) },
+ });
+ }),
+ );
+ return seen;
+}
+
+export function replies(list: FakeReply[]): (request: SeenRequest, index: number) => FakeReply {
+ return (_request, index) => list[Math.min(index, list.length - 1)];
+}
+
+export function auth(overrides: Partial<AuthProps> = {}) {
+ return {
+ type: 'CUSTOM_AUTH' as const,
+ props: { apiToken: TOKEN, region: 'https://production-sfo.browserless.io', customBaseUrl: '', ...overrides },
+ };
+}
+
+export function actionContext({ propsValue, authProps }: { propsValue: Record<string, unknown>; authProps?: Partial<AuthProps> }) {
+ const written: WrittenFile[] = [];
+ const base = createMockActionContext({ propsValue });
+ return {
+ written,
+ context: {
+ ...base,
+ auth: auth(authProps),
+ propsValue,
+ files: {
+ write: async ({ fileName, data }: { fileName: string; data: Buffer }) => {
+ written.push({ fileName, data });
+ return `https://files.example/${fileName}`;
+ },
+ },
+ },
+ };
+}
+
+export function resolvePath({ value, path }: { value: unknown; path: string }): unknown {
+ if (path === '') {
+ return value;
+ }
+ return path.split('.').reduce<unknown>((current, segment) => {
+ if (current === null || current === undefined || typeof current !== 'object') {
+ return undefined;
+ }
+ return Reflect.get(current, segment);
+ }, value);
+}
+
+export function missingSchemaPaths({ output, fields }: { output: unknown; fields: SchemaField[] }): string[] {
+ return fields.flatMap((field) => {
+ const path = field.value ?? field.key;
+ const resolved = resolvePath({ value: output, path });
+ if (resolved === undefined) {
+ return [path];
+ }
+ if (field.children && resolved !== null && typeof resolved === 'object' && !Array.isArray(resolved)) {
+ return missingSchemaPaths({ output: resolved, fields: field.children }).map((child) => `${path}.${child}`);
+ }
+ if (field.listItems && Array.isArray(resolved) && resolved.length > 0) {
+ return missingSchemaPaths({ output: resolved[0], fields: field.listItems }).map((child) => `${path}[].${child}`);
+ }
+ return [];
+ });
+}
+
+export async function runAction({ action, propsValue, authProps }: { action: Runnable; propsValue: Record<string, unknown>; authProps?: Partial<AuthProps> }) {
+ const { context, written } = actionContext({ propsValue, authProps });
+ const output = await action.run(context);
+ return { output, written };
+}
+
+export const TOKEN = 'bl-test-token';
+
+export type SchemaField = { key: string; value?: string; children?: SchemaField[]; listItems?: SchemaField[] };
+
+export type AuthProps = { apiToken: string; region: string; customBaseUrl: string };
+
+export type WrittenFile = { fileName: string; data: Buffer };
+
+export type SeenRequest = {
+ url: string;
+ origin: string;
+ method: string;
+ path: string;
+ query: URLSearchParams;
+ headers: Headers;
+ redirect: string | null;
+ body: unknown;
+};
+
+export type FakeReply = {
+ status?: number;
+ body?: unknown;
+ text?: string;
+ binary?: Buffer;
+ headers?: Record<string, string>;
+ throws?: Error;
+};
+type Runnable = { run(context: ReturnType<typeof actionContext>['context']): Promise<unknown> };
diff --git a/packages/pieces/community/browserless/test/metadata.test.ts b/packages/pieces/community/browserless/test/metadata.test.ts
new file mode 100644
index 00000000000..f89cdb83cb9
--- /dev/null
+++ b/packages/pieces/community/browserless/test/metadata.test.ts
@@ -0,0 +1,93 @@
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { browserless } from '../src';
+import { TOKEN, actionContext, replies, stubFetch } from './helpers';
+
+const actions = Object.values(browserless.actions());
+const nonCustom = actions.filter((action) => action.name !== 'custom_api_call');
+const AUDIENCE: Record<string, string> = {
+ capture_screenshot: 'human',
+ generate_pdf: 'human',
+ get_website_performance: 'human',
+ take_screenshot: 'ai',
+ create_pdf: 'ai',
+ audit_page_performance: 'ai',
+};
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+describe('piece metadata', () => {
+ test('19 actions, existing names kept', () => {
+ expect(actions.map((action) => action.name).sort()).toEqual(
+ [
+ 'cancel_crawl',
+ 'capture_screenshot',
+ 'custom_api_call',
+ 'generate_pdf',
+ 'get_crawl',
+ 'get_page_content',
+ 'get_website_performance',
+ 'list_crawls',
+ 'map_website',
+ 'run_bql_query',
+ 'run_function',
+ 'scrape_url',
+ 'search_web',
+ 'smart_scrape',
+ 'start_crawl',
+ 'unblock_page',
+ 'take_screenshot',
+ 'create_pdf',
+ 'audit_page_performance',
+ ].sort(),
+ );
+ });
+ test.each(nonCustom.map((action) => [action.name, action]))('%s declares audience, classification, aiMetadata and outputSchema', (_name, action) => {
+ expect(action.audience).toBe(AUDIENCE[action.name] ?? 'both');
+ expect(['READ', 'SEARCH', 'WRITE', 'DESTRUCTIVE']).toContain(action.classification);
+ expect(action.aiMetadata?.description?.length ?? 0).toBeGreaterThan(80);
+ expect(typeof action.aiMetadata?.idempotent).toBe('boolean');
+ expect(action.outputSchema?.fields.length).toBeGreaterThan(0);
+ });
+ test('classification by side effect', () => {
+ const byClass = (cls: string) => nonCustom.filter((action) => action.classification === cls).map((action) => action.name).sort();
+ expect(byClass('DESTRUCTIVE')).toEqual(['cancel_crawl']);
+ expect(byClass('WRITE')).toEqual(['run_bql_query', 'run_function']);
+ expect(byClass('SEARCH')).toEqual(['list_crawls', 'map_website', 'search_web', 'start_crawl']);
+ });
+ test('ai actions never return base64 or the raw Lighthouse report', () => {
+ for (const action of nonCustom.filter((item) => item.audience === 'ai')) {
+ const keys = (action.outputSchema?.fields ?? []).map((field) => field.value ?? field.key);
+ expect(keys.some((key) => /base64|fullReport/i.test(key))).toBe(false);
+ }
+ });
+ test('existing action prop keys are kept', () => {
+ const keys = (name: string) => Object.keys(browserless.actions()[name].props);
+ expect(keys('capture_screenshot')).toEqual(expect.arrayContaining(['url', 'imageType', 'quality', 'fullPage', 'width', 'height', 'waitForSelector', 'delay', 'omitBackground', 'clipX', 'clipY', 'clipWidth', 'clipHeight']));
+ expect(keys('generate_pdf')).toEqual(expect.arrayContaining(['url', 'html', 'format', 'landscape', 'printBackground', 'waitForSelector', 'waitForFunction', 'userAgent', 'timeout', 'bestAttempt']));
+ expect(keys('scrape_url')).toEqual(expect.arrayContaining(['url', 'elements', 'waitForSelector', 'debugConsole', 'cookies', 'timeout', 'waitUntil', 'viewportWidth']));
+ expect(keys('run_bql_query')).toEqual(expect.arrayContaining(['query', 'variables', 'operationName', 'timeout', 'stealth', 'proxy', 'cookies']));
+ expect(keys('get_website_performance')).toEqual(expect.arrayContaining(['url', 'categories', 'device', 'throttling', 'onlyCategories', 'budgets', 'stealth', 'blockAds']));
+ });
+});
+
+describe('custom API call', () => {
+ const customCall = browserless.actions()['custom_api_call'];
+ test('relative paths go to the connection host with the token', async () => {
+ const seen = stubFetch(replies([{ body: { version: '2.40.0' } }]));
+ const { context } = actionContext({ propsValue: { url: { url: '/meta' }, method: 'GET', headers: {}, queryParams: {}, failsafe: false } });
+ await customCall.run(context);
+ expect(seen[0].origin).toBe('https://production-sfo.browserless.io');
+ expect(seen[0].path).toBe('/meta');
+ expect(seen[0].query.get('token')).toBe(TOKEN);
+ });
+ test('absolute URLs on another host are refused before the token is sent', async () => {
+ const seen = stubFetch(replies([{ body: {} }]));
+ for (const url of ['https://production-sfo.browserless.io.evil.io/meta', 'https://evil.io/meta', 'http://production-sfo.browserless.io/meta']) {
+ const { context } = actionContext({ propsValue: { url: { url }, method: 'GET', headers: {}, queryParams: {}, failsafe: false } });
+ await expect(customCall.run(context)).rejects.toThrow(/only sends your Browserless token/);
+ }
+ expect(seen).toHaveLength(0);
+ });
+});
diff --git a/packages/pieces/community/browserless/test/new-actions.test.ts b/packages/pieces/community/browserless/test/new-actions.test.ts
new file mode 100644
index 00000000000..9e19f695ee1
--- /dev/null
+++ b/packages/pieces/community/browserless/test/new-actions.test.ts
@@ -0,0 +1,333 @@
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { cancelCrawl } from '../src/lib/actions/cancel-crawl';
+import { getCrawl } from '../src/lib/actions/get-crawl';
+import { getPageContent } from '../src/lib/actions/get-page-content';
+import { listCrawls } from '../src/lib/actions/list-crawls';
+import { mapWebsite } from '../src/lib/actions/map-website';
+import { runFunction } from '../src/lib/actions/run-function';
+import { searchWeb } from '../src/lib/actions/search-web';
+import { smartScrape } from '../src/lib/actions/smart-scrape';
+import { startCrawl } from '../src/lib/actions/start-crawl';
+import { unblockPage } from '../src/lib/actions/unblock-page';
+import { browserlessOutputSchemas } from '../src/lib/output-schemas';
+import { missingSchemaPaths, replies, runAction, stubFetch } from './helpers';
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+describe('get_page_content', () => {
+ test('posts to /content and returns html with the site status', async () => {
+ const seen = stubFetch(replies([{ text: '<html><h1>Example</h1></html>', headers: { 'x-response-code': '200', 'x-response-status': 'OK', 'x-response-url': 'https://example.com/' } }]));
+ const { output } = await runAction({
+ action: getPageContent,
+ propsValue: { url: ' https://example.com ', waitUntil: 'networkidle2', navigationTimeout: 15000, waitForSelector: 'h1', waitForTimeout: 0, rejectResourceTypes: ['image', 'font'], bestAttempt: true, blockAds: true, timeout: 45000 },
+ });
+ expect(seen[0].path).toBe('/content');
+ expect(seen[0].query.get('blockAds')).toBe('true');
+ expect(seen[0].query.get('timeout')).toBe('45000');
+ expect(seen[0].body).toEqual({
+ url: 'https://example.com',
+ gotoOptions: { waitUntil: 'networkidle2', timeout: 15000 },
+ waitForSelector: { selector: 'h1' },
+ waitForTimeout: 0,
+ bestAttempt: true,
+ rejectResourceTypes: ['image', 'font'],
+ });
+ expect(output).toEqual({ url: 'https://example.com', site_status_code: 200, site_status_text: 'OK', final_url: 'https://example.com/', html: '<html><h1>Example</h1></html>', html_length: 29, truncated: false });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.getPageContent.fields })).toEqual([]);
+ });
+ test('caps HTML at Maximum Characters and reports the full length; 0 means no limit', async () => {
+ const page = `<html>${'x'.repeat(150_000)}</html>`;
+ stubFetch(replies([{ text: page }]));
+ const capped = await runAction({ action: getPageContent, propsValue: { url: 'https://example.com' } });
+ expect(capped.output).toMatchObject({ html_length: page.length, truncated: true });
+ expect(String(Reflect.get(Object(capped.output), 'html')).length).toBe(100_000);
+ const small = await runAction({ action: getPageContent, propsValue: { url: 'https://example.com', maxCharacters: 10 } });
+ expect(small.output).toMatchObject({ html: '<html>xxxx', truncated: true });
+ const full = await runAction({ action: getPageContent, propsValue: { url: 'https://example.com', maxCharacters: 0 } });
+ expect(full.output).toMatchObject({ html: page, truncated: false });
+ });
+ test('rejects a non-http URL and a timeout above the step limit', async () => {
+ const seen = stubFetch(replies([{ text: '' }]));
+ await expect(runAction({ action: getPageContent, propsValue: { url: 'example.com' } })).rejects.toThrow(/http:\/\/ or https:\/\//);
+ await expect(runAction({ action: getPageContent, propsValue: { url: 'https://example.com', timeout: 900000 } })).rejects.toThrow(/at most 540000/);
+ expect(seen).toHaveLength(0);
+ });
+});
+
+describe('smart_scrape', () => {
+ const okResponse = {
+ ok: true,
+ statusCode: 200,
+ content: '<html><h1>Title</h1></html>',
+ contentType: 'text/html; charset=utf-8',
+ headers: { 'content-type': 'text/html' },
+ strategy: 'http-fetch',
+ attempted: ['http-fetch'],
+ message: null,
+ actions: null,
+ screenshot: Buffer.from('png-bytes').toString('base64'),
+ pdf: null,
+ markdown: '# Title',
+ rawText: 'Title',
+ links: ['https://example.com/a'],
+ metadata: { title: 'Page Title', description: 'Desc', language: 'en', sourceURL: 'https://example.com', statusCode: 200 },
+ };
+ test('flattens the result and stores the screenshot as a file', async () => {
+ const seen = stubFetch(replies([{ body: okResponse }]));
+ const { output, written } = await runAction({
+ action: smartScrape,
+ propsValue: { url: 'https://example.com', formats: ['markdown', 'html', 'links', 'screenshot'], onlyMainContent: true, excludeTags: ['.ads'], waitFor: 1000, proxy: 'datacenter' },
+ });
+ expect(seen[0].path).toBe('/smart-scrape');
+ expect(seen[0].body).toEqual({ url: 'https://example.com', formats: ['markdown', 'html', 'links', 'screenshot'], onlyMainContent: true, excludeTags: ['.ads'], waitFor: 1000, proxy: 'datacenter' });
+ expect(written[0]).toMatchObject({ fileName: 'smart-scrape.png' });
+ expect(written[0].data.toString()).toBe('png-bytes');
+ expect(output).toMatchObject({ truncated: false, markdown: '# Title', html: '<html><h1>Title</h1></html>', title: 'Page Title', strategy: 'http-fetch', links_count: 1, screenshot_file: 'https://files.example/smart-scrape.png', pdf_file: null, content: null });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.smartScrape.fields })).toEqual([]);
+ });
+ test('defaults to markdown and fails on ok:false (HTTP 200)', async () => {
+ const seen = stubFetch(replies([{ body: { ok: false, statusCode: null, message: 'All strategies failed: blocked', attempted: ['http-fetch', 'browser'] } }]));
+ await expect(runAction({ action: smartScrape, propsValue: { url: 'https://blocked.example' } })).rejects.toThrow(/Smart Scrape failed for https:\/\/blocked.example: All strategies failed: blocked/);
+ expect(seen[0].body).toEqual({ url: 'https://blocked.example', formats: ['markdown'] });
+ });
+ test('markdown is capped at Maximum Characters', async () => {
+ stubFetch(replies([{ body: { ...okResponse, markdown: 'm'.repeat(50), screenshot: null } }]));
+ const { output } = await runAction({ action: smartScrape, propsValue: { url: 'https://example.com', maxCharacters: 20 } });
+ expect(output).toMatchObject({ markdown: 'm'.repeat(20), truncated: true });
+ });
+ test('include and exclude selectors cannot be combined', async () => {
+ stubFetch(replies([{ body: okResponse }]));
+ await expect(runAction({ action: smartScrape, propsValue: { url: 'https://example.com', includeTags: ['main'], excludeTags: ['nav'] } })).rejects.toThrow(/cannot be combined/);
+ });
+ test('non-HTML targets return content instead of html', async () => {
+ stubFetch(replies([{ body: { ...okResponse, contentType: 'application/json', content: { a: 1 }, screenshot: null } }]));
+ const { output } = await runAction({ action: smartScrape, propsValue: { url: 'https://api.example.com/x.json', formats: ['html'] } });
+ expect(output).toMatchObject({ html: null, content: { a: 1 } });
+ });
+ test('JSON content longer than Maximum Characters is cut to JSON text and flagged', async () => {
+ const content = { text: 'x'.repeat(200) };
+ stubFetch(replies([{ body: { ...okResponse, contentType: 'application/json', content, screenshot: null } }]));
+ const { output } = await runAction({ action: smartScrape, propsValue: { url: 'https://api.example.com/x.json', maxCharacters: 50 } });
+ expect(output).toMatchObject({ content: JSON.stringify(content).slice(0, 50), truncated: true });
+ });
+ test('JSON content within the limit, or with no limit, stays parsed', async () => {
+ const content = { text: 'x'.repeat(200) };
+ stubFetch(replies([{ body: { ...okResponse, contentType: 'application/json', content, screenshot: null } }]));
+ const { output } = await runAction({ action: smartScrape, propsValue: { url: 'https://api.example.com/x.json', maxCharacters: 0 } });
+ expect(output).toMatchObject({ content, truncated: false });
+ });
+ test('agents that leave Maximum Characters empty still get the 100000 cap on JSON', async () => {
+ const content = ['y'.repeat(150_000)];
+ stubFetch(replies([{ body: { ...okResponse, contentType: 'application/json', content, screenshot: null } }]));
+ const { output } = await runAction({ action: smartScrape, propsValue: { url: 'https://api.example.com/x.json' } });
+ expect(output).toMatchObject({ content: JSON.stringify(content).slice(0, 100_000), truncated: true });
+ });
+});
+
+describe('search_web', () => {
+ const response = {
+ success: true,
+ data: {
+ web: [{ title: 'Browserless', url: 'https://www.browserless.io', description: 'Headless', position: 1, markdown: '# B', metadata: { statusCode: 200, strategy: 'http-fetch' } }],
+ news: [{ title: 'News', url: 'https://news.example/1', description: 'n', position: 1, date: '2026-10-01', imageUrl: 'https://img.example/1.jpg' }],
+ images: [{ title: 'Logo', imageUrl: 'https://img.example/logo.png', imageWidth: 100, imageHeight: 50, url: 'https://www.browserless.io', position: 1 }],
+ },
+ totalResults: 3,
+ };
+ test('sends the query and flattens all sources', async () => {
+ const seen = stubFetch(replies([{ body: response }]));
+ const { output } = await runAction({
+ action: searchWeb,
+ propsValue: { query: ' headless browser ', sources: ['web', 'news', 'images'], limit: 5, tbs: 'week', lang: 'EN', country: 'US', scrapeFormats: ['markdown'], onlyMainContent: true, proxy: 'datacenter' },
+ });
+ expect(seen[0].path).toBe('/search');
+ expect(seen[0].body).toEqual({
+ query: 'headless browser',
+ limit: 5,
+ sources: ['web', 'news', 'images'],
+ tbs: 'week',
+ lang: 'en',
+ country: 'us',
+ proxy: 'datacenter',
+ scrapeOptions: { formats: ['markdown'], onlyMainContent: true },
+ });
+ expect(output).toMatchObject({ total_results: 3, result_count: 3, no_results: false });
+ expect(output).toMatchObject({ results: [{ source: 'web', markdown: '# B', page_status_code: 200 }, { source: 'news', date: '2026-10-01' }, { source: 'images', image_url: 'https://img.example/logo.png' }] });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.searchWeb.fields })).toEqual([]);
+ });
+ test('no_results is an empty list, other errors fail', async () => {
+ stubFetch(replies([{ body: { success: false, data: {}, totalResults: 0, errorCode: 'no_results' } }, { body: { success: false, error: 'Search engines did not answer', errorCode: 'upstream_timeout' } }]));
+ const empty = await runAction({ action: searchWeb, propsValue: { query: 'zzzz' } });
+ expect(empty.output).toMatchObject({ result_count: 0, no_results: true, results: [] });
+ await expect(runAction({ action: searchWeb, propsValue: { query: 'x' } })).rejects.toThrow(/Search engines did not answer/);
+ });
+});
+
+describe('map_website', () => {
+ test('accepts the documented {success, links} shape', async () => {
+ const seen = stubFetch(replies([{ body: { success: true, links: [{ url: 'https://example.com/a', title: 'A' }, { url: 'https://example.com/b' }] } }]));
+ const { output } = await runAction({ action: mapWebsite, propsValue: { url: 'https://example.com', search: 'pricing', limit: 50, sitemap: 'include', includeSubdomains: false, ignoreQueryParameters: true, proxy: 'datacenter' } });
+ expect(seen[0].body).toEqual({ url: 'https://example.com', search: 'pricing', limit: 50, sitemap: 'include', includeSubdomains: false, ignoreQueryParameters: true, proxy: 'datacenter' });
+ expect(output).toEqual({ url: 'https://example.com', count: 2, links: [{ url: 'https://example.com/a', title: 'A', description: null }, { url: 'https://example.com/b', title: null, description: null }] });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.mapWebsite.fields })).toEqual([]);
+ });
+ test('accepts the OpenAPI array-of-strings shape and fails on success:false', async () => {
+ const seen = stubFetch(replies([{ body: ['https://example.com/a'] }, { body: { success: false, error: 'Invalid URL' } }]));
+ const { output } = await runAction({ action: mapWebsite, propsValue: { url: 'https://example.com' } });
+ expect(output).toMatchObject({ count: 1, links: [{ url: 'https://example.com/a' }] });
+ expect(seen[0].body).toEqual({ url: 'https://example.com', limit: 500, includeSubdomains: true, ignoreQueryParameters: true });
+ await expect(runAction({ action: mapWebsite, propsValue: { url: 'https://example.com' } })).rejects.toThrow(/Invalid URL/);
+ });
+});
+
+describe('crawl actions', () => {
+ test('start_crawl posts options and returns the ID', async () => {
+ const seen = stubFetch(replies([{ body: { success: true, id: 'crawl_abc123', url: 'https://production-sfo.browserless.io/crawl/crawl_abc123' } }]));
+ const { output } = await runAction({
+ action: startCrawl,
+ propsValue: { url: 'https://example.com/docs', limit: 10, maxDepth: 2, includePaths: ['^/docs/'], excludePaths: [], formats: ['markdown'], onlyMainContent: true, proxy: 'datacenter', sitemap: 'auto' },
+ });
+ expect(seen[0].path).toBe('/crawl');
+ expect(seen[0].body).toEqual({
+ url: 'https://example.com/docs',
+ limit: 10,
+ maxDepth: 2,
+ includePaths: ['^/docs/'],
+ sitemap: 'auto',
+ scrapeOptions: { formats: ['markdown'], onlyMainContent: true, proxy: 'datacenter' },
+ });
+ expect(output).toEqual({ crawl_id: 'crawl_abc123', url: 'https://example.com/docs', status: 'in-progress' });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.startCrawl.fields })).toEqual([]);
+ });
+ test('start_crawl applies the 10-page cap and main-content default when an agent leaves them empty', async () => {
+ const seen = stubFetch(replies([{ body: { success: true, id: 'crawl_abc123' } }]));
+ await runAction({ action: startCrawl, propsValue: { url: 'https://example.com' } });
+ expect(seen[0].body).toEqual({ url: 'https://example.com', limit: 10, scrapeOptions: { onlyMainContent: true } });
+ });
+ test('get_crawl reads a batch and the next skip', async () => {
+ const seen = stubFetch(
+ replies([
+ {
+ body: {
+ status: 'completed',
+ total: 15,
+ completed: 15,
+ failed: 0,
+ expiresAt: '2025-07-01T12:00:00.000Z',
+ next: 'https://production-sfo.browserless.io/crawl/crawl_abc123?skip=10',
+ data: [
+ {
+ status: 'completed',
+ contentUrl: 'https://crawl-artifacts.s3.amazonaws.com/page.json?X-Amz-Signature=x',
+ metadata: { title: 'Home', description: 'd', language: 'en', scrapedAt: '2025-06-30T10:00:00.000Z', sourceURL: 'https://example.com', statusCode: 200, error: null },
+ },
+ ],
+ },
+ },
+ ]),
+ );
+ const { output } = await runAction({ action: getCrawl, propsValue: { crawlId: ' crawl_abc123 ', skip: 0 } });
+ expect(seen[0].method).toBe('GET');
+ expect(seen[0].path).toBe('/crawl/crawl_abc123');
+ expect(seen[0].query.get('skip')).toBe('0');
+ expect(output).toMatchObject({ status: 'completed', total: 15, has_more: true, next_skip: 10, page_count: 1, pages: [{ url: 'https://example.com', title: 'Home', status_code: 200 }] });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.getCrawl.fields })).toEqual([]);
+ });
+ test('crawl IDs are validated before use in the path', async () => {
+ const seen = stubFetch(replies([{ body: {} }]));
+ await expect(runAction({ action: getCrawl, propsValue: { crawlId: '../session' } })).rejects.toThrow(/Crawl ID must look like/);
+ await expect(runAction({ action: cancelCrawl, propsValue: { crawlId: 'a/b' } })).rejects.toThrow(/Crawl ID must look like/);
+ expect(seen).toHaveLength(0);
+ });
+ test('list_crawls passes filters and the cursor', async () => {
+ const seen = stubFetch(replies([{ body: { crawls: [{ id: 'crawl_1', url: 'https://example.com', status: 'completed', total: 15, completed: 15, createdAt: '2025-06-30T09:00:00.000Z', completedAt: '2025-06-30T09:05:00.000Z' }], nextCursor: 'eyJza2lwIjoxMH0' } }]));
+ const { output } = await runAction({ action: listCrawls, propsValue: { status: 'completed', limit: 10, cursor: 'abc' } });
+ expect(seen[0].path).toBe('/crawl');
+ expect(seen[0].query.get('status')).toBe('completed');
+ expect(seen[0].query.get('limit')).toBe('10');
+ expect(seen[0].query.get('cursor')).toBe('abc');
+ expect(output).toMatchObject({ count: 1, has_more: true, next_cursor: 'eyJza2lwIjoxMH0', crawls: [{ id: 'crawl_1', created_at: '2025-06-30T09:00:00.000Z' }] });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.listCrawls.fields })).toEqual([]);
+ });
+ test('cancel_crawl: DELETE, 409 reported as already finished, 404 fails', async () => {
+ const seen = stubFetch(
+ replies([
+ { body: { status: 'cancelled' } },
+ { status: 409, body: { id: 'crawl_abc123', status: 'completed', message: 'Crawl is already completed' } },
+ { status: 404, body: { message: 'Crawl not found' } },
+ ]),
+ );
+ const first = await runAction({ action: cancelCrawl, propsValue: { crawlId: 'crawl_abc123' } });
+ expect(seen[0].method).toBe('DELETE');
+ expect(seen[0].path).toBe('/crawl/crawl_abc123');
+ expect(first.output).toEqual({ crawl_id: 'crawl_abc123', cancelled: true, status: 'cancelled', message: null });
+ expect(missingSchemaPaths({ output: first.output, fields: browserlessOutputSchemas.cancelCrawl.fields })).toEqual([]);
+ const second = await runAction({ action: cancelCrawl, propsValue: { crawlId: 'crawl_abc123' } });
+ expect(second.output).toEqual({ crawl_id: 'crawl_abc123', cancelled: false, status: 'completed', message: 'Crawl is already completed' });
+ await expect(runAction({ action: cancelCrawl, propsValue: { crawlId: 'crawl_abc123' } })).rejects.toThrow(/404/);
+ });
+});
+
+describe('unblock_page', () => {
+ test('sends toggles in the body and proxy/captcha options as query params', async () => {
+ const seen = stubFetch(
+ replies([
+ {
+ body: {
+ content: '<html>ok</html>',
+ cookies: [{ name: 'cf_clearance', value: 'abc', domain: '.example.com', path: '/', expires: 1790000000, httpOnly: true, secure: true, sameSite: 'None' }],
+ browserWSEndpoint: null,
+ ttl: 0,
+ screenshot: Buffer.from('jpg').toString('base64'),
+ solved: true,
+ },
+ },
+ ]),
+ );
+ const { output, written } = await runAction({
+ action: unblockPage,
+ propsValue: { url: 'https://example.com', returnContent: true, returnCookies: true, returnScreenshot: true, solveCaptchas: true, proxy: 'residential', proxyCountry: 'US', timeout: 120000 },
+ });
+ expect(seen[0].path).toBe('/unblock');
+ expect(seen[0].query.get('proxy')).toBe('residential');
+ expect(seen[0].query.get('proxyCountry')).toBe('us');
+ expect(seen[0].query.get('solveCaptchas')).toBe('true');
+ expect(seen[0].query.get('timeout')).toBe('120000');
+ expect(seen[0].body).toEqual({ url: 'https://example.com', content: true, cookies: true, screenshot: true, browserWSEndpoint: false });
+ expect(written[0].fileName).toBe('unblocked.jpg');
+ expect(output).toMatchObject({ truncated: false, solved: true, html_length: 15, cookie_count: 1, cookie_header: 'cf_clearance=abc', screenshot_file: 'https://files.example/unblocked.jpg', cookies: [{ name: 'cf_clearance', http_only: true, same_site: 'None' }] });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.unblockPage.fields })).toEqual([]);
+ });
+});
+
+describe('run_function', () => {
+ test('JSON result is parsed', async () => {
+ const seen = stubFetch(replies([{ body: { title: 'Example Domain' } }]));
+ const { output } = await runAction({ action: runFunction, propsValue: { code: 'export default async () => ({ data: 1 })', context: { url: 'https://example.com' }, timeout: 30000 } });
+ expect(seen[0].path).toBe('/function');
+ expect(seen[0].query.get('timeout')).toBe('30000');
+ expect(seen[0].body).toEqual({ code: 'export default async () => ({ data: 1 })', context: { url: 'https://example.com' } });
+ expect(output).toEqual({ content_type: 'application/json', result: { title: 'Example Domain' }, file: null, size_bytes: 26 });
+ expect(missingSchemaPaths({ output, fields: browserlessOutputSchemas.runFunction.fields })).toEqual([]);
+ });
+ test('live envelope { data, type } is unwrapped; binary data becomes a file', async () => {
+ stubFetch(replies([{ body: { data: { title: 'Example Domain' }, type: 'application/json' } }, { body: { data: { 0: 37, 1: 80, 2: 68, 3: 70 }, type: 'application/pdf' } }]));
+ const json = await runAction({ action: runFunction, propsValue: { code: 'x' } });
+ expect(json.output).toMatchObject({ content_type: 'application/json', result: { title: 'Example Domain' }, file: null });
+ const pdf = await runAction({ action: runFunction, propsValue: { code: 'x' } });
+ expect(pdf.written[0].fileName).toBe('function-result.pdf');
+ expect(pdf.written[0].data.toString()).toBe('%PDF');
+ expect(pdf.output).toEqual({ content_type: 'application/pdf', result: null, file: 'https://files.example/function-result.pdf', size_bytes: 4 });
+ });
+ test('binary result is stored as a file', async () => {
+ stubFetch(replies([{ binary: Buffer.from('%PDF-1.7'), headers: { 'content-type': 'application/pdf' } }]));
+ const { output, written } = await runAction({ action: runFunction, propsValue: { code: 'x' } });
+ expect(written[0].fileName).toBe('function-result.pdf');
+ expect(output).toEqual({ content_type: 'application/pdf', result: null, file: 'https://files.example/function-result.pdf', size_bytes: 8 });
+ });
+ test('empty code is refused', async () => {
+ await expect(runAction({ action: runFunction, propsValue: { code: ' ' } })).rejects.toThrow(/Enter the code/);
+ });
+});
diff --git a/packages/pieces/community/browserless/vitest.config.ts b/packages/pieces/community/browserless/vitest.config.ts
new file mode 100644
index 00000000000..79c97261904
--- /dev/null
+++ b/packages/pieces/community/browserless/vitest.config.ts
@@ -0,0 +1,19 @@
+import path from 'path'
+import { defineConfig } from 'vitest/config'
+
+const repoRoot = path.resolve(__dirname, '../../../..')
+
+export default defineConfig({
+ root: __dirname,
+ test: {
+ globals: true,
+ environment: 'node',
+ include: ['test/**/*.test.ts'],
+ },
+ resolve: {
+ alias: {
+ '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'),
+ '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'),
+ },
+ },
+})