Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { defineCommand, type IFileSystem } from "just-bash";
const ORIGIN = "bash";
const INTEGRATION = "_firecrawl-bash";
const INSTALL = "npx -y firecrawl-cli@latest init";
const SEARCH_SOURCES = ["web", "images", "news"];
const SEARCH_CATEGORIES = ["research", "pdf", "developer", "gov"];
// Every format the CLI accepts; the hosted shell serves SCRAPE_FORMATS of them.
const CLI_FORMATS = [
"markdown",
"html",
"rawHtml",
"links",
"images",
"screenshot",
"summary",
"changeTracking",
"json",
"attributes",
"branding",
];
const SCRAPE_FORMATS = [
"markdown",
"html",
"rawHtml",
"links",
"images",
"summary",
];
const API_KEY_FLAG =
"--api-key isn't accepted in the hosted shell. Send your key in the HTTP request's Authorization: Bearer fc-... header instead";
const KEY_NEEDED =
"Firecrawl's free tier without an API key isn't available from your network (it needs an IPv4 address). Create a free API key at https://www.firecrawl.dev/signin and send it as Authorization: Bearer fc-...";
export interface FirecrawlCaller {
/** This server's own address: calls go through the same routes a REST caller uses. */
baseUrl: string;
ip: string | null;
/** The caller's Authorization header as sent, so the API parses it as it would directly. */
authorization: string | null;
/** Caller headers kept on each call: User-Agent and a World ID credential. */
headers: Record<string, string>;
maxCalls: number;
/** When the command must be done; each call asks the API to finish before it. */
deadline: number;
}
// Room to read the response and print it before the command's deadline.
const CALL_MARGIN_MS = 3_000;
type Endpoint = "search" | "scrape";
/** Calls the API as the shell's caller and records what was called, for analytics. */
export class FirecrawlClient {
readonly endpoints: Endpoint[] = [];
readonly failedEndpoints: Endpoint[] = [];
constructor(private readonly caller: FirecrawlCaller) {}
async post(
endpoint: Endpoint,
body: Record<string, unknown>,
signal?: AbortSignal,
): Promise<any> {
const { baseUrl, ip, authorization, headers, maxCalls, deadline } =
this.caller;
if (this.endpoints.length >= maxCalls) {
throw new Error(
`call limit reached (${maxCalls} Firecrawl calls per command)`,
);
}
// A call the API is still working on at the deadline would be billed but lost.
const timeout = deadline - Date.now() - CALL_MARGIN_MS;
if (timeout < 1_000) {
throw new Error("not enough time left before the command's deadline");
}
this.endpoints.push(endpoint);
const res = await fetch(`${baseUrl}/v2/${endpoint}`, {
method: "POST",
headers: {
...headers,
"Content-Type": "application/json",
...(authorization ? { Authorization: authorization } : {}),
// This request arrives from loopback, a trusted hop under
// EXPRESS_TRUST_PROXY, so auth and quotas see the caller's IP.
...(ip ? { "X-Forwarded-For": ip } : {}),
},
body: JSON.stringify({
...body,
timeout,
origin: ORIGIN,
integration: INTEGRATION,
}),
signal,
});
const json: any = await res.json().catch(() => ({}));
if (!res.ok || json?.success === false) {
this.failedEndpoints.push(endpoint);
// Keyless refuses an IPv6 caller with a bare 401.
const bare = !json?.error || json.error === "Unauthorized";
if (!authorization && res.status === 401 && bare) {
throw new Error(KEY_NEEDED);
}
throw new Error(apiError(json, res.status));
}
return json;
}
}
/** The API's error, with the first few validation issues it lists. */
function apiError(json: any, status: number): string {
const message = json?.error || `Firecrawl request failed (HTTP ${status})`;
const issues: string[] = Array.isArray(json?.details)
? json.details
.slice(0, 3)
.filter((d: any) => typeof d?.message === "string")
.map((d: any) =>
Array.isArray(d.path) && d.path.length
? `${d.path.join(".")}: ${d.message}`
: d.message,
)
: [];
return issues.length ? `${message} (${issues.join("; ")})` : message;
}
export const FIRECRAWL_HELP = `Usage: firecrawl [command] [options]
A hosted subset of the Firecrawl CLI. The flags and output follow the real CLI
(/docs/sdks/cli.md); install it for everything else: ${INSTALL}
Commands:
search <query> Search the web
--limit <number> Maximum number of results (default: 5)
--sources <sources> Comma-separated sources: web, images, news (default: web)
--categories <categories> Comma-separated categories: ${SEARCH_CATEGORIES.join(", ")}
--tbs <value> Time-based search: qdr:h, qdr:d, qdr:w, qdr:m, qdr:y
--location <location> Location for geo-targeting
--country <code> ISO country code for geo-targeting
--scrape Enable scraping of search results
--scrape-formats <formats> Comma-separated scrape formats when --scrape is enabled (default: markdown)
-o, --output <path> Output file path (default: stdout)
--json Output as compact JSON
--pretty Output as pretty JSON
scrape [urls...] Scrape URLs. Multiple URLs are saved to .firecrawl/
-f, --format <formats> Output format(s): ${SCRAPE_FORMATS.join(", ")}.
Single format outputs raw content; multiple formats output JSON.
-H, --html Output raw HTML (shortcut for --format html)
-S, --summary Output summary (shortcut for --format summary)
--only-main-content Include only the main content (the API default)
--full-page Keep the whole page, including navigation, headers and footers
--wait-for <ms> Wait time before scraping in milliseconds
--include-tags <tags> Comma-separated list of tags to include
--exclude-tags <tags> Comma-separated list of tags to exclude
--max-age <milliseconds> Maximum age of cached content in milliseconds
--country <code> ISO country code for geo-targeted scraping
-o, --output <path> Output file path (default: stdout)
--json Output as JSON format
--pretty Pretty print JSON output
help Display this help
`;
/** Thrown for anything the hosted subset leaves out; exits 2, unlike a failed call. */
class Unsupported extends Error {}
type Flags = Record<string, string | true>;
interface FlagSpec {
values: string[];
booleans: string[];
aliases: Record<string, string>;
}
const SEARCH_FLAGS: FlagSpec = {
values: [
"limit",
"sources",
"categories",
"tbs",
"location",
"country",
"scrape-formats",
"output",
],
booleans: ["scrape", "json", "pretty"],
aliases: { o: "output" },
};
const SCRAPE_FLAGS: FlagSpec = {
values: [
"url",
"format",
"wait-for",
"include-tags",
"exclude-tags",
"max-age",
"country",
"output",
],
booleans: [
"html",
"summary",
"only-main-content",
"full-page",
"json",
"pretty",
],
aliases: {
u: "url",
f: "format",
formats: "format",
H: "html",
S: "summary",
o: "output",
},
};
function parseArgs(args: string[], spec: FlagSpec) {
const positional: string[] = [];
const flags: Flags = {};
for (let i = 0; i < args.length; i++) {
const arg = args[i];
if (arg === "--") {
positional.push(...args.slice(i + 1));
break;
}
const match = /^(?:--([\w-]+)|-([A-Za-z]))(?:=(.*))?$/s.exec(arg);
if (!match) {
positional.push(arg);
continue;
}
const raw = match[1] ?? match[2];
const name = spec.aliases[raw] ?? raw;
const shown = match[1] ? `--${raw}` : `-${raw}`;
if (spec.booleans.includes(name)) {
flags[name] = true;
} else if (!spec.values.includes(name)) {
throw new Unsupported(`option '${shown}' isn't in the hosted shell`);
} else if (match[3] !== undefined) {
flags[name] = match[3];
} else if (i + 1 < args.length) {
flags[name] = args[++i];
} else {
throw new Error(`option '${shown}' argument missing`);
}
}
return { positional, flags };
}
const text = (flags: Flags, name: string) =>
typeof flags[name] === "string" ? (flags[name] as string) : undefined;
function integer(flags: Flags, name: string): number | undefined {
const value = text(flags, name);
if (value === undefined) return undefined;
const parsed = Number.parseInt(value, 10);
if (Number.isNaN(parsed)) {
throw new Error(`option '--${name}' expects a number`);
}
return parsed;
}
const list = (value: string | undefined) =>
value
?.split(",")
.map(item => item.trim())
.filter(Boolean);
interface Context {
client: FirecrawlClient;
fs: IFileSystem;
cwd: string;
signal?: AbortSignal;
stderr: string[];
}
/** The CLI's writeOutput: a file when a path is given, otherwise stdout ending in a newline. */
async function output(c: Context, content: string, path: string | undefined) {
if (!path) return content.endsWith("\n") ? content : `${content}\n`;
const absolute = path.startsWith("/")
? path
: `${c.cwd.replace(/\/$/, "")}/${path}`;
const dir = absolute.slice(0, absolute.lastIndexOf("/")) || "/";
await c.fs.mkdir(dir, { recursive: true });
await c.fs.writeFile(absolute, content);
return "";
}
const wantsJson = (flags: Flags, path: string | undefined) =>
Boolean(flags.json) || Boolean(path?.toLowerCase().endsWith(".json"));
const stringify = (flags: Flags, value: unknown) =>
flags.pretty ? JSON.stringify(value, null, 2) : JSON.stringify(value);
function scrapedContent(markdown: unknown): string[] {
if (typeof markdown !== "string" || !markdown) return [];
const indented = markdown
.split("\n")
.map(line => ` ${line}`)
.join("\n");
return ["", " --- Content ---", indented, " --- End Content ---"];
}
function readableSearch(data: any, labelWeb: boolean): string {
const lines: string[] = [];
if (data.web?.length) {
if (labelWeb) lines.push("=== Web Results ===", "");
for (const r of data.web) {
lines.push(r.title || "Untitled", ` URL: ${r.url}`);
if (r.description) lines.push(` ${r.description}`);
if (r.category) lines.push(` Category: ${r.category}`);
lines.push(...scrapedContent(r.markdown), "");
}
}
if (data.images?.length) {
if (lines.length) lines.push("");
lines.push("=== Image Results ===", "");
for (const r of data.images) {
lines.push(
r.title || "Untitled",
` Image URL: ${r.imageUrl}`,
` Source: ${r.url}`,
);
if (r.imageWidth && r.imageHeight) {
lines.push(` Size: ${r.imageWidth}x${r.imageHeight}`);
}
lines.push("");
}
}
if (data.news?.length) {
if (lines.length) lines.push("");
lines.push("=== News Results ===", "");
for (const r of data.news) {
lines.push(r.title || "Untitled", ` URL: ${r.url}`);
if (r.date) lines.push(` Date: ${r.date}`);
if (r.snippet) lines.push(` ${r.snippet}`);
lines.push(...scrapedContent(r.markdown), "");
}
}
return lines.join("\n");
}
async function search(c: Context, args: string[]): Promise<string> {
const { positional, flags } = parseArgs(args, SEARCH_FLAGS);
const query = positional.join(" ");
if (!query) throw new Error("missing required argument 'query'");
const named = list(text(flags, "sources"))?.map(s => s.toLowerCase());
const sources = named?.length ? [...new Set(named)] : ["web"];
for (const source of sources) {
if (!SEARCH_SOURCES.includes(source)) {
throw new Unsupported(
`source "${source}" isn't in the hosted shell, which has ${SEARCH_SOURCES.join(", ")}`,
);
}
}
const categories = list(text(flags, "categories"))?.map(c => c.toLowerCase());
for (const category of categories ?? []) {
if (category === "github") {
throw new Error(
"The github category was removed. Use --categories developer",
);
}
if (!SEARCH_CATEGORIES.includes(category)) {
throw new Error(
`Invalid category "${category}". Valid categories are: ${SEARCH_CATEGORIES.join(", ")}`,
);
}
}
const scrapeFormats = list(text(flags, "scrape-formats")) ?? ["markdown"];
const optional = (key: string, flag = key) =>
text(flags, flag) ? { [key]: text(flags, flag) } : {};
const envelope = await c.client.post(
"search",
{
query,
limit: integer(flags, "limit") ?? 5,
sources: sources.map(type => ({ type })),
// The CLI gets highlighted descriptions by default; ask for the same.
highlights: true,
...(categories ? { categories: categories.map(type => ({ type })) } : {}),
...optional("tbs"),
...optional("location"),
...optional("country"),
...(flags.scrape
? { scrapeOptions: { formats: scrapeFormats.map(type => ({ type })) } }
: {}),
},
c.signal,
);
const data = envelope.data ?? {};
const path = text(flags, "output");
if (!data.web?.length && !data.images?.length && !data.news?.length) {
return "No results found.\n";
}
if (flags.json || flags.pretty) {
const result = {
success: true,
data,
...(envelope.warning ? { warning: envelope.warning } : {}),
...(envelope.id ? { id: envelope.id } : {}),
...(envelope.creditsUsed !== undefined
? { creditsUsed: envelope.creditsUsed }
: {}),
};
return output(c, stringify(flags, result), path);
}
return output(c, readableSearch(data, sources.length > 1), path);
}
function scrapeBody(url: string, formats: string[], flags: Flags) {
const waitFor = integer(flags, "wait-for");
const maxAge = integer(flags, "max-age");
const includeTags = list(text(flags, "include-tags"));
const excludeTags = list(text(flags, "exclude-tags"));
const country = text(flags, "country");
return {
url,
formats,
...(flags["full-page"]
? { onlyMainContent: false }
: flags["only-main-content"]
? { onlyMainContent: true }
: {}),
...(waitFor !== undefined ? { waitFor } : {}),
...(includeTags ? { includeTags } : {}),
...(excludeTags ? { excludeTags } : {}),
...(maxAge !== undefined ? { maxAge } : {}),
...(country ? { location: { country } } : {}),
};
}
/** A single text format prints as raw content, as the CLI's extractContent does. */
function rawContent(data: any, format: string): string | null {
const value =
format === "html" || format === "rawHtml"
? data.html || data.rawHtml
: data[format];
if (Array.isArray(value)) return value.join("\n");
return typeof value === "string" && value ? value : null;
}
/** The CLI's parseFormats: comma-separated, any case, in the API's casing. */
function parseFormats(value: string): string[] {
const formats = value
.split(",")
.map(format => format.trim())
.filter(Boolean)
.map(format => {
const known = CLI_FORMATS.find(
name => name.toLowerCase() === format.toLowerCase(),
);
if (!known) {
throw new Error(
`Invalid format(s): ${format}. Valid formats are: ${CLI_FORMATS.join(", ")}`,
);
}
return known;
});
return [...new Set(formats)];
}
function isFormatList(value: string): boolean {
try {
return parseFormats(value).length > 0;
} catch {
return false;
}
}
const normalizeUrl = (url: string) =>
/^https?:\/\//i.test(url) ? url : `https://${url}`;
/** The CLI's isScrapeUrl: an http(s) URL, or a host with a dot, localhost or an IPv6 literal. */
function isScrapeUrl(value: string): boolean {
if (!/^https?:\/\//i.test(value)) {
const host = value.split(/[/?#]/, 1)[0];
const hostLike =
host.includes(".") ||
/^localhost(?::\d+)?$/i.test(host) ||
host.startsWith("[");
if (!hostLike) return false;
}
try {
return Boolean(new URL(normalizeUrl(value)).hostname);
} catch {
return false;
}
}
/** The CLI's isUrl, which decides when `firecrawl <url>` means scrape. */
function isShorthandUrl(value: string): boolean {
if (/^https?:\/\//i.test(value)) return true;
return (
value.includes(".") &&
!value.startsWith("-") &&
/^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}(\/.*)?$/.test(
value,
)
);
}
function urlToFilename(url: string): string {
try {
const parsed = new URL(url);
const host = parsed.hostname.replace(/^www\./, "");
const path = parsed.pathname.replace(/^\/|\/$/g, "").replace(/\//g, "-");
return path ? `${host}-${path}.md` : `${host}.md`;
} catch {
return `${url.replace(/[^a-zA-Z0-9.-]/g, "_")}.md`;
}
}
async function scrape(c: Context, args: string[]): Promise<string> {
const { positional, flags } = parseArgs(args, SCRAPE_FLAGS);
// As in the CLI, each word is a URL or formats: `firecrawl scrape example.com markdown,links`.
const urls: string[] = [];
const positionalFormats: string[] = [];
for (const word of [
...positional,
...(text(flags, "url") ? [text(flags, "url") as string] : []),
]) {
if (isScrapeUrl(word)) {
const url = normalizeUrl(word);
if (!urls.includes(url)) urls.push(url);
} else if (isFormatList(word))
positionalFormats.push(...parseFormats(word));
else {
throw new Error(
`${JSON.stringify(word)} isn't a URL. For a website, use an http(s) URL, for example https://example.com`,
);
}
}
if (!urls.length) {
throw new Error(
"Provide a URL to scrape, e.g. firecrawl scrape https://example.com",
);
}
if (flags["only-main-content"] && flags["full-page"]) {
throw new Error("Use either --only-main-content or --full-page, not both");
}
const formats = positionalFormats.length
? [...new Set(positionalFormats)]
: flags.html
? ["html"]
: flags.summary
? ["summary"]
: text(flags, "format")
? parseFormats(text(flags, "format") as string)
: ["markdown"];
for (const format of formats) {
if (!SCRAPE_FORMATS.includes(format)) {
throw new Unsupported(
`format "${format}" isn't in the hosted shell, which has ${SCRAPE_FORMATS.join(", ")}`,
);
}
}
if (urls.length > 1) {
c.stderr.push(`Scraping ${urls.length} URLs...`);
let done = 0;
let failed = 0;
await Promise.all(
urls.map(async url => {
try {
const body = scrapeBody(url, formats, flags);
const data = (await c.client.post("scrape", body, c.signal)).data;
const path = `.firecrawl/${urlToFilename(url)}`;
// As the CLI does: markdown, or the whole response, whatever the format.
await output(c, data?.markdown || JSON.stringify(data ?? {}), path);
c.stderr.push(`[${++done}/${urls.length}] Saved: ${path}`);
} catch (error) {
failed++;
c.stderr.push(
`[${++done}/${urls.length}] Error: ${url} - ${(error as Error).message}`,
);
}
}),
);
c.stderr.push(
"",
`Completed: ${urls.length - failed}/${urls.length} succeeded${failed ? `, ${failed} failed` : ""}`,
);
if (failed === urls.length) throw new Error("every URL failed");
return "";
}
const body = scrapeBody(urls[0], formats, flags);
const data = (await c.client.post("scrape", body, c.signal)).data ?? {};
const path = text(flags, "output");
if (wantsJson(flags, path)) return output(c, stringify(flags, data), path);
const raw = formats.length === 1 ? rawContent(data, formats[0]) : null;
if (raw !== null) return output(c, raw, path);
const picked = Object.fromEntries(
[...formats, "metadata"]
.filter(key => data[key] !== undefined)
.map(key => [key, data[key]]),
);
return output(
c,
stringify(flags, formats.length === 1 ? data : picked),
path,
);
}
export function firecrawlCommand(client: FirecrawlClient) {
return defineCommand("firecrawl", async (argv, ctx) => {
// As in the CLI, `firecrawl <url>` is short for `firecrawl scrape <url>`.
const [name, ...rest] = isShorthandUrl(argv[0] ?? "")
? ["scrape", ...argv]
: argv;
const stderr: string[] = [];
const result = (stdout: string, exitCode: number) => ({
stdout,
stdoutKind: "text" as const,
stderr: stderr.length ? `${stderr.join("\n")}\n` : "",
exitCode,
});
// A bare `help` is only the help command up front: `search help desk` is a query.
const end = rest.indexOf("--");
const options = end === -1 ? rest : rest.slice(0, end);
if ([name, ...options].some(arg => /^(--api-key|-k)(=|$)/.test(arg))) {
stderr.push(`Error: ${API_KEY_FLAG}`);
return result("", 1);
}
const asksHelp = options.some(arg => arg === "--help" || arg === "-h");
if (!name || ["help", "--help", "-h"].includes(name) || asksHelp) {
return result(FIRECRAWL_HELP, 0);
}
const hint = `Install the Firecrawl CLI for the rest: ${INSTALL} (see /docs/sdks/cli.md)`;
if (name !== "search" && name !== "scrape") {
stderr.push(
`firecrawl ${name} isn't in this hosted shell, which has search and scrape. ${hint}`,
);
return result("", 2);
}
const c: Context = {
client,
fs: ctx.fs,
cwd: ctx.cwd,
signal: ctx.signal,
stderr,
};
try {
return result(await (name === "search" ? search : scrape)(c, rest), 0);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (error instanceof Unsupported) {
stderr.push(`Error: ${message}. ${hint}`);
return result("", 2);
}
stderr.push(`Error: ${message}`);
return result("", 1);
}
});
}