Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { z } from "zod";
import { config } from "../../../../config";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
import { robustFetch } from "../../lib/fetch";
import { getInnerJson } from "@mendable/firecrawl-rs";
export async function scrapeURLWithPlaywright(
meta: Meta,
): Promise<EngineScrapeResult> {
const response = await robustFetch({
url: config.PLAYWRIGHT_MICROSERVICE_URL!,
headers: {
"Content-Type": "application/json",
},
body: {
url: meta.rewrittenUrl ?? meta.url,
wait_after_load: meta.options.waitFor,
timeout: meta.abort.scrapeTimeout(),
headers: meta.options.headers,
skip_tls_verification: meta.options.skipTlsVerification,
},
method: "POST",
logger: meta.logger.child("scrapeURLWithPlaywright/robustFetch"),
schema: z.object({
content: z.string(),
pageStatusCode: z.number(),
pageError: z.string().optional(),
contentType: z.string().optional(),
// Optional: older playwright-service builds don't send it.
url: z.string().optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
});
if (response.contentType?.includes("application/json")) {
response.content = await getInnerJson(response.content);
}
return {
// The landed URL reported by the service. Redirect threat and blocklist
// re-checks compare it against the requested URL. Falls back to the
// requested URL for older services or a non-http(s) value (about:blank).
url: landedHttpUrl(response.url) ?? meta.rewrittenUrl ?? meta.url,
html: response.content,
statusCode: response.pageStatusCode,
error: response.pageError,
contentType: response.contentType,
proxyUsed: "basic",
};
}
function landedHttpUrl(url: string | undefined): string | undefined {
if (!url) return undefined;
try {
const { protocol } = new URL(url);
return protocol === "http:" || protocol === "https:" ? url : undefined;
} catch {
return undefined;
}
}
export function playwrightMaxReasonableTime(meta: Meta): number {
return (meta.options.waitFor ?? 0) + 30000;
}