Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import crypto from "crypto";
import { config } from "../../../config";
import {
ALLOW_TEST_SUITE_WEBSITE,
HAS_FIRE_ENGINE,
TEST_PRODUCTION,
TEST_SELF_HOST,
TEST_SUITE_WEBSITE,
itIf,
} from "../lib";
import {
Identity,
idmux,
scrapeStopInteractiveBrowserRaw,
scrapeInteractRaw,
scrapeRaw,
scrapeTimeout,
} from "./lib";
describe("Scrape browser interact replay", () => {
let identity: Identity;
let otherIdentity: Identity;
beforeAll(async () => {
identity = await idmux({
name: "scrape-browser-replay",
concurrency: 20,
credits: 1_000_000,
});
otherIdentity = await idmux({
name: "scrape-browser-replay-other",
concurrency: 10,
credits: 1_000_000,
});
}, 10000 + scrapeTimeout);
const canRunReplayHappyPath =
!TEST_SELF_HOST &&
ALLOW_TEST_SUITE_WEBSITE &&
!!config.HANGAR_URL &&
(TEST_PRODUCTION || HAS_FIRE_ENGINE);
itIf(canRunReplayHappyPath)(
"replays scrape URL/waitFor/actions before interactive code runs",
async () => {
const marker = crypto.randomUUID();
const url = `${TEST_SUITE_WEBSITE}?testId=${crypto.randomUUID()}`;
let scrapeId: string | null = null;
try {
const scrapeResponse = await scrapeRaw(
{
url,
origin: "website-replay-test",
waitFor: 500,
actions: [
{
type: "executeJavascript",
script: `window.__firecrawlReplayMarker = "${marker}";`,
},
],
},
identity,
);
expect(scrapeResponse.statusCode).toBe(200);
expect(scrapeResponse.body.success).toBe(true);
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
scrapeId = scrapeResponse.body.scrape_id as string;
const executeResponse = await scrapeInteractRaw(
scrapeId,
{
language: "node",
timeout: 60,
code: `
const replayMarker = await page.evaluate(() => window.__firecrawlReplayMarker ?? null, undefined, undefined, false);
console.log(replayMarker ?? "missing-marker");
`,
},
identity,
);
expect(executeResponse.statusCode).toBe(200);
expect(executeResponse.body.success).toBe(true);
expect(executeResponse.body.stdout).toContain(marker);
expect(executeResponse.body.cdpUrl).toMatch(/^wss?:\/\//);
const readonlyView = new URL(executeResponse.body.liveViewUrl);
const interactiveView = new URL(
executeResponse.body.interactiveLiveViewUrl,
);
expect(readonlyView.pathname).toBe("/live");
expect(interactiveView.origin + interactiveView.pathname).toBe(
readonlyView.origin + readonlyView.pathname,
);
expect(readonlyView.hash).not.toBe("");
expect(interactiveView.hash).not.toBe("");
expect(interactiveView.hash).not.toBe(readonlyView.hash);
} finally {
if (scrapeId) {
await scrapeStopInteractiveBrowserRaw(scrapeId, identity);
}
}
},
scrapeTimeout,
);
itIf(canRunReplayHappyPath)(
"keeps a non-blank replay tab in the foreground for follow-up execs",
async () => {
const url = `${TEST_SUITE_WEBSITE}?testId=${crypto.randomUUID()}`;
let scrapeId: string | null = null;
try {
const scrapeResponse = await scrapeRaw(
{
url,
origin: "website-replay-test",
actions: [
{
type: "executeJavascript",
script: "window.open('about:blank', '_blank');",
},
],
},
identity,
);
expect(scrapeResponse.statusCode).toBe(200);
expect(scrapeResponse.body.success).toBe(true);
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
scrapeId = scrapeResponse.body.scrape_id as string;
const executeResponse = await scrapeInteractRaw(
scrapeId,
{
language: "node",
timeout: 60,
code: `
const visibleUrls = [];
for (const candidate of page.context().pages()) {
try {
const isVisible = await candidate.evaluate(
() => document.visibilityState === "visible",
);
if (isVisible) {
visibleUrls.push(candidate.url());
}
} catch {}
}
const visibleNonBlankUrl =
visibleUrls.find(value => value !== "about:blank") ?? "about:blank";
console.log(visibleNonBlankUrl);
`,
},
identity,
);
expect(executeResponse.statusCode).toBe(200);
expect(executeResponse.body.success).toBe(true);
const visibleUrl =
executeResponse.body.stdout
?.trim()
.split("\n")
.filter(Boolean)
.pop() ?? "";
expect(visibleUrl).not.toBe("about:blank");
expect(visibleUrl).toContain(TEST_SUITE_WEBSITE);
} finally {
if (scrapeId) {
await scrapeStopInteractiveBrowserRaw(scrapeId, identity);
}
}
},
scrapeTimeout,
);
itIf(canRunReplayHappyPath)(
"opens a single content tab (no stray blank tab) when a session starts",
async () => {
const url = `${TEST_SUITE_WEBSITE}?testId=${crypto.randomUUID()}`;
let scrapeId: string | null = null;
try {
const scrapeResponse = await scrapeRaw(
{
url,
origin: "website-replay-test",
},
identity,
);
expect(scrapeResponse.statusCode).toBe(200);
expect(scrapeResponse.body.success).toBe(true);
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
scrapeId = scrapeResponse.body.scrape_id as string;
// Session creation primes agent-browser and consolidates tabs, so
// user code must see exactly one tab: the content page.
const executeResponse = await scrapeInteractRaw(
scrapeId,
{
language: "node",
timeout: 60,
code: `
console.log(JSON.stringify(page.context().pages().map(p => p.url())));
`,
},
identity,
);
expect(executeResponse.statusCode).toBe(200);
expect(executeResponse.body.success).toBe(true);
const lastLine =
executeResponse.body.stdout
?.trim()
.split("\n")
.filter(Boolean)
.pop() ?? "[]";
const tabUrls = JSON.parse(lastLine) as string[];
expect(tabUrls).toHaveLength(1);
expect(tabUrls[0]).not.toBe("about:blank");
expect(tabUrls[0]).toContain(TEST_SUITE_WEBSITE);
} finally {
if (scrapeId) {
await scrapeStopInteractiveBrowserRaw(scrapeId, identity);
}
}
},
scrapeTimeout,
);
itIf(!TEST_SELF_HOST)(
"returns 400 for invalid scrape job id format",
async () => {
const response = await scrapeInteractRaw(
"not-a-valid-uuid",
{
code: "console.log('hi')",
language: "node",
},
identity,
);
expect(response.statusCode).toBe(400);
expect(response.body.success).toBe(false);
expect(response.body.error).toBe(
"Invalid job ID format. Job ID must be a valid UUID.",
);
},
);
itIf(!TEST_SELF_HOST)(
"returns 404 when scrape job does not exist",
async () => {
const response = await scrapeInteractRaw(
crypto.randomUUID(),
{
code: "console.log('hi')",
language: "node",
},
identity,
);
expect(response.statusCode).toBe(404);
expect(response.body.success).toBe(false);
expect(response.body.error).toBe("Job not found.");
},
);
itIf(ALLOW_TEST_SUITE_WEBSITE && !!config.IDMUX_URL)(
"returns 403 when scrape job belongs to another team",
async () => {
if (identity.teamId === otherIdentity.teamId) {
return;
}
const scrapeResponse = await scrapeRaw(
{
url: `${TEST_SUITE_WEBSITE}?testId=${crypto.randomUUID()}`,
origin: "website-replay-test",
},
identity,
);
expect(scrapeResponse.statusCode).toBe(200);
expect(scrapeResponse.body.success).toBe(true);
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
const scrapeId = scrapeResponse.body.scrape_id as string;
const executeResponse = await scrapeInteractRaw(
scrapeId,
{
code: "console.log('should fail')",
language: "node",
},
otherIdentity,
);
expect(executeResponse.statusCode).toBe(403);
expect(executeResponse.body.success).toBe(false);
expect(executeResponse.body.error).toBe("Forbidden.");
},
scrapeTimeout,
);
itIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)(
"returns replay-context error when scrape data is not retained",
async () => {
const scrapeResponse = await scrapeRaw(
{
url: `${TEST_SUITE_WEBSITE}?testId=${crypto.randomUUID()}`,
origin: "website-replay-test",
zeroDataRetention: true,
},
identity,
);
expect(scrapeResponse.statusCode).toBe(200);
expect(scrapeResponse.body.success).toBe(true);
expect(typeof scrapeResponse.body.scrape_id).toBe("string");
const scrapeId = scrapeResponse.body.scrape_id as string;
const executeResponse = await scrapeInteractRaw(
scrapeId,
{
code: "console.log('should not run')",
language: "node",
},
identity,
);
expect(executeResponse.statusCode).toBe(409);
expect(executeResponse.body.success).toBe(false);
expect(executeResponse.body.error).toContain(
"Replay context is unavailable",
);
},
scrapeTimeout,
);
});