Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import request from "supertest";
import { getJobFromGCS } from "../../../lib/gcs-jobs";
import {
scrape,
scrapeRaw,
crawl,
batchScrape,
scrapeStatusRaw,
zdrcleaner,
idmux,
searchRaw,
Identity,
} from "./lib";
import {
describeIf,
itIf,
concurrentIf,
TEST_API_URL,
TEST_PRODUCTION,
TEST_SUITE_WEBSITE,
HAS_PRODUCT_SERVICE,
scrapeTimeout,
} from "../lib";
import {
getLogs,
expectScrapeIsCleanedUp,
expectCrawlIsCleanedUp,
expectScrapesOfRequestAreCleanedUp,
expectScrapesAreFullyCleanedAfterZDRCleaner,
expectBatchScrapeIsCleanedUp,
} from "../zdr-helpers";
import { HAS_JOB_LOG, waitForJobLogRows } from "../job-log";
describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
const scopes = ["Team-scoped", "Request-scoped"] as const;
describeIf(HAS_JOB_LOG).each(scopes)("%s", scope => {
it("should clean up a scrape immediately", async () => {
let identity = await idmux({
name: `zdr/${scope}/scrape`,
credits: 10000,
flags: {
scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",
},
});
const testId = crypto.randomUUID();
const scrape1 = await scrape(
{
url: "https://firecrawl.dev/?test=" + testId,
zeroDataRetention: scope === "Request-scoped" ? true : undefined,
},
identity,
);
const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);
expect(gcsJob).toBeNull();
await expectScrapeIsCleanedUp(scrape1.metadata.scrapeId!);
if (scope === "Request-scoped") {
const status = await scrapeStatusRaw(
scrape1.metadata.scrapeId!,
identity,
);
expect(status.statusCode).toBe(404);
}
}, 60000);
it(
"should keep no error text for a failed scrape",
async () => {
const identity = await idmux({
name: `zdr/${scope}/failed-scrape`,
credits: 10000,
flags: {
scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",
},
});
// An unresolvable host fails the scrape with an error naming the host.
const hostname = `zdr-error-${crypto.randomUUID()}.invalid`;
const response = await scrapeRaw(
{
url: `https://${hostname}/`,
zeroDataRetention: scope === "Request-scoped" ? true : undefined,
},
identity,
);
expect(response.body.success).toBe(false);
const rows = await waitForJobLogRows(
"scrapes",
"team_id = {teamId: UUID}",
{ teamId: identity.teamId! },
);
expect(rows).toHaveLength(1);
expect(rows[0].is_successful).toBe(false);
expect(rows[0].error).toBeTruthy();
expect(String(rows[0].error)).not.toContain(hostname);
await expectScrapeIsCleanedUp(String(rows[0].id));
},
scrapeTimeout,
);
it(
"should clean up a crawl",
async () => {
const preLogs = await getLogs();
let identity = await idmux({
name: `zdr/${scope}/crawl`,
credits: 10000,
flags: {
scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",
},
});
const crawl1 = await crawl(
{
url: "https://firecrawl.dev",
limit: 10,
zeroDataRetention: scope === "Request-scoped" ? true : undefined,
},
identity,
);
const postLogs = (await getLogs()).slice(preLogs.length);
if (postLogs.length > 0) {
console.warn("Logs changed during crawl", postLogs);
}
expect(postLogs).toHaveLength(0);
// wait 20 seconds for crawl finish cron to fire
await new Promise(resolve => setTimeout(resolve, 20000));
await expectCrawlIsCleanedUp(crawl1.id);
const scrapes = await expectScrapesOfRequestAreCleanedUp(crawl1.id);
await zdrcleaner(identity.teamId!);
await expectScrapesAreFullyCleanedAfterZDRCleaner(
scrapes,
scope,
identity,
scrapeStatusRaw,
);
},
600000 + 20000,
);
it(
"should clean up a batch scrape",
async () => {
const preLogs = await getLogs();
let identity = await idmux({
name: `zdr/${scope}/batch-scrape`,
credits: 10000,
flags: {
scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",
},
});
const crawl1 = await batchScrape(
{
urls: ["https://firecrawl.dev", "https://mendable.ai"],
zeroDataRetention: scope === "Request-scoped" ? true : undefined,
},
identity,
);
const postLogs = (await getLogs()).slice(preLogs.length);
if (postLogs.length > 0) {
console.warn("Logs changed during batch scrape", postLogs);
}
expect(postLogs).toHaveLength(0);
// wait 20 seconds for batch scrape finish cron to fire
await new Promise(resolve => setTimeout(resolve, 20000));
await expectBatchScrapeIsCleanedUp(crawl1.id);
const scrapes = await expectScrapesOfRequestAreCleanedUp(crawl1.id, 2);
await zdrcleaner(identity.teamId!);
await expectScrapesAreFullyCleanedAfterZDRCleaner(
scrapes,
scope,
identity,
scrapeStatusRaw,
);
},
600000 + 20000,
);
});
itIf(HAS_JOB_LOG)(
"should allow screenshots and clean them up",
async () => {
const identity = await idmux({
name: "zdr/screenshot",
credits: 10000,
flags: { scrapeZDR: "allowed" },
});
const scrape1 = await scrape(
{
url: "https://firecrawl.dev",
formats: ["screenshot"],
zeroDataRetention: true,
},
identity,
);
expect(scrape1.screenshot).toBeDefined();
const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);
expect(gcsJob).toBeNull();
await expectScrapeIsCleanedUp(scrape1.metadata.scrapeId!);
},
scrapeTimeout,
);
itIf(HAS_JOB_LOG)(
"should allow pdf actions and clean them up",
async () => {
const identity = await idmux({
name: "zdr/pdf-action",
credits: 10000,
flags: { scrapeZDR: "allowed" },
});
const scrape1 = await scrape(
{
url: "https://firecrawl.dev",
actions: [{ type: "pdf" }],
zeroDataRetention: true,
},
identity,
);
expect(scrape1.actions?.pdfs).toBeDefined();
expect(scrape1.actions?.pdfs?.length).toBe(1);
const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!);
expect(gcsJob).toBeNull();
await expectScrapeIsCleanedUp(scrape1.metadata.scrapeId!);
},
scrapeTimeout,
);
it("should allow search when searchZDR is forced", async () => {
const identity = await idmux({
name: "zdr/search-forced",
credits: 10000,
flags: { searchZDR: "forced" },
});
const response = await searchRaw(
{ query: "firecrawl", limit: 1 },
identity,
);
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
}, 60000);
it("should refuse data source discovery when searchZDR is forced", async () => {
const identity = await idmux({
name: "zdr/discover-searchZDR",
credits: 100,
flags: { searchZDR: "forced-zdr" },
});
const response = await request(TEST_API_URL)
.get("/exchange/discover")
.query({ q: "company funding rounds" })
.set("Authorization", `Bearer ${identity.apiKey}`);
expect(response.statusCode).toBe(403);
expect(response.body.error).toBe(
"Provider discovery does not support zero data retention.",
);
}, 60000);
describe.each(scopes)("%s audio, video and product formats", scope => {
let identity: Identity;
beforeAll(async () => {
identity = await idmux({
name: `zdr/${scope}/formats`,
credits: 10000,
flags: {
scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",
},
});
}, scrapeTimeout);
const zeroDataRetention = scope === "Request-scoped" ? true : undefined;
it.concurrent(
"refuses the audio format without attempting the download",
async () => {
// Not a supported media URL: a download attempt would fail the scrape.
const response = await scrape(
{
url: "https://firecrawl.dev",
formats: ["markdown", "audio"],
zeroDataRetention,
},
identity,
);
expect(response.markdown).toBeDefined();
expect(response.audio).toBeUndefined();
expect(response.warning).toContain(
"Audio format is not supported with zero data retention.",
);
},
scrapeTimeout,
);
it.concurrent(
"refuses the video format without attempting the download",
async () => {
const response = await scrape(
{
url: "https://firecrawl.dev",
formats: ["markdown", "video"],
zeroDataRetention,
},
identity,
);
expect(response.markdown).toBeDefined();
expect(response.video).toBeUndefined();
expect(response.videos).toBeUndefined();
expect(response.warning).toContain(
"Video format is not supported with zero data retention.",
);
},
scrapeTimeout,
);
it.concurrent(
"refuses the product format",
async () => {
const response = await scrape(
{
url: `${TEST_SUITE_WEBSITE}/product`,
formats: ["markdown", { type: "product" }],
zeroDataRetention,
},
identity,
);
expect(response.markdown).toBeDefined();
expect(response.product).toBeUndefined();
expect(response.warning).toContain(
"Product format is not supported with zero data retention.",
);
},
scrapeTimeout,
);
});
describe("audio, video and product formats without zero data retention on the request", () => {
let identity: Identity;
beforeAll(async () => {
identity = await idmux({
name: "zdr/allowed/formats-without-zdr",
credits: 10000,
flags: { scrapeZDR: "allowed" },
});
}, scrapeTimeout);
it.concurrent(
"still returns audio",
async () => {
const response = await scrape(
{
url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
formats: ["audio"],
},
identity,
);
expect(response.audio).toMatch(/^https:\/\//);
},
scrapeTimeout,
);
it.concurrent(
"still returns video",
async () => {
const response = await scrape(
{
url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
formats: ["video"],
},
identity,
);
expect(response.video).toMatch(/^https:\/\//);
},
scrapeTimeout * 2,
);
concurrentIf(HAS_PRODUCT_SERVICE)(
"still returns the product",
async () => {
const response = await scrape(
{
url: `${TEST_SUITE_WEBSITE}/product`,
formats: [{ type: "product" }],
},
identity,
);
expect(response.product?.title).toBe("Firecrawl Test Widget");
},
scrapeTimeout,
);
});
});