Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import request from "supertest";
import { config } from "../../config";
import { v7 as uuidv7 } from "uuid";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";
// const TEST_URL = 'http://localhost:3002'
const TEST_URL = "http://127.0.0.1:3002";
describe("E2E Tests for API Routes", () => {
let originalUseDbAuth: boolean | undefined;
beforeAll(() => {
originalUseDbAuth = config.USE_DB_AUTHENTICATION;
config.USE_DB_AUTHENTICATION = true;
});
afterAll(() => {
config.USE_DB_AUTHENTICATION = originalUseDbAuth;
});
describe("GET /e2e-test", () => {
it.concurrent("should return OK message", async () => {
const response = await request(TEST_URL).get("/e2e-test");
expect(response.statusCode).toBe(200);
expect(response.text).toContain("OK");
});
});
describe("POST /v0/scrape", () => {
it.concurrent("should require authorization", async () => {
const response = await request(TEST_URL).post("/v0/scrape");
expect(response.statusCode).toBe(401);
});
it.concurrent(
"should return an error response with an invalid API key",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401);
},
);
it.concurrent("should return an error for a unsupported URL", async () => {
const unsupportedUrl = "https://facebook.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: unsupportedUrl });
expect(response.statusCode).toBe(403);
expect(response.body.error).toContain(UNSUPPORTED_SITE_MESSAGE);
});
it.concurrent(
"should return a successful response with a valid API key",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl-test-site.vercel.app" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("markdown");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data).not.toHaveProperty("html");
expect(response.body.data.content).toContain("Firecrawl Test Site");
expect(response.body.data.metadata).toHaveProperty("title");
expect(response.body.data.metadata).toHaveProperty("description");
expect(response.body.data.metadata).toHaveProperty("ogTitle");
expect(response.body.data.metadata).toHaveProperty("ogDescription");
expect(response.body.data.metadata).toHaveProperty("ogUrl");
expect(response.body.data.metadata).toHaveProperty("ogImage");
expect(response.body.data.metadata).toHaveProperty("ogLocaleAlternate");
expect(response.body.data.metadata).toHaveProperty("sourceURL");
expect(response.body.data.metadata).toHaveProperty("pageStatusCode");
expect(response.body.data.metadata.pageError).toBeUndefined();
expect(response.body.data.metadata.title).toBe(
"Firecrawl Test Website",
);
expect(response.body.data.metadata.description).toBe(
"Welcome to the Firecrawl Test Website!",
);
expect(response.body.data.metadata.ogTitle).toBe(
"Firecrawl Test Website",
);
expect(response.body.data.metadata.ogDescription).toBe(
"Welcome to the Firecrawl Test Website!",
);
expect(response.body.data.metadata.ogUrl).toContain(
"firecrawl-test-site",
);
expect(response.body.data.metadata.ogImage).toContain(
"firecrawl-test-site",
);
expect(response.body.data.metadata.ogLocaleAlternate).toStrictEqual([]);
expect(response.body.data.metadata.sourceURL).toBe(
"https://firecrawl-test-site.vercel.app",
);
expect(response.body.data.metadata.pageStatusCode).toBe(200);
},
30000,
); // 30 seconds timeout
it.concurrent(
"should return a successful response with a valid API key and includeHtml set to true",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl-test-site.vercel.app",
pageOptions: { includeHtml: true },
});
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("markdown");
expect(response.body.data).toHaveProperty("html");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.content).toContain("Firecrawl Test Site");
expect(response.body.data.markdown).toContain("Firecrawl Test Site");
expect(response.body.data.html).toContain("<h1");
expect(response.body.data.metadata.pageStatusCode).toBe(200);
expect(response.body.data.metadata.pageError).toBeUndefined();
},
30000,
); // 30 seconds timeout
it.concurrent(
"should return a successful response with a valid API key and includeRawHtml set to true",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl-test-site.vercel.app",
pageOptions: { includeRawHtml: true },
});
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("markdown");
expect(response.body.data).toHaveProperty("rawHtml");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.content).toContain("Firecrawl Test Site");
expect(response.body.data.markdown).toContain("Firecrawl Test Site");
expect(response.body.data.rawHtml).toContain("<h1");
expect(response.body.data.metadata.pageStatusCode).toBe(200);
expect(response.body.data.metadata.pageError).toBeUndefined();
},
30000,
); // 30 seconds timeout
it.concurrent(
"should return a successful response for a valid scrape with PDF file",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://arxiv.org/pdf/astro-ph/9301001.pdf" });
await new Promise(r => setTimeout(r, 6000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.content).toContain(
"We present spectrophotometric observations of the Broad Line Radio Galaxy",
);
expect(response.body.data.metadata.pageStatusCode).toBe(200);
expect(response.body.data.metadata.pageError).toBeUndefined();
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response for a valid scrape with PDF file without explicit .pdf extension",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://arxiv.org/pdf/astro-ph/9301001" });
await new Promise(r => setTimeout(r, 6000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.content).toContain(
"We present spectrophotometric observations of the Broad Line Radio Galaxy",
);
expect(response.body.data.metadata.pageStatusCode).toBe(200);
expect(response.body.data.metadata.pageError).toBeUndefined();
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response for a valid scrape with PDF file and parsePDF set to false",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://arxiv.org/pdf/astro-ph/9301001.pdf",
pageOptions: { parsePDF: false },
});
await new Promise(r => setTimeout(r, 6000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.content).toContain(
"/Title(arXiv:astro-ph/9301001v1 7 Jan 1993)>>endobj",
);
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response with a valid API key with removeTags option",
async () => {
const responseWithoutRemoveTags = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com/" });
expect(responseWithoutRemoveTags.statusCode).toBe(200);
expect(responseWithoutRemoveTags.body).toHaveProperty("data");
expect(responseWithoutRemoveTags.body.data).toHaveProperty("content");
expect(responseWithoutRemoveTags.body.data).toHaveProperty("markdown");
expect(responseWithoutRemoveTags.body.data).toHaveProperty("metadata");
expect(responseWithoutRemoveTags.body.data).not.toHaveProperty("html");
expect(responseWithoutRemoveTags.body.data.content).toContain(
"Scrape This Site",
);
expect(responseWithoutRemoveTags.body.data.content).toContain(
"Lessons and Videos",
); // #footer
expect(responseWithoutRemoveTags.body.data.content).toContain(
"[Sandbox](",
); // .nav
expect(responseWithoutRemoveTags.body.data.content).toContain(
"web scraping",
); // strong
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com/",
pageOptions: { removeTags: [".nav", "#footer", "strong"] },
});
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("markdown");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data).not.toHaveProperty("html");
expect(response.body.data.content).toContain("Scrape This Site");
expect(response.body.data.content).not.toContain("Lessons and Videos"); // #footer
expect(response.body.data.content).not.toContain("[Sandbox]("); // .nav
expect(response.body.data.content).not.toContain("web scraping"); // strong
},
30000,
); // 30 seconds timeout
// TODO: add this test back once we nail the waitFor option to be more deterministic
// it.concurrent("should return a successful response with a valid API key and waitFor option", async () => {
// const startTime = Date.now();
// const response = await request(TEST_URL)
// .post("/v0/scrape")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://firecrawl.dev", pageOptions: { waitFor: 7000 } });
// const endTime = Date.now();
// const duration = endTime - startTime;
// expect(response.statusCode).toBe(200);
// expect(response.body).toHaveProperty("data");
// expect(response.body.data).toHaveProperty("content");
// expect(response.body.data).toHaveProperty("markdown");
// expect(response.body.data).toHaveProperty("metadata");
// expect(response.body.data).not.toHaveProperty("html");
// expect(response.body.data.content).toContain("🔥 Firecrawl");
// expect(duration).toBeGreaterThanOrEqual(7000);
// }, 12000); // 12 seconds timeout
it.concurrent(
"should return a successful response for a scrape with 400 page",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/400" });
await new Promise(r => setTimeout(r, 5000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.metadata.pageStatusCode).toBe(400);
expect(response.body.data.metadata.pageError.toLowerCase()).toContain(
"bad request",
);
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response for a scrape with 401 page",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/401" });
await new Promise(r => setTimeout(r, 5000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.metadata.pageStatusCode).toBe(401);
expect(response.body.data.metadata.pageError.toLowerCase()).toContain(
"unauthorized",
);
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response for a scrape with 403 page",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/403" });
await new Promise(r => setTimeout(r, 5000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.metadata.pageStatusCode).toBe(403);
expect(response.body.data.metadata.pageError.toLowerCase()).toContain(
"forbidden",
);
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response for a scrape with 404 page",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/404" });
await new Promise(r => setTimeout(r, 5000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.metadata.pageStatusCode).toBe(404);
expect(response.body.data.metadata.pageError.toLowerCase()).toContain(
"not found",
);
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response for a scrape with 405 page",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/405" });
await new Promise(r => setTimeout(r, 5000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.metadata.pageStatusCode).toBe(405);
expect(response.body.data.metadata.pageError.toLowerCase()).toContain(
"method not allowed",
);
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response for a scrape with 500 page",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://httpstat.us/500" });
await new Promise(r => setTimeout(r, 5000));
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("data");
expect(response.body.data).toHaveProperty("content");
expect(response.body.data).toHaveProperty("metadata");
expect(response.body.data.metadata.pageStatusCode).toBe(500);
expect(response.body.data.metadata.pageError.toLowerCase()).toContain(
"internal server error",
);
},
60000,
); // 60 seconds
});
describe("POST /v0/crawl", () => {
it.concurrent("should require authorization", async () => {
const response = await request(TEST_URL).post("/v0/crawl");
expect(response.statusCode).toBe(401);
});
it.concurrent(
"should return an error response with an invalid API key",
async () => {
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401);
},
);
it.concurrent("should return an error for a unsupported URL", async () => {
const unsupportedUrl = "https://twitter.com/fake-test";
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: unsupportedUrl });
expect(response.statusCode).toBe(403);
expect(response.body.error).toContain(UNSUPPORTED_SITE_MESSAGE);
});
it.concurrent(
"should return a successful response with a valid API key for crawl",
async () => {
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("jobId");
expect(response.body.jobId).toMatch(
/^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/,
);
},
);
it.concurrent(
"should prevent duplicate requests using the same idempotency key",
async () => {
const uniqueIdempotencyKey = uuidv7();
// First request with the idempotency key
const firstResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.set("x-idempotency-key", uniqueIdempotencyKey)
.send({ url: "https://docs.firecrawl.dev" });
expect(firstResponse.statusCode).toBe(200);
// Second request with the same idempotency key
const secondResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.set("x-idempotency-key", uniqueIdempotencyKey)
.send({ url: "https://docs.firecrawl.dev" });
expect(secondResponse.statusCode).toBe(409);
expect(secondResponse.body.error).toBe("Idempotency key already used");
},
);
it.concurrent(
"should return a successful response with a valid API key and valid includes option",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
limit: 10,
crawlerOptions: {
includes: ["blog/*"],
},
});
let response;
let isFinished = false;
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
isFinished = response.body.status === "completed";
if (!isFinished) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const completedResponse = response;
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
);
expect(urls.length).toBeGreaterThan(5);
urls.forEach((url: string) => {
expect(url.startsWith("https://www.mendable.ai/blog/")).toBeTruthy();
});
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].content).toContain("Mendable");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(
200,
);
expect(
completedResponse.body.data[0].metadata.pageError,
).toBeUndefined();
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response with a valid API key and valid excludes option",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
limit: 10,
crawlerOptions: {
excludes: ["blog/*"],
},
});
let isFinished = false;
let response;
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
isFinished = response.body.status === "completed";
if (!isFinished) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const completedResponse = response;
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
);
expect(urls.length).toBeGreaterThan(5);
urls.forEach((url: string) => {
expect(url.startsWith("https://wwww.mendable.ai/blog/")).toBeFalsy();
});
},
90000,
); // 90 seconds
it.concurrent(
"should return a successful response with a valid API key and limit to 3",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
crawlerOptions: { limit: 3 },
});
let isFinished = false;
let response;
while (!isFinished) {
response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
isFinished = response.body.status === "completed";
if (!isFinished) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const completedResponse = response;
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data.length).toBe(3);
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].content).toContain("Mendable");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(
200,
);
expect(
completedResponse.body.data[0].metadata.pageError,
).toBeUndefined();
},
60000,
); // 60 seconds
it.concurrent(
"should return a successful response with max depth option for a valid crawl job",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com",
crawlerOptions: { maxDepth: 1 },
});
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
// wait for 60 seconds
let isCompleted = false;
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(
200,
);
expect(
completedResponse.body.data[0].metadata.pageError,
).toBeUndefined();
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
);
expect(urls.length).toBeGreaterThan(1);
// Check if all URLs have a maximum depth of 1
urls.forEach((url: string) => {
const pathSplits = new URL(url).pathname.split("/");
const depth =
pathSplits.length -
(pathSplits[0].length === 0 &&
pathSplits[pathSplits.length - 1].length === 0
? 1
: 0);
expect(depth).toBeLessThanOrEqual(2);
});
},
180000,
);
it.concurrent(
"should return a successful response with relative max depth option for a valid crawl job",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.scrapethissite.com/pages/",
crawlerOptions: { maxDepth: 1 },
});
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
// wait for 60 seconds
let isCompleted = false;
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
);
expect(urls.length).toBeGreaterThan(1);
// Check if all URLs have an absolute maximum depth of 3 after the base URL depth was 2 and the maxDepth was 1
urls.forEach((url: string) => {
const pathSplits = new URL(url).pathname.split("/");
const depth =
pathSplits.length -
(pathSplits[0].length === 0 &&
pathSplits[pathSplits.length - 1].length === 0
? 1
: 0);
expect(depth).toBeLessThanOrEqual(3);
});
},
180000,
);
it.concurrent(
"should return a successful response with relative max depth option for a valid crawl job with maxDepths equals to zero",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://www.mendable.ai",
crawlerOptions: { maxDepth: 0 },
});
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
// wait for 60 seconds
let isCompleted = false;
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
const testurls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
);
//console.log(testurls)
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
const urls = completedResponse.body.data.map(
(item: any) => item.metadata?.sourceURL,
);
expect(urls.length).toBeGreaterThanOrEqual(1);
// Check if all URLs have an absolute maximum depth of 3 after the base URL depth was 2 and the maxDepth was 1
urls.forEach((url: string) => {
const pathSplits = new URL(url).pathname.split("/");
const depth =
pathSplits.length -
(pathSplits[0].length === 0 &&
pathSplits[pathSplits.length - 1].length === 0
? 1
: 0);
expect(depth).toBeLessThanOrEqual(1);
});
},
180000,
);
// it.concurrent("should return a successful response with a valid API key and valid limit option", async () => {
// const crawlResponse = await request(TEST_URL)
// .post("/v0/crawl")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({
// url: "https://mendable.ai",
// crawlerOptions: { limit: 10 },
// });
// const response = await request(TEST_URL)
// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
// expect(response.statusCode).toBe(200);
// expect(response.body).toHaveProperty("status");
// expect(response.body.status).toBe("active");
// let isCompleted = false;
// while (!isCompleted) {
// const statusCheckResponse = await request(TEST_URL)
// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
// expect(statusCheckResponse.statusCode).toBe(200);
// isCompleted = statusCheckResponse.body.status === "completed";
// if (!isCompleted) {
// await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
// }
// }
// const completedResponse = await request(TEST_URL)
// .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
// expect(completedResponse.statusCode).toBe(200);
// expect(completedResponse.body).toHaveProperty("status");
// expect(completedResponse.body.status).toBe("completed");
// expect(completedResponse.body).toHaveProperty("data");
// expect(completedResponse.body.data.length).toBe(10);
// expect(completedResponse.body.data[0]).toHaveProperty("content");
// expect(completedResponse.body.data[0]).toHaveProperty("markdown");
// expect(completedResponse.body.data[0]).toHaveProperty("metadata");
// expect(completedResponse.body.data[0].content).toContain("Mendable");
// expect(completedResponse.body.data[0].content).not.toContain("main menu");
// }, 60000); // 60 seconds
it.concurrent(
"should return a successful response for a valid crawl job with includeHtml set to true option",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl-test-site.vercel.app",
pageOptions: { includeHtml: true },
});
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
let isCompleted = false;
while (!isCompleted) {
const statusCheckResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusCheckResponse.statusCode).toBe(200);
isCompleted = statusCheckResponse.body.status === "completed";
if (!isCompleted) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(
200,
);
expect(
completedResponse.body.data[0].metadata.pageError,
).toBeUndefined();
// 120 seconds
expect(completedResponse.body.data[0]).toHaveProperty("html");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].content).toContain(
"Firecrawl Test Site",
);
expect(completedResponse.body.data[0].markdown).toContain(
"Firecrawl Test Site",
);
expect(completedResponse.body.data[0].html).toContain("<h1");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(
200,
);
expect(
completedResponse.body.data[0].metadata.pageError,
).toBeUndefined();
},
180000,
);
it.concurrent(
"should crawl external content links when allowed",
async () => {
const crawlInitResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
crawlerOptions: {
allowExternalContentLinks: true,
ignoreSitemap: true,
returnOnlyUrls: true,
limit: 50,
},
});
expect(crawlInitResponse.statusCode).toBe(200);
expect(crawlInitResponse.body).toHaveProperty("jobId");
let crawlStatus: string = "scraping";
let crawlData = [];
while (crawlStatus !== "completed") {
const statusResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlInitResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
crawlStatus = statusResponse.body.status;
if (statusResponse.body.data) {
crawlData = statusResponse.body.data;
}
if (crawlStatus !== "completed") {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
expect(crawlData.length).toBeGreaterThan(0);
expect(crawlData).toEqual(
expect.arrayContaining([
expect.objectContaining({
url: expect.stringContaining(
"https://firecrawl.dev/?ref=mendable+banner",
),
}),
expect.objectContaining({
url: expect.stringContaining("https://mendable.ai/pricing"),
}),
expect.objectContaining({
url: expect.stringContaining("https://x.com/CalebPeffer"),
}),
]),
);
},
180000,
); // 3 minutes timeout
});
describe("POST /v0/crawlWebsitePreview", () => {
it.concurrent("should require authorization", async () => {
const response = await request(TEST_URL).post("/v0/crawlWebsitePreview");
expect(response.statusCode).toBe(401);
});
it.concurrent(
"should return an error response with an invalid API key",
async () => {
const response = await request(TEST_URL)
.post("/v0/crawlWebsitePreview")
.set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(401);
},
);
// it.concurrent("should return an error for a unsupported URL", async () => {
// const unsupportedUrl = "https://instagram.com/fake-test";
// const response = await request(TEST_URL)
// .post("/v0/crawlWebsitePreview")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: unsupportedUrl });
// // is returning 429 instead of 403
// expect(response.statusCode).toBe(403);
// expect(response.body.error).toContain("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.");
// });
it.concurrent(
"should return a timeout error when scraping takes longer than the specified timeout",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev", timeout: 1000 });
expect(response.statusCode).toBe(408);
},
3000,
);
});
describe("POST /v0/search", () => {
it.concurrent("should require authorization", async () => {
const response = await request(TEST_URL).post("/v0/search");
expect(response.statusCode).toBe(401);
});
it.concurrent(
"should return an error response with an invalid API key",
async () => {
const response = await request(TEST_URL)
.post("/v0/search")
.set("Authorization", `Bearer invalid-api-key`)
.set("Content-Type", "application/json")
.send({ query: "test" });
expect(response.statusCode).toBe(401);
},
);
it.concurrent(
"should return a successful response with a valid API key for search",
async () => {
const response = await request(TEST_URL)
.post("/v0/search")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ query: "test" });
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("success");
expect(response.body.success).toBe(true);
expect(response.body).toHaveProperty("data");
},
30000,
); // 30 seconds timeout
});
describe("GET /v0/crawl/status/:jobId", () => {
it.concurrent("should require authorization", async () => {
const response = await request(TEST_URL).get("/v0/crawl/status/123");
expect(response.statusCode).toBe(401);
});
it.concurrent(
"should return an error response with an invalid API key",
async () => {
const response = await request(TEST_URL)
.get("/v0/crawl/status/123")
.set("Authorization", `Bearer invalid-api-key`);
expect(response.statusCode).toBe(401);
},
);
it.concurrent(
"should return Job not found for invalid job ID",
async () => {
const response = await request(TEST_URL)
.get("/v0/crawl/status/invalidJobId")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(404);
},
);
it.concurrent(
"should return a successful crawl status response for a valid crawl job",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://mendable.ai/blog" });
expect(crawlResponse.statusCode).toBe(200);
let isCompleted = false;
let completedResponse;
while (!isCompleted) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
if (response.body.status === "completed") {
isCompleted = true;
completedResponse = response;
} else {
await new Promise(r => setTimeout(r, 1000)); // Wait for 1 second before checking again
}
}
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].content).toContain("Mendable");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(
200,
);
expect(
completedResponse.body.data[0].metadata.pageError,
).toBeUndefined();
const childrenLinks = completedResponse.body.data.filter(
doc =>
doc.metadata &&
doc.metadata.sourceURL &&
doc.metadata.sourceURL.includes("mendable.ai/blog"),
);
expect(childrenLinks.length).toBe(completedResponse.body.data.length);
},
180000,
); // 120 seconds
it.concurrent(
"should return a successful response for a valid crawl job with PDF files without explicit .pdf extension ",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://arxiv.org/pdf/astro-ph/9301001",
crawlerOptions: {
limit: 10,
excludes: [
"list/*",
"login",
"abs/*",
"static/*",
"about/*",
"archive/*",
],
},
});
expect(crawlResponse.statusCode).toBe(200);
let isCompleted = false;
let completedResponse;
while (!isCompleted) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
if (response.body.status === "completed") {
isCompleted = true;
completedResponse = response;
} else {
await new Promise(r => setTimeout(r, 1000)); // Wait for 1 second before checking again
}
}
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data.length).toEqual(1);
expect(completedResponse.body.data).toEqual(
expect.arrayContaining([
expect.objectContaining({
content: expect.stringContaining(
"asymmetries might represent, for instance, preferred source orientations to our line of sight.",
),
}),
]),
);
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(
200,
);
expect(
completedResponse.body.data[0].metadata.pageError,
).toBeUndefined();
},
180000,
); // 120 seconds
it.concurrent(
"should return a successful response for a valid crawl job with includeHtml set to true option (2)",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://firecrawl-test-site.vercel.app",
pageOptions: { includeHtml: true },
});
expect(crawlResponse.statusCode).toBe(200);
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
expect(["active", "waiting"]).toContain(response.body.status);
let isFinished = false;
let completedResponse;
while (!isFinished) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
if (response.body.status === "completed") {
isFinished = true;
completedResponse = response;
} else {
await new Promise(r => setTimeout(r, 1000)); // Wait for 1 second before checking again
}
}
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0]).toHaveProperty("html");
expect(completedResponse.body.data[0].content).toContain(
"Firecrawl Test Site",
);
expect(completedResponse.body.data[0].markdown).toContain(
"Firecrawl Test Site",
);
expect(completedResponse.body.data[0].html).toContain("<h1");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(
200,
);
expect(
completedResponse.body.data[0].metadata.pageError,
).toBeUndefined();
},
60000,
);
}); // 60 seconds
it.concurrent(
"should return a successful response for a valid crawl job with allowBackwardCrawling set to true option",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai/blog",
pageOptions: { includeHtml: true },
crawlerOptions: { allowBackwardCrawling: true },
});
expect(crawlResponse.statusCode).toBe(200);
let isFinished = false;
let completedResponse;
while (!isFinished) {
const response = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("status");
if (response.body.status === "completed") {
isFinished = true;
completedResponse = response;
} else {
await new Promise(r => setTimeout(r, 1000)); // Wait for 1 second before checking again
}
}
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("completed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data[0]).toHaveProperty("content");
expect(completedResponse.body.data[0]).toHaveProperty("markdown");
expect(completedResponse.body.data[0]).toHaveProperty("metadata");
expect(completedResponse.body.data[0]).toHaveProperty("html");
expect(completedResponse.body.data[0].content).toContain("Mendable");
expect(completedResponse.body.data[0].markdown).toContain("Mendable");
expect(completedResponse.body.data[0].metadata.pageStatusCode).toBe(200);
expect(completedResponse.body.data[0].metadata.pageError).toBeUndefined();
const onlyChildrenLinks = completedResponse.body.data.filter(doc => {
return (
doc.metadata &&
doc.metadata.sourceURL &&
doc.metadata.sourceURL.includes("mendable.ai/blog")
);
});
expect(completedResponse.body.data.length).toBeGreaterThan(
onlyChildrenLinks.length,
);
},
60000,
);
it.concurrent(
"If someone cancels a crawl job, it should turn into failed status",
async () => {
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://jestjs.io" });
expect(crawlResponse.statusCode).toBe(200);
await new Promise(r => setTimeout(r, 20000));
const responseCancel = await request(TEST_URL)
.delete(`/v0/crawl/cancel/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(responseCancel.statusCode).toBe(200);
expect(responseCancel.body).toHaveProperty("status");
expect(responseCancel.body.status).toBe("cancelled");
await new Promise(r => setTimeout(r, 10000));
const completedResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(completedResponse.statusCode).toBe(200);
expect(completedResponse.body).toHaveProperty("status");
expect(completedResponse.body.status).toBe("failed");
expect(completedResponse.body).toHaveProperty("data");
expect(completedResponse.body.data).toBeNull();
expect(completedResponse.body).toHaveProperty("partial_data");
expect(completedResponse.body.partial_data[0]).toHaveProperty("content");
expect(completedResponse.body.partial_data[0]).toHaveProperty("markdown");
expect(completedResponse.body.partial_data[0]).toHaveProperty("metadata");
expect(
completedResponse.body.partial_data[0].metadata.pageStatusCode,
).toBe(200);
expect(
completedResponse.body.partial_data[0].metadata.pageError,
).toBeUndefined();
},
60000,
); // 60 seconds
describe("POST /v0/scrape with LLM Extraction", () => {
it.concurrent(
"should extract data using LLM extraction mode",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
pageOptions: {
onlyMainContent: true,
},
extractorOptions: {
mode: "llm-extraction",
extractionPrompt:
"Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
extractionSchema: {
type: "object",
properties: {
company_mission: {
type: "string",
},
supports_sso: {
type: "boolean",
},
is_open_source: {
type: "boolean",
},
},
required: ["company_mission", "supports_sso", "is_open_source"],
},
},
});
// Ensure that the job was successfully created before proceeding with LLM extraction
expect(response.statusCode).toBe(200);
// Assuming the LLM extraction object is available in the response body under `data.llm_extraction`
let llmExtraction = response.body.data.llm_extraction;
// Check if the llm_extraction object has the required properties with correct types and values
expect(llmExtraction).toHaveProperty("company_mission");
expect(typeof llmExtraction.company_mission).toBe("string");
expect(llmExtraction).toHaveProperty("supports_sso");
expect(llmExtraction.supports_sso).toBe(true);
expect(typeof llmExtraction.supports_sso).toBe("boolean");
expect(llmExtraction).toHaveProperty("is_open_source");
expect(llmExtraction.is_open_source).toBe(false);
expect(typeof llmExtraction.is_open_source).toBe("boolean");
},
60000,
); // 60 secs
it.concurrent(
"should extract data using LLM extraction mode with RawHtml",
async () => {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://mendable.ai",
extractorOptions: {
mode: "llm-extraction-from-raw-html",
extractionPrompt:
"Based on the information on the page, what are the primary and secondary CTA buttons?",
extractionSchema: {
type: "object",
properties: {
primary_cta: {
type: "string",
},
secondary_cta: {
type: "string",
},
},
required: ["primary_cta", "secondary_cta"],
},
},
});
// Ensure that the job was successfully created before proceeding with LLM extraction
expect(response.statusCode).toBe(200);
// Assuming the LLM extraction object is available in the response body under `data.llm_extraction`
let llmExtraction = response.body.data.llm_extraction;
// Check if the llm_extraction object has the required properties with correct types and values
expect(llmExtraction).toHaveProperty("primary_cta");
expect(typeof llmExtraction.primary_cta).toBe("string");
expect(llmExtraction).toHaveProperty("secondary_cta");
expect(typeof llmExtraction.secondary_cta).toBe("string");
},
60000,
); // 60 secs
});
// describe("POST /v0/scrape for Top 100 Companies", () => {
// it.concurrent("should extract data for the top 100 companies", async () => {
// const response = await request(TEST_URL)
// .post("/v0/scrape")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({
// url: "https://companiesmarketcap.com/",
// pageOptions: {
// onlyMainContent: true
// },
// extractorOptions: {
// mode: "llm-extraction",
// extractionPrompt: "Extract the name, market cap, price, and today's change for the top 20 companies listed on the page.",
// extractionSchema: {
// type: "object",
// properties: {
// companies: {
// type: "array",
// items: {
// type: "object",
// properties: {
// rank: { type: "number" },
// name: { type: "string" },
// marketCap: { type: "string" },
// price: { type: "string" },
// todayChange: { type: "string" }
// },
// required: ["rank", "name", "marketCap", "price", "todayChange"]
// }
// }
// },
// required: ["companies"]
// }
// }
// });
// // Print the response body to the console for debugging purposes
// console.log("Response companies:", response.body.data.llm_extraction.companies);
// // Check if the response has the correct structure and data types
// expect(response.status).toBe(200);
// expect(Array.isArray(response.body.data.llm_extraction.companies)).toBe(true);
// expect(response.body.data.llm_extraction.companies.length).toBe(40);
// // Sample check for the first company
// const firstCompany = response.body.data.llm_extraction.companies[0];
// expect(firstCompany).toHaveProperty("name");
// expect(typeof firstCompany.name).toBe("string");
// expect(firstCompany).toHaveProperty("marketCap");
// expect(typeof firstCompany.marketCap).toBe("string");
// expect(firstCompany).toHaveProperty("price");
// expect(typeof firstCompany.price).toBe("string");
// expect(firstCompany).toHaveProperty("todayChange");
// expect(typeof firstCompany.todayChange).toBe("string");
// }, 120000); // 120 secs
// });
describe("POST /v0/crawl with fast mode", () => {
it.concurrent(
"should complete the crawl under 20 seconds",
async () => {
const startTime = Date.now();
const crawlResponse = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${config.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({
url: "https://flutterbricks.com",
crawlerOptions: {
mode: "fast",
},
});
expect(crawlResponse.statusCode).toBe(200);
const jobId = crawlResponse.body.jobId;
let statusResponse;
let isFinished = false;
while (!isFinished) {
statusResponse = await request(TEST_URL)
.get(`/v0/crawl/status/${jobId}`)
.set("Authorization", `Bearer ${config.TEST_API_KEY}`);
expect(statusResponse.statusCode).toBe(200);
isFinished = statusResponse.body.status === "completed";
if (!isFinished) {
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
}
}
// const endTime = Date.now();
// const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds
// console.log(`Time elapsed: ${timeElapsed} seconds`);
expect(statusResponse.body.status).toBe("completed");
expect(statusResponse.body).toHaveProperty("data");
expect(statusResponse.body.data[0]).toHaveProperty("content");
expect(statusResponse.body.data[0]).toHaveProperty("markdown");
expect(statusResponse.body.data[0]).toHaveProperty("metadata");
expect(statusResponse.body.data[0].metadata.pageStatusCode).toBe(200);
expect(statusResponse.body.data[0].metadata.pageError).toBeUndefined();
const results = statusResponse.body.data;
// results.forEach((result, i) => {
// console.log(result.metadata.sourceURL);
// });
expect(results.length).toBeGreaterThanOrEqual(10);
expect(results.length).toBeLessThanOrEqual(15);
},
20000,
);
// it.concurrent("should complete the crawl in more than 10 seconds", async () => {
// const startTime = Date.now();
// const crawlResponse = await request(TEST_URL)
// .post("/v0/crawl")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({
// url: "https://flutterbricks.com",
// });
// expect(crawlResponse.statusCode).toBe(200);
// const jobId = crawlResponse.body.jobId;
// let statusResponse;
// let isFinished = false;
// while (!isFinished) {
// statusResponse = await request(TEST_URL)
// .get(`/v0/crawl/status/${jobId}`)
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
// expect(statusResponse.statusCode).toBe(200);
// isFinished = statusResponse.body.status === "completed";
// if (!isFinished) {
// await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
// }
// }
// const endTime = Date.now();
// const timeElapsed = (endTime - startTime) / 1000; // Convert to seconds
// console.log(`Time elapsed: ${timeElapsed} seconds`);
// expect(statusResponse.body.status).toBe("completed");
// expect(statusResponse.body).toHaveProperty("data");
// expect(statusResponse.body.data[0]).toHaveProperty("content");
// expect(statusResponse.body.data[0]).toHaveProperty("markdown");
// const results = statusResponse.body.data;
// // results.forEach((result, i) => {
// // console.log(result.metadata.sourceURL);
// // });
// expect(results.length).toBeGreaterThanOrEqual(10);
// expect(results.length).toBeLessThanOrEqual(15);
// }, 50000);// 15 seconds timeout to account for network delays
});
describe("GET /is-production", () => {
it.concurrent("should return the production status", async () => {
const response = await request(TEST_URL).get("/is-production");
expect(response.statusCode).toBe(200);
expect(response.body).toHaveProperty("isProduction");
});
});
describe("Rate Limiter", () => {
it.concurrent(
"should return 429 when rate limit is exceeded for preview token",
async () => {
for (let i = 0; i < 5; i++) {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.PREVIEW_TOKEN}`)
.set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com" });
expect(response.statusCode).toBe(200);
}
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${config.PREVIEW_TOKEN}`)
.set("Content-Type", "application/json")
.send({ url: "https://www.scrapethissite.com" });
expect(response.statusCode).toBe(429);
},
90000,
);
});
// it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {
// for (let i = 0; i < parseInt(config.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) {
// const response = await request(TEST_URL)
// .post("/v0/scrape")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://www.scrapethissite.com" });
// expect(response.statusCode).toBe(200);
// }
// const response = await request(TEST_URL)
// .post("/v0/scrape")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://www.scrapethissite.com" });
// expect(response.statusCode).toBe(429);
// }, 60000);
// it.concurrent("should return 429 when rate limit is exceeded for API key", async () => {
// for (let i = 0; i < parseInt(config.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) {
// const response = await request(TEST_URL)
// .post("/v0/crawl")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://www.scrapethissite.com" });
// expect(response.statusCode).toBe(200);
// }
// const response = await request(TEST_URL)
// .post("/v0/crawl")
// .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
// .set("Content-Type", "application/json")
// .send({ url: "https://www.scrapethissite.com" });
// expect(response.statusCode).toBe(429);
// }, 60000);
});