Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

41.2 KB
Raw Blame
import request from "supertest";
import { config } from "../../config";
import { configDotenv } from "dotenv";
import { ScrapeRequestInput } from "../../controllers/v1/types";
import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings";

configDotenv();
const TEST_URL = "http://127.0.0.1:3002";

describe("E2E Tests for v1 API Routes", () => {
  beforeAll(() => {
    config.USE_DB_AUTHENTICATION = true;
  });

  afterAll(() => {
    delete config.USE_DB_AUTHENTICATION;
  });

  describe("GET /is-production", () => {
    it.concurrent("should return the production status", async () => {
      const response: any = await request(TEST_URL).get("/is-production");

      console.log("config.USE_DB_AUTHENTICATION", config.USE_DB_AUTHENTICATION);
      console.log("?", config.USE_DB_AUTHENTICATION);
      const useDbAuthentication = config.USE_DB_AUTHENTICATION;
      console.log("!!useDbAuthentication", !!useDbAuthentication);
      console.log("!useDbAuthentication", !useDbAuthentication);

      expect(response.statusCode).toBe(200);
      expect(response.body).toHaveProperty("isProduction");
    });
  });

  describe("POST /v1/scrape", () => {
    it.concurrent("should require authorization", async () => {
      const response: any = await request(TEST_URL)
        .post("/v1/scrape")
        .send({ url: "https://firecrawl.dev" });

      expect(response.statusCode).toBe(401);
    });

    it.concurrent("should throw error for unsupported URL", async () => {
      const scrapeRequest: ScrapeRequestInput = {
        url: "https://facebook.com/fake-test",
      };

      const response = await request(TEST_URL)
        .post("/v1/scrape")
        .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
        .set("Content-Type", "application/json")
        .send(scrapeRequest);

      expect(response.statusCode).toBe(403);
      expect(response.body.error).toBe(UNSUPPORTED_SITE_MESSAGE);
    });

    it.concurrent(
      "should return an error response with an invalid API key",
      async () => {
        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer invalid-api-key`)
          .set("Content-Type", "application/json")
          .send({ url: "https://firecrawl.dev" });
        expect(response.statusCode).toBe(401);
      },
    );

    it.concurrent(
      "should return a successful response with a valid API key",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://firecrawl-test-site.vercel.app",
        };

        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);

        expect(response.statusCode).toBe(200);

        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).not.toHaveProperty("content");
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data).not.toHaveProperty("html");
        expect(response.body.data.markdown).toContain("Firecrawl Test Site");
        expect(response.body.data.metadata.error).toBeUndefined();
        expect(response.body.data.metadata.title).toBe(
          "Firecrawl Test Website",
        );
        expect(response.body.data.metadata.description).toBe(
          "Welcome to the Firecrawl Test Website!",
        );
        expect(response.body.data.metadata.ogTitle).toBe(
          "Firecrawl Test Website",
        );
        expect(response.body.data.metadata.ogDescription).toBe(
          "Welcome to the Firecrawl Test Website!",
        );
        expect(response.body.data.metadata.ogUrl).toContain(
          "firecrawl-test-site",
        );
        expect(response.body.data.metadata.ogImage).toContain(
          "firecrawl-test-site",
        );
        expect(response.body.data.metadata.ogLocaleAlternate).toStrictEqual([]);
        expect(response.body.data.metadata.sourceURL).toBe(
          "https://firecrawl-test-site.vercel.app",
        );
        expect(response.body.data.metadata.statusCode).toBe(200);
      },
      30000,
    ); // 30 seconds timeout

    it.concurrent(
      "should return a successful response with a valid API key",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://arxiv.org/abs/2410.04840",
        };

        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);

        expect(response.statusCode).toBe(200);

        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).not.toHaveProperty("content");
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data).not.toHaveProperty("html");
        expect(response.body.data.markdown).toContain("Strong Model Collapse");
        expect(response.body.data.metadata.error).toBeUndefined();
        expect(response.body.data.metadata.description).toContain(
          "Abstract page for arXiv paper 2410.04840: Strong Model Collapse",
        );
        expect(response.body.data.metadata.citation_title).toBe(
          "Strong Model Collapse",
        );
        expect(response.body.data.metadata.citation_author).toEqual([
          "Dohmatob, Elvis",
          "Feng, Yunzhen",
          "Subramonian, Arjun",
          "Kempe, Julia",
        ]);
        expect(response.body.data.metadata.citation_date).toBe("2024/10/07");
        expect(response.body.data.metadata.citation_online_date).toBe(
          "2024/10/08",
        );
        expect(response.body.data.metadata.citation_pdf_url).toBe(
          "http://arxiv.org/pdf/2410.04840",
        );
        expect(response.body.data.metadata.citation_arxiv_id).toBe(
          "2410.04840",
        );
        expect(response.body.data.metadata.citation_abstract).toContain(
          "Within the scaling laws paradigm",
        );
        expect(response.body.data.metadata.sourceURL).toBe(
          "https://arxiv.org/abs/2410.04840",
        );
        expect(response.body.data.metadata.statusCode).toBe(200);
      },
      30000,
    );
    it.concurrent(
      "should return a successful response with a valid API key and includeHtml set to true",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://firecrawl-test-site.vercel.app",
          formats: ["markdown", "html"],
        };

        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("html");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.markdown).toContain("Firecrawl Test Site");
        expect(response.body.data.html).toContain("<h1");
        expect(response.body.data.metadata.statusCode).toBe(200);
        expect(response.body.data.metadata.error).toBeUndefined();
      },
      30000,
    );
    it.concurrent(
      "should return a successful response for a valid scrape with PDF file",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://arxiv.org/pdf/astro-ph/9301001.pdf",
          //   formats: ["markdown", "html"],
        };
        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);
        await new Promise(r => setTimeout(r, 6000));

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.markdown).toContain(
          "Broad Line Radio Galaxy",
        );
        expect(response.body.data.metadata.statusCode).toBe(200);
        expect(response.body.data.metadata.error).toBeUndefined();
      },
      60000,
    );

    it.concurrent(
      "should return a successful response for a valid scrape with PDF file without explicit .pdf extension",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://arxiv.org/pdf/astro-ph/9301001",
        };
        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);
        await new Promise(r => setTimeout(r, 6000));

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.markdown).toContain(
          "Broad Line Radio Galaxy",
        );
        expect(response.body.data.metadata.statusCode).toBe(200);
        expect(response.body.data.metadata.error).toBeUndefined();
      },
      60000,
    );

    it.concurrent(
      "should return a successful response with a valid API key with removeTags option",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://www.scrapethissite.com/",
          onlyMainContent: false, // default is true
        };
        const responseWithoutRemoveTags: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);
        expect(responseWithoutRemoveTags.statusCode).toBe(200);
        expect(responseWithoutRemoveTags.body).toHaveProperty("data");

        if (!("data" in responseWithoutRemoveTags.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(responseWithoutRemoveTags.body.data).toHaveProperty("markdown");
        expect(responseWithoutRemoveTags.body.data).toHaveProperty("metadata");
        expect(responseWithoutRemoveTags.body.data).not.toHaveProperty("html");
        expect(responseWithoutRemoveTags.body.data.markdown).toContain(
          "[FAQ](/faq/)",
        ); // .nav
        expect(responseWithoutRemoveTags.body.data.markdown).toContain(
          "Hartley Brody 2023",
        ); // #footer

        const scrapeRequestWithRemoveTags: ScrapeRequestInput = {
          url: "https://www.scrapethissite.com/",
          excludeTags: [".nav", "#footer", "strong"],
          onlyMainContent: false, // default is true
        };
        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequestWithRemoveTags);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data).not.toHaveProperty("html");
        expect(response.body.data.markdown).not.toContain("Hartley Brody 2023");
        expect(response.body.data.markdown).not.toContain("[FAQ](/faq/)"); //
      },
      30000,
    );

    it.concurrent(
      "should return a successful response for a scrape with 400 page",
      async () => {
        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({ url: "https://httpstat.us/400" });
        await new Promise(r => setTimeout(r, 5000));

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.metadata.statusCode).toBe(400);
      },
      60000,
    );

    it.concurrent(
      "should return a successful response for a scrape with 401 page",
      async () => {
        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({ url: "https://httpstat.us/401" });
        await new Promise(r => setTimeout(r, 5000));

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.metadata.statusCode).toBe(401);
      },
      60000,
    );

    // Removed it as we want to retry fallback to the next scraper
    // it.concurrent('should return a successful response for a scrape with 403 page', async () => {
    //   const response: any = await request(TEST_URL)
    //     .post('/v1/scrape')
    //     .set('Authorization', `Bearer ${config.TEST_API_KEY}`)
    //     .set('Content-Type', 'application/json')
    //     .send({ url: 'https://httpstat.us/403' });
    //   await new Promise((r) => setTimeout(r, 5000));

    //   expect(response.statusCode).toBe(200);
    //   expect(response.body).toHaveProperty('data');
    //   if (!("data" in response.body)) {
    //     throw new Error("Expected response body to have 'data' property");
    //   }
    //   expect(response.body.data).toHaveProperty('markdown');
    //   expect(response.body.data).toHaveProperty('metadata');
    //   expect(response.body.data.metadata.statusCode).toBe(403);
    // }, 60000);

    it.concurrent(
      "should return a successful response for a scrape with 404 page",
      async () => {
        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({ url: "https://httpstat.us/404" });
        await new Promise(r => setTimeout(r, 5000));

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.metadata.statusCode).toBe(404);
      },
      60000,
    );

    // it.concurrent('should return a successful response for a scrape with 405 page', async () => {
    //   const response: any = await request(TEST_URL)
    //     .post('/v1/scrape')
    //     .set('Authorization', `Bearer ${config.TEST_API_KEY}`)
    //     .set('Content-Type', 'application/json')
    //     .send({ url: 'https://httpstat.us/405' });
    //   await new Promise((r) => setTimeout(r, 5000));

    //   expect(response.statusCode).toBe(200);
    //   expect(response.body).toHaveProperty('data');
    //   if (!("data" in response.body)) {
    //     throw new Error("Expected response body to have 'data' property");
    //   }
    //   expect(response.body.data).toHaveProperty('markdown');
    //   expect(response.body.data).toHaveProperty('metadata');
    //   expect(response.body.data.metadata.statusCode).toBe(405);
    // }, 60000);

    // it.concurrent('should return a successful response for a scrape with 500 page', async () => {
    //   const response: any = await request(TEST_URL)
    //     .post('/v1/scrape')
    //     .set('Authorization', `Bearer ${config.TEST_API_KEY}`)
    //     .set('Content-Type', 'application/json')
    //     .send({ url: 'https://httpstat.us/500' });
    //   await new Promise((r) => setTimeout(r, 5000));

    //   expect(response.statusCode).toBe(200);
    //   expect(response.body).toHaveProperty('data');
    //   if (!("data" in response.body)) {
    //     throw new Error("Expected response body to have 'data' property");
    //   }
    //   expect(response.body.data).toHaveProperty('markdown');
    //   expect(response.body.data).toHaveProperty('metadata');
    //   expect(response.body.data.metadata.statusCode).toBe(500);
    // }, 60000);

    it.concurrent(
      "should return a timeout error when scraping takes longer than the specified timeout",
      async () => {
        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({ url: "https://firecrawl.dev", timeout: 1000 });

        expect(response.statusCode).toBe(408);
      },
      3000,
    );

    it.concurrent(
      "should return a successful response with a valid API key and includeHtml set to true",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://firecrawl-test-site.vercel.app",
          formats: ["html", "rawHtml"],
        };

        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).not.toHaveProperty("markdown");
        expect(response.body.data).toHaveProperty("html");
        expect(response.body.data).toHaveProperty("rawHtml");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.html).toContain("<h1");
        expect(response.body.data.rawHtml).toContain("<html");
        expect(response.body.data.metadata.statusCode).toBe(200);
        expect(response.body.data.metadata.error).toBeUndefined();
      },
      30000,
    );

    it.concurrent(
      "should return a successful response with waitFor",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://ycombinator.com/companies",
          formats: ["markdown"],
          waitFor: 8000,
        };

        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).toHaveProperty("markdown");
        expect(response.body.data).not.toHaveProperty("html");
        expect(response.body.data).not.toHaveProperty("links");
        expect(response.body.data).not.toHaveProperty("rawHtml");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.markdown).toContain("PagerDuty");
        expect(response.body.data.metadata.statusCode).toBe(200);
        expect(response.body.data.metadata.error).toBeUndefined();
      },
      30000,
    );

    it.concurrent(
      "should return a successful response with a valid links on page",
      async () => {
        const scrapeRequest: ScrapeRequestInput = {
          url: "https://firecrawl-test-site.vercel.app",
          formats: ["links"],
        };

        const response: any = await request(TEST_URL)
          .post("/v1/scrape")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(scrapeRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("data");
        if (!("data" in response.body)) {
          throw new Error("Expected response body to have 'data' property");
        }
        expect(response.body.data).not.toHaveProperty("html");
        expect(response.body.data).not.toHaveProperty("rawHtml");
        expect(response.body.data).toHaveProperty("links");
        expect(response.body.data).toHaveProperty("metadata");
        expect(response.body.data.links).toContain("https://firecrawl.dev");
        expect(response.body.data.metadata.statusCode).toBe(200);
        expect(response.body.data.metadata.error).toBeUndefined();
      },
      30000,
    );
  });

  describe("POST /v1/map", () => {
    it.concurrent("should require authorization", async () => {
      const response: any = await request(TEST_URL)
        .post("/v1/map")
        .send({ url: "https://firecrawl.dev" });
      expect(response.statusCode).toBe(401);
    });

    it.concurrent(
      "should return an error response with an invalid API key",
      async () => {
        const response: any = await request(TEST_URL)
          .post("/v1/map")
          .set("Authorization", `Bearer invalid-api-key`)
          .set("Content-Type", "application/json")
          .send({ url: "https://firecrawl.dev" });
        expect(response.statusCode).toBe(401);
      },
    );

    it.concurrent(
      "should return a successful response with a valid API key",
      async () => {
        const mapRequest = {
          url: "https://firecrawl-test-site.vercel.app",
        };

        const response: any = await request(TEST_URL)
          .post("/v1/map")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(mapRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("success", true);
        expect(response.body).toHaveProperty("links");
        if (!("links" in response.body)) {
          throw new Error("Expected response body to have 'links' property");
        }
        const links = response.body.links as unknown[];
        expect(Array.isArray(links)).toBe(true);
        expect(links.length).toBeGreaterThan(0);
      },
    );

    it.concurrent(
      "should return a successful response with a valid API key and search",
      async () => {
        const mapRequest = {
          url: "https://usemotion.com",
          search: "pricing",
        };

        const response: any = await request(TEST_URL)
          .post("/v1/map")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(mapRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("success", true);
        expect(response.body).toHaveProperty("links");
        if (!("links" in response.body)) {
          throw new Error("Expected response body to have 'links' property");
        }
        const links = response.body.links as unknown[];
        expect(Array.isArray(links)).toBe(true);
        expect(links.length).toBeGreaterThan(0);
        expect(links[0]).toContain("usemotion.com/pricing");
      },
    );

    it.concurrent(
      "should return a successful response with a valid API key and search and allowSubdomains",
      async () => {
        const mapRequest = {
          url: "https://firecrawl.dev",
          search: "docs",
          includeSubdomains: true,
        };

        const response: any = await request(TEST_URL)
          .post("/v1/map")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(mapRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("success", true);
        expect(response.body).toHaveProperty("links");
        if (!("links" in response.body)) {
          throw new Error("Expected response body to have 'links' property");
        }
        const links = response.body.links as unknown[];
        expect(Array.isArray(links)).toBe(true);
        expect(links.length).toBeGreaterThan(0);

        const containsDocsFirecrawlDev = links.some((link: string) =>
          link.includes("docs.firecrawl.dev"),
        );
        expect(containsDocsFirecrawlDev).toBe(true);
      },
    );

    it.concurrent(
      "should return a successful response with a valid API key and search and allowSubdomains and www",
      async () => {
        const mapRequest = {
          url: "https://www.firecrawl.dev",
          search: "docs",
          includeSubdomains: true,
        };

        const response: any = await request(TEST_URL)
          .post("/v1/map")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(mapRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("success", true);
        expect(response.body).toHaveProperty("links");
        if (!("links" in response.body)) {
          throw new Error("Expected response body to have 'links' property");
        }
        const links = response.body.links as unknown[];
        expect(Array.isArray(links)).toBe(true);
        expect(links.length).toBeGreaterThan(0);

        const containsDocsFirecrawlDev = links.some((link: string) =>
          link.includes("docs.firecrawl.dev"),
        );
        expect(containsDocsFirecrawlDev).toBe(true);
      },
      10000,
    );

    it.concurrent(
      "should return a successful response with a valid API key and search and not allowSubdomains and www",
      async () => {
        const mapRequest = {
          url: "https://www.firecrawl.dev",
          search: "docs",
          includeSubdomains: false,
        };

        const response: any = await request(TEST_URL)
          .post("/v1/map")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send(mapRequest);

        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("success", true);
        expect(response.body).toHaveProperty("links");
        if (!("links" in response.body)) {
          throw new Error("Expected response body to have 'links' property");
        }
        const links = response.body.links as unknown[];
        expect(Array.isArray(links)).toBe(true);
        expect(links.length).toBeGreaterThan(0);
        expect(links[0]).not.toContain("docs.firecrawl.dev");
      },
    );

    it.concurrent("should return an error for invalid URL", async () => {
      const mapRequest = {
        url: "invalid-url",
        includeSubdomains: true,
        search: "test",
      };

      const response: any = await request(TEST_URL)
        .post("/v1/map")
        .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
        .set("Content-Type", "application/json")
        .send(mapRequest);

      expect(response.statusCode).toBe(400);
      expect(response.body).toHaveProperty("success", false);
      expect(response.body).toHaveProperty("error");
    });
  });

  describe("POST /v1/crawl", () => {
    it.concurrent("should require authorization", async () => {
      const response: any = await request(TEST_URL)
        .post("/v1/crawl")
        .send({ url: "https://firecrawl.dev" });
      expect(response.statusCode).toBe(401);
    });

    it.concurrent("should throw error for unsupported URL", async () => {
      const scrapeRequest: ScrapeRequestInput = {
        url: "https://facebook.com/fake-test",
      };

      const response = await request(TEST_URL)
        .post("/v1/crawl")
        .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
        .set("Content-Type", "application/json")
        .send(scrapeRequest);

      expect(response.statusCode).toBe(403);
      expect(response.body.error).toBe(UNSUPPORTED_SITE_MESSAGE);
    });

    it.concurrent(
      "should return an error response with an invalid API key",
      async () => {
        const response: any = await request(TEST_URL)
          .post("/v1/crawl")
          .set("Authorization", `Bearer invalid-api-key`)
          .set("Content-Type", "application/json")
          .send({ url: "https://firecrawl.dev" });
        expect(response.statusCode).toBe(401);
      },
    );

    it.concurrent("should return a successful response", async () => {
      const response = await request(TEST_URL)
        .post("/v1/crawl")
        .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
        .set("Content-Type", "application/json")
        .send({ url: "https://firecrawl.dev" });

      expect(response.statusCode).toBe(200);
      expect(response.body).toHaveProperty("id");
      expect(response.body.id).toMatch(
        /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/,
      );
      expect(response.body).toHaveProperty("success", true);
      expect(response.body).toHaveProperty("url");
      expect(response.body.url).toContain("/v1/crawl/");
    });

    it.concurrent(
      "should return a successful response with a valid API key and valid includes option",
      async () => {
        const crawlResponse = await request(TEST_URL)
          .post("/v1/crawl")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({
            url: "https://firecrawl.dev",
            limit: 40,
            includePaths: ["blog/*"],
          });

        let response;
        let isFinished = false;

        while (!isFinished) {
          response = await request(TEST_URL)
            .get(`/v1/crawl/${crawlResponse.body.id}`)
            .set("Authorization", `Bearer ${config.TEST_API_KEY}`);

          expect(response.statusCode).toBe(200);
          expect(response.body).toHaveProperty("status");
          isFinished = response.body.status === "completed";

          if (!isFinished) {
            await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
          }
        }

        await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
        const completedResponse = await request(TEST_URL)
          .get(`/v1/crawl/${crawlResponse.body.id}`)
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`);

        const urls = completedResponse.body.data.map(
          (item: any) => item.metadata?.sourceURL,
        );
        expect(urls.length).toBeGreaterThan(5);
        urls.forEach((url: string) => {
          expect(url).toContain("firecrawl.dev/blog");
        });

        expect(completedResponse.statusCode).toBe(200);
        expect(completedResponse.body).toHaveProperty("status");
        expect(completedResponse.body.status).toBe("completed");
        expect(completedResponse.body).toHaveProperty("data");
        expect(completedResponse.body.data[0]).toHaveProperty("markdown");
        expect(completedResponse.body.data[0]).toHaveProperty("metadata");
        expect(completedResponse.body.data[0]).not.toHaveProperty("content"); // v0
        expect(completedResponse.body.data[0].metadata.statusCode).toBe(200);
        expect(completedResponse.body.data[0].metadata.error).toBeUndefined();
      },
      180000,
    ); // 180 seconds

    it.concurrent(
      "should return a successful response with a valid API key and valid excludes option",
      async () => {
        const crawlResponse = await request(TEST_URL)
          .post("/v1/crawl")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({
            url: "https://firecrawl.dev",
            limit: 40,
            excludePaths: ["blog/*"],
          });

        let isFinished = false;
        let response;

        while (!isFinished) {
          response = await request(TEST_URL)
            .get(`/v1/crawl/${crawlResponse.body.id}`)
            .set("Authorization", `Bearer ${config.TEST_API_KEY}`);

          expect(response.statusCode).toBe(200);
          expect(response.body).toHaveProperty("status");
          isFinished = response.body.status === "completed";

          if (!isFinished) {
            await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
          }
        }

        await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
        const completedResponse = await request(TEST_URL)
          .get(`/v1/crawl/${crawlResponse.body.id}`)
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`);

        const urls = completedResponse.body.data.map(
          (item: any) => item.metadata?.sourceURL,
        );
        expect(urls.length).toBeGreaterThan(3);
        urls.forEach((url: string) => {
          expect(url.startsWith("https://www.firecrawl.dev/blog/")).toBeFalsy();
        });
      },
      90000,
    ); // 90 seconds

    it.concurrent(
      "should return a successful response with max depth option for a valid crawl job",
      async () => {
        const crawlResponse = await request(TEST_URL)
          .post("/v1/crawl")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({
            url: "https://www.scrapethissite.com",
            maxDepth: 1,
          });
        expect(crawlResponse.statusCode).toBe(200);

        const response = await request(TEST_URL)
          .get(`/v1/crawl/${crawlResponse.body.id}`)
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
        expect(response.statusCode).toBe(200);
        expect(response.body).toHaveProperty("status");
        expect(["active", "waiting", "completed", "scraping"]).toContain(
          response.body.status,
        );
        // wait for 60 seconds
        let isCompleted = false;
        while (!isCompleted) {
          const statusCheckResponse = await request(TEST_URL)
            .get(`/v1/crawl/${crawlResponse.body.id}`)
            .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
          expect(statusCheckResponse.statusCode).toBe(200);
          isCompleted = statusCheckResponse.body.status === "completed";
          if (!isCompleted) {
            await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
          }
        }
        const completedResponse = await request(TEST_URL)
          .get(`/v1/crawl/${crawlResponse.body.id}`)
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`);

        expect(completedResponse.statusCode).toBe(200);
        expect(completedResponse.body).toHaveProperty("status");
        expect(completedResponse.body.status).toBe("completed");
        expect(completedResponse.body).toHaveProperty("data");
        expect(completedResponse.body.data[0]).not.toHaveProperty("content");
        expect(completedResponse.body.data[0]).toHaveProperty("markdown");
        expect(completedResponse.body.data[0]).toHaveProperty("metadata");
        expect(completedResponse.body.data[0].metadata.statusCode).toBe(200);
        expect(completedResponse.body.data[0].metadata.error).toBeUndefined();
        const urls = completedResponse.body.data.map(
          (item: any) => item.metadata?.sourceURL,
        );
        expect(urls.length).toBeGreaterThan(1);

        // Check if all URLs have a maximum depth of 1
        urls.forEach((url: string) => {
          const pathSplits = new URL(url).pathname.split("/");
          const depth =
            pathSplits.length -
            (pathSplits[0].length === 0 &&
            pathSplits[pathSplits.length - 1].length === 0
              ? 1
              : 0);
          expect(depth).toBeLessThanOrEqual(2);
        });
      },
      180000,
    );
  });

  describe("GET /v1/crawl/:jobId", () => {
    it.concurrent("should require authorization", async () => {
      const response = await request(TEST_URL).get("/v1/crawl/123");
      expect(response.statusCode).toBe(401);
    });

    it.concurrent(
      "should return an error response with an invalid API key",
      async () => {
        const response = await request(TEST_URL)
          .get("/v1/crawl/123")
          .set("Authorization", `Bearer invalid-api-key`);
        expect(response.statusCode).toBe(401);
      },
    );

    it.concurrent("should reject an invalid job ID", async () => {
      const response = await request(TEST_URL)
        .get("/v1/crawl/invalidJobId")
        .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
      expect(response.statusCode).toBe(400);
      expect(response.body.success).toBe(false);
      expect(response.body.error).toBe(
        "Invalid job ID format. Job ID must be a valid UUID.",
      );
    });

    it.concurrent(
      "should return a successful crawl status response for a valid crawl job",
      async () => {
        const crawlResponse = await request(TEST_URL)
          .post("/v1/crawl")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({ url: "https://docs.firecrawl.dev" });
        expect(crawlResponse.statusCode).toBe(200);

        let isCompleted = false;

        while (!isCompleted) {
          const response = await request(TEST_URL)
            .get(`/v1/crawl/${crawlResponse.body.id}`)
            .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
          expect(response.statusCode).toBe(200);
          expect(response.body).toHaveProperty("status");

          if (response.body.status === "completed") {
            isCompleted = true;
          } else {
            await new Promise(r => setTimeout(r, 1000)); // Wait for 1 second before checking again
          }
        }

        await new Promise(resolve => setTimeout(resolve, 1000)); // wait for data to be saved on the database
        const completedResponse = await request(TEST_URL)
          .get(`/v1/crawl/${crawlResponse.body.id}`)
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`);

        expect(completedResponse.body).toHaveProperty("status");
        expect(completedResponse.body.status).toBe("completed");
        expect(completedResponse.body).toHaveProperty("data");
        expect(completedResponse.body.data[0]).not.toHaveProperty("content");
        expect(completedResponse.body.data[0]).toHaveProperty("markdown");
        expect(completedResponse.body.data[0]).toHaveProperty("metadata");
        expect(completedResponse.body.data[0].metadata.statusCode).toBe(200);
        expect(completedResponse.body.data[0].metadata.error).toBeUndefined();

        const childrenLinks = completedResponse.body.data.filter(
          doc => doc.metadata && doc.metadata.sourceURL,
        );

        expect(childrenLinks.length).toBe(completedResponse.body.data.length);
      },
      180000,
    ); // 120 seconds

    it.concurrent(
      "If someone cancels a crawl job, it should turn into failed status",
      async () => {
        const crawlResponse = await request(TEST_URL)
          .post("/v1/crawl")
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
          .set("Content-Type", "application/json")
          .send({ url: "https://docs.firecrawl.dev", limit: 10 });

        expect(crawlResponse.statusCode).toBe(200);

        await new Promise(r => setTimeout(r, 10000));

        const responseCancel = await request(TEST_URL)
          .delete(`/v1/crawl/${crawlResponse.body.id}`)
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`);
        expect(responseCancel.statusCode).toBe(200);
        expect(responseCancel.body).toHaveProperty("status");
        expect(responseCancel.body.status).toBe("cancelled");

        await new Promise(r => setTimeout(r, 10000));
        const completedResponse = await request(TEST_URL)
          .get(`/v1/crawl/${crawlResponse.body.id}`)
          .set("Authorization", `Bearer ${config.TEST_API_KEY}`);

        expect(completedResponse.statusCode).toBe(200);
        expect(completedResponse.body).toHaveProperty("status");
        expect(completedResponse.body.status).toBe("cancelled");
        expect(completedResponse.body).toHaveProperty("data");
        expect(completedResponse.body.data[0]).toHaveProperty("markdown");
        expect(completedResponse.body.data[0]).toHaveProperty("metadata");
        expect(completedResponse.body.data[0].metadata.statusCode).toBe(200);
        expect(completedResponse.body.data[0].metadata.error).toBeUndefined();
      },
      60000,
    ); // 60 seconds
  });
});