Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

22.2 KB
Raw Blame
import type { MockedFunction } from "vitest";
import "dotenv/config";

import { config } from "../../config";
config.ENV = "test";

import { scrapeURL } from ".";
import { scrapeOptions } from "../../controllers/v2/types";
import { Engine } from "./engines";
import { CostTracking } from "../../lib/cost-tracking";
import { AgentIndexOnlyError, NoCachedDataError } from "./error";

// Mock parseMarkdown but delegate to real implementation for other tests
vi.mock("../../lib/html-to-markdown", async importOriginal => {
  const actual =
    await importOriginal<typeof import("../../lib/html-to-markdown")>();
  return {
    ...actual,
    parseMarkdown: vi.fn(actual.parseMarkdown),
  };
});

import { parseMarkdown } from "../../lib/html-to-markdown";

const testEngines: (Engine | undefined)[] = [
  undefined,
  "fire-engine;chrome-cdp",
  "fire-engine;tlsclient",
  "fetch",
];

const testEnginesScreenshot: (Engine | undefined)[] = [
  undefined,
  "fire-engine;chrome-cdp",
];

describe("Standalone scrapeURL tests", () => {
  it.each([
    ["https://example.com/raw", "text/html; charset=utf-8"],
    ["https://example.com/raw.pdf", "application/pdf"],
    [
      "https://example.com/raw.docx",
      "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
    ],
  ])(
    "returns rawBase64 through fire-engine Chrome for %s",
    async (url, contentType) => {
      const out = await scrapeURL(
        "test:raw-base64",
        url,
        scrapeOptions.parse({
          formats: ["rawBase64"],
          useMock: "raw-base64",
          fastMode: true,
          blockAds: false,
          proxy: "stealth",
        }),
        { forceEngine: "fetch", teamId: "test", orgId: null },
        new CostTracking(),
      );

      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.rawBase64).toBe("PGh0bWw+cmF3PC9odG1sPg==");
        expect(out.document).not.toHaveProperty("markdown");
        expect(out.document).not.toHaveProperty("html");
        expect(out.document).not.toHaveProperty("rawHtml");
        expect(out.document.metadata.contentType).toBe(contentType);
      }
    },
  );

  it("rejects a rawBase64 engine response without file content", async () => {
    const out = await scrapeURL(
      "test:raw-base64-missing",
      "https://example.com/raw-error",
      scrapeOptions.parse({
        formats: ["rawBase64"],
        useMock: "raw-base64",
      }),
      { teamId: "test", orgId: null },
      new CostTracking(),
    );

    expect(out.success).toBe(false);
  });

  it("returns the canonical agent-index-only error for rawBase64", async () => {
    const out = await scrapeURL(
      "test:raw-base64-agent-index-only",
      "https://example.com/raw",
      scrapeOptions.parse({
        formats: ["rawBase64"],
        useMock: "raw-base64",
      }),
      { teamId: "test", orgId: null, agentIndexOnly: true },
      new CostTracking(),
    );

    expect(out.success).toBe(false);
    if (!out.success) {
      expect(out.error).toBeInstanceOf(AgentIndexOnlyError);
    }
  });

  it("rejects rawBase64 when minAge is set", async () => {
    const out = await scrapeURL(
      "test:raw-base64-min-age",
      "https://example.com/raw",
      scrapeOptions.parse({
        formats: ["rawBase64"],
        minAge: 0,
      }),
      { teamId: "test", orgId: null },
      new CostTracking(),
    );

    expect(out.success).toBe(false);
    if (!out.success) {
      expect(out.error).toBeInstanceOf(NoCachedDataError);
    }
  });

  describe.each(testEngines)("Engine %s", (forceEngine: Engine | undefined) => {
    it("Basic scrape", async () => {
      const out = await scrapeURL(
        "test:scrape-basic",
        "https://firecrawl-test-site.vercel.app",
        scrapeOptions.parse({}),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).not.toHaveProperty("content");
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document).not.toHaveProperty("html");
        expect(out.document.markdown).toContain("Firecrawl Test Site");
        expect(out.document.metadata.error).toBeUndefined();
        expect(out.document.metadata.title).toBe("Firecrawl Test Website");
        expect(out.document.metadata.description).toBe(
          "Welcome to the Firecrawl Test Website!",
        );
        expect(out.document.metadata.ogTitle).toBe("Firecrawl Test Website");
        expect(out.document.metadata.ogDescription).toBe(
          "Welcome to the Firecrawl Test Website!",
        );
        expect(out.document.metadata.ogUrl).toContain("firecrawl-test-site");
        expect(out.document.metadata.ogImage).toContain("firecrawl-test-site");
        expect(out.document.metadata.ogLocaleAlternate).toStrictEqual([]);
        expect(out.document.metadata.sourceURL).toBe(
          "https://firecrawl-test-site.vercel.app",
        );
        expect(out.document.metadata.statusCode).toBe(200);
      }
    }, 30000);

    it("Scrape with formats markdown and html", async () => {
      const out = await scrapeURL(
        "test:scrape-formats-markdown-html",
        "https://firecrawl-test-site.vercel.app",
        scrapeOptions.parse({
          formats: ["markdown", "html"],
        }),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("html");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.markdown).toContain("Firecrawl Test Site");
        expect(out.document.html).toContain("<h1");
        expect(out.document.metadata.statusCode).toBe(200);
        expect(out.document.metadata.error).toBeUndefined();
      }
    }, 30000);

    it("Scrape with onlyMainContent disabled", async () => {
      const out = await scrapeURL(
        "test:scrape-onlyMainContent-false",
        "https://www.scrapethissite.com/",
        scrapeOptions.parse({
          onlyMainContent: false,
        }),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document).not.toHaveProperty("html");
        expect(out.document.markdown).toContain("[FAQ](/faq/)"); // .nav
        expect(out.document.markdown).toContain("Hartley Brody 2023"); // #footer
      }
    }, 30000);

    it("Scrape with excludeTags", async () => {
      const out = await scrapeURL(
        "test:scrape-excludeTags",
        "https://www.scrapethissite.com/",
        scrapeOptions.parse({
          onlyMainContent: false,
          excludeTags: [".nav", "#footer", "strong"],
        }),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document).not.toHaveProperty("html");
        expect(out.document.markdown).not.toContain("Hartley Brody 2023");
        expect(out.document.markdown).not.toContain("[FAQ](/faq/)");
      }
    }, 30000);

    it("Scrape of a page with 400 status code", async () => {
      const out = await scrapeURL(
        "test:scrape-400",
        "https://httpstat.us/400",
        scrapeOptions.parse({}),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.metadata.statusCode).toBe(400);
      }
    }, 30000);

    it("Scrape of a page with 401 status code", async () => {
      const out = await scrapeURL(
        "test:scrape-401",
        "https://httpstat.us/401",
        scrapeOptions.parse({}),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.metadata.statusCode).toBe(401);
      }
    }, 30000);

    it("Scrape of a page with 403 status code", async () => {
      const out = await scrapeURL(
        "test:scrape-403",
        "https://httpstat.us/403",
        scrapeOptions.parse({}),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.metadata.statusCode).toBe(403);
      }
    }, 30000);

    it("Scrape of a page with 404 status code", async () => {
      const out = await scrapeURL(
        "test:scrape-404",
        "https://httpstat.us/404",
        scrapeOptions.parse({}),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.metadata.statusCode).toBe(404);
      }
    }, 30000);

    it("Scrape of a page with 405 status code", async () => {
      const out = await scrapeURL(
        "test:scrape-405",
        "https://httpstat.us/405",
        scrapeOptions.parse({}),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.metadata.statusCode).toBe(405);
      }
    }, 30000);

    it("Scrape of a page with 500 status code", async () => {
      const out = await scrapeURL(
        "test:scrape-500",
        "https://httpstat.us/500",
        scrapeOptions.parse({}),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.metadata.statusCode).toBe(500);
      }
    }, 30000);

    it("Scrape a redirected page", async () => {
      const out = await scrapeURL(
        "test:scrape-redirect",
        "https://scrapethissite.com/",
        scrapeOptions.parse({}),
        { forceEngine, teamId: "test", orgId: null },
        new CostTracking(),
      );

      // expect(out.logs.length).toBeGreaterThan(0);
      expect(out.success).toBe(true);
      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document.markdown).toContain("Explore Sandbox");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.metadata.sourceURL).toBe(
          "https://scrapethissite.com/",
        );
        expect(out.document.metadata.url).toBe(
          "https://www.scrapethissite.com/",
        );
        expect(out.document.metadata.statusCode).toBe(200);
        expect(out.document.metadata.error).toBeUndefined();
      }
    }, 30000);
  });

  describe.each(testEnginesScreenshot)(
    "Screenshot on engine %s",
    (forceEngine: Engine | undefined) => {
      it("Scrape with screenshot", async () => {
        const out = await scrapeURL(
          "test:scrape-screenshot",
          "https://www.scrapethissite.com/",
          scrapeOptions.parse({
            formats: ["screenshot"],
          }),
          { forceEngine, teamId: "test", orgId: null },
          new CostTracking(),
        );

        // expect(out.logs.length).toBeGreaterThan(0);
        expect(out.success).toBe(true);
        if (out.success) {
          expect(out.document.warning).toBeUndefined();
          expect(out.document).toHaveProperty("screenshot");
          expect(typeof out.document.screenshot).toBe("string");
          expect(
            out.document.screenshot!.startsWith(
              "https://service.firecrawl.dev/storage/v1/object/public/media/",
            ),
          );
          // TODO: attempt to fetch screenshot
          expect(out.document).toHaveProperty("metadata");
          expect(out.document.metadata.statusCode).toBe(200);
          expect(out.document.metadata.error).toBeUndefined();
        }
      }, 30000);

      it("Scrape with full-page screenshot", async () => {
        const out = await scrapeURL(
          "test:scrape-screenshot-fullPage",
          "https://www.scrapethissite.com/",
          scrapeOptions.parse({
            formats: ["screenshot@fullPage"],
          }),
          { forceEngine, teamId: "test", orgId: null },
          new CostTracking(),
        );

        // expect(out.logs.length).toBeGreaterThan(0);
        expect(out.success).toBe(true);
        if (out.success) {
          expect(out.document.warning).toBeUndefined();
          expect(out.document).toHaveProperty("screenshot");
          expect(typeof out.document.screenshot).toBe("string");
          expect(
            out.document.screenshot!.startsWith(
              "https://service.firecrawl.dev/storage/v1/object/public/media/",
            ),
          );
          // TODO: attempt to fetch screenshot
          expect(out.document).toHaveProperty("metadata");
          expect(out.document.metadata.statusCode).toBe(200);
          expect(out.document.metadata.error).toBeUndefined();
        }
      }, 30000);
    },
  );

  it("Scrape of a PDF file", async () => {
    const out = await scrapeURL(
      "test:scrape-pdf",
      "https://arxiv.org/pdf/astro-ph/9301001.pdf",
      scrapeOptions.parse({}),
      { teamId: "test", orgId: null },
      new CostTracking(),
    );

    // expect(out.logs.length).toBeGreaterThan(0);
    expect(out.success).toBe(true);
    if (out.success) {
      expect(out.document.warning).toBeUndefined();
      expect(out.document).toHaveProperty("metadata");
      expect(out.document.markdown).toContain("Broad Line Radio Galaxy");
      expect(out.document.metadata.statusCode).toBe(200);
      expect(out.document.metadata.error).toBeUndefined();
    }
  }, 60000);

  it("Scrape a DOCX file", async () => {
    const out = await scrapeURL(
      "test:scrape-docx",
      "https://nvca.org/wp-content/uploads/2019/06/NVCA-Model-Document-Stock-Purchase-Agreement.docx",
      scrapeOptions.parse({}),
      { teamId: "test", orgId: null },
      new CostTracking(),
    );

    // expect(out.logs.length).toBeGreaterThan(0);
    expect(out.success).toBe(true);
    if (out.success) {
      expect(out.document.warning).toBeUndefined();
      expect(out.document).toHaveProperty("metadata");
      expect(out.document.markdown).toContain(
        "SERIES A PREFERRED STOCK PURCHASE AGREEMENT",
      );
      expect(out.document.metadata.statusCode).toBe(200);
      expect(out.document.metadata.error).toBeUndefined();
    }
  }, 60000);

  it("Scrapes a XLSX file", async () => {
    const out = await scrapeURL(
      "test:scrape-xlsx",
      "https://download.microsoft.com/download/1/4/E/14EDED28-6C58-4055-A65C-23B4DA81C4DE/Financial%20Sample.xlsx",
      scrapeOptions.parse({}),
      { teamId: "test", orgId: null },
      new CostTracking(),
    );

    // expect(out.logs.length).toBeGreaterThan(0);
    expect(out.success).toBe(true);
    if (out.success) {
      expect(out.document.warning).toBeUndefined();
      expect(out.document).toHaveProperty("metadata");
      // sheet name
      expect(out.document.markdown).toContain("Sheet1");
      // headers
      expect(out.document.markdown).toContain("Segment");
      expect(out.document.markdown).toContain("Product");
      expect(out.document.markdown).toContain("Country");
      expect(out.document.metadata.statusCode).toBe(200);
      expect(out.document.metadata.error).toBeUndefined();
    }
  }, 60000);

  it("LLM extract with prompt and schema", async () => {
    const out = await scrapeURL(
      "test:llm-extract-prompt-schema",
      "https://firecrawl.dev",
      scrapeOptions.parse({
        formats: ["extract"],
        extract: {
          prompt:
            "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
          schema: {
            type: "object",
            properties: {
              company_mission: { type: "string" },
              supports_sso: { type: "boolean" },
              is_open_source: { type: "boolean" },
            },
            required: ["company_mission", "supports_sso", "is_open_source"],
            additionalProperties: false,
          },
        },
      }),
      { teamId: "test", orgId: null },
      new CostTracking(),
    );

    // expect(out.logs.length).toBeGreaterThan(0);
    expect(out.success).toBe(true);
    if (out.success) {
      expect(out.document.warning).toBeUndefined();
      expect(out.document).toHaveProperty("extract");
      expect(out.document.extract).toHaveProperty("company_mission");
      expect(out.document.extract).toHaveProperty("supports_sso");
      expect(out.document.extract).toHaveProperty("is_open_source");
      expect(typeof out.document.extract.company_mission).toBe("string");
      expect(out.document.extract.supports_sso).toBe(false);
      expect(out.document.extract.is_open_source).toBe(true);
    }
  }, 120000);

  it("LLM extract with schema only", async () => {
    const out = await scrapeURL(
      "test:llm-extract-schema",
      "https://firecrawl.dev",
      scrapeOptions.parse({
        formats: ["extract"],
        extract: {
          schema: {
            type: "object",
            properties: {
              company_mission: { type: "string" },
              supports_sso: { type: "boolean" },
              is_open_source: { type: "boolean" },
            },
            required: ["company_mission", "supports_sso", "is_open_source"],
            additionalProperties: false,
          },
        },
      }),
      { teamId: "test", orgId: null },
      new CostTracking(),
    );

    // expect(out.logs.length).toBeGreaterThan(0);
    expect(out.success).toBe(true);
    if (out.success) {
      expect(out.document.warning).toBeUndefined();
      expect(out.document).toHaveProperty("extract");
      expect(out.document.extract).toHaveProperty("company_mission");
      expect(out.document.extract).toHaveProperty("supports_sso");
      expect(out.document.extract).toHaveProperty("is_open_source");
      expect(typeof out.document.extract.company_mission).toBe("string");
      expect(out.document.extract.supports_sso).toBe(false);
      expect(out.document.extract.is_open_source).toBe(true);
    }
  }, 120000);

  test.concurrent.each(new Array(100).fill(0).map((_, i) => i))(
    "Concurrent scrape #%i",
    async i => {
      const url = "https://www.scrapethissite.com/?i=" + i;
      const id = "test:concurrent:" + url;
      const out = await scrapeURL(
        id,
        url,
        scrapeOptions.parse({}),
        { teamId: "test", orgId: null },
        new CostTracking(),
      );

      const replacer = (key: string, value: any) => {
        if (value instanceof Error) {
          return {
            ...value,
            message: value.message,
            name: value.name,
            cause: value.cause,
            stack: value.stack,
          };
        } else {
          return value;
        }
      };

      if (!out.success) console.error(JSON.stringify(out, replacer));
      expect(out.success).toBe(true);

      if (out.success) {
        expect(out.document.warning).toBeUndefined();
        expect(out.document).toHaveProperty("markdown");
        expect(out.document).toHaveProperty("metadata");
        expect(out.document.metadata.error).toBeUndefined();
        expect(out.document.metadata.statusCode).toBe(200);
      }
    },
    30000,
  );

  it("Sitemap scrape should not convert to markdown", async () => {
    const mockParseMarkdown = parseMarkdown as MockedFunction<
      typeof parseMarkdown
    >;
    mockParseMarkdown.mockClear();

    const out = await scrapeURL(
      "test:sitemap-no-markdown",
      "https://www.scrapethissite.com/sitemap.xml",
      scrapeOptions.parse({
        formats: ["rawHtml"],
      }),
      { teamId: "sitemap", orgId: null },
      new CostTracking(),
    );

    expect(out.success).toBe(true);
    if (out.success) {
      expect(out.document.warning).toBeUndefined();
      // Verify markdown conversion was never called
      expect(mockParseMarkdown).not.toHaveBeenCalled();
      // Sitemap scrapes should not have markdown field
      expect(out.document).not.toHaveProperty("markdown");
      // But should have rawHtml
      expect(out.document).toHaveProperty("rawHtml");
      expect(out.document.rawHtml).toBeTruthy();
      expect(out.document).toHaveProperty("metadata");
      expect(out.document.metadata.error).toBeUndefined();
    }
  }, 30000);
});