Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

2.8 KB
Raw Blame
import path from "path";
import fs from "fs";
import { convertDocumentToMarkdown } from "@mendable/firecrawl-rs";

describe("Document Converter tests", () => {
  const samplesDir = path.join(process.cwd(), "samples");

  const expectedMarkdownBase = (documentText: string) =>
    `**Hello!**\n\n${documentText} file to test the Firecrawl Document Converter.\n\n*Italic*\n\n**Bold**\n\nUnderlined\n\n~~Strikethrough~~\n\n| Header 1 | Header 2 |\n| --- | --- |\n| Value 1 | Value 2 |\n`;

  const sampleFiles = [
    { file: "sample.docx", name: "DOCX" },
    { file: "sample.odt", name: "ODT" },
    { file: "sample.rtf", name: "RTF" },
  ];

  describe.each(sampleFiles)("$name document conversion", ({ file, name }) => {
    const filePath = path.join(samplesDir, file);

    beforeAll(() => {
      if (!fs.existsSync(filePath)) {
        throw new Error(`Sample file ${filePath} does not exist`);
      }
    });

    it.concurrent(
      `should convert ${name} document and return expected markdown`,
      async () => {
        const fileBuffer = fs.readFileSync(filePath);

        // No extension hint: the format must be detected from the bytes
        const markdown = await convertDocumentToMarkdown(
          new Uint8Array(fileBuffer),
        );

        expect(markdown).toBe(expectedMarkdownBase(name));
      },
      10000,
    );
  });

  describe("XLSX document conversion", () => {
    const expectedMarkdown = `## Sheet1\n\n| sample file | test |\n| --- | --- |\n|  |  |\n| Name | Price |\n| iPhone | 1000 |\n| iPad | 800 |\n| Macbook | 1200 |\n\n## Sheet2\n\n|  |  |\n| --- | --- |\n| other tab |  |\n|  |  |\n| Name | Price |\n| ChatGPT | $20.00 |\n| Claude | $17.00 |\n| Perplexity | $20.00 |\n`;

    it("should convert XLSX document and return expected markdown", async () => {
      const filePath = path.join(samplesDir, "sample.xlsx");
      const fileBuffer = fs.readFileSync(filePath);
      const markdown = await convertDocumentToMarkdown(
        new Uint8Array(fileBuffer),
      );
      expect(markdown).toBe(expectedMarkdown);
    });
  });

  describe("CSV document conversion", () => {
    it("should convert CSV to a markdown table using the extension hint", async () => {
      // CSV has no magic bytes, so the extension hint selects the parser
      const csv = Buffer.from("Name,Price\niPhone,1000\niPad,800\n");
      const markdown = await convertDocumentToMarkdown(
        new Uint8Array(csv),
        ".csv",
      );
      expect(markdown).toBe(
        "| Name | Price |\n| --- | --- |\n| iPhone | 1000 |\n| iPad | 800 |\n",
      );
    });
  });

  describe("unrecognized content", () => {
    it("should reject content that is not a supported document", async () => {
      await expect(
        convertDocumentToMarkdown(new Uint8Array([1, 2, 3, 4, 5])),
      ).rejects.toThrow();
    });
  });
});