A fast, helpful, and open-source document parser
wip: starting to add unit tests
Clelia (Astra) Bertelliec7ce3f93c9f
diff --git a/src/core/parser.test.ts b/src/core/parser.test.tsnew file mode 100644index 0000000..99c9262--- /dev/null+++ b/src/core/parser.test.ts@@ -0,0 +1,541 @@+import { vi, describe, it, expect } from "vitest";++const {+ mockPages,+ mockPdfDocument,+ mockParsedPages,+ mockBoundingBoxes,+ mockPdfConvertedResult,+ mockOcrResults,+ mockParseResultJson,+ mockParsedPagesOcr,+ mockParseResultJsonOcr,+ mockTextItems,+ mockParsedPagesWithBoundingBoxes,+ mockParsedPagesWithBoundingBoxesOcr,+} = vi.hoisted(() => {+ const mockPdfConvertedResult = {+ pdfPath: "/tmp/converted.pdf",+ originalExtension: ".docx",+ };++ const mockPdfDocument = {+ numPages: 5,+ data: new TextEncoder().encode("hello world"),+ };++ const mockPages = [+ {+ pageNum: 1,+ width: 612,+ height: 792,+ textItems: [],+ paths: [],+ images: [],+ annotations: [],+ },+ {+ pageNum: 2,+ width: 612,+ height: 792,+ textItems: [],+ paths: [],+ images: [],+ },+ {+ pageNum: 3,+ width: 612,+ height: 792,+ textItems: [],+ paths: [],+ images: [],+ annotations: [],+ },+ {+ pageNum: 4,+ width: 612,+ height: 792,+ textItems: [],+ paths: [],+ images: [],+ },+ {+ pageNum: 5,+ width: 612,+ height: 792,+ textItems: [],+ paths: [],+ images: [],+ annotations: [],+ },+ ];++ const mockBoundingBoxes = [{ x1: 0, y1: 0, x2: 300, y2: 400 }];++ const mockParsedPages = [+ {+ pageNum: 1,+ width: 612,+ height: 792,+ text: "Sample text for page 1",+ textItems: [],+ boundingBoxes: undefined,+ },+ {+ pageNum: 2,+ width: 612,+ height: 792,+ text: "Sample text for page 2",+ textItems: [],+ },+ {+ pageNum: 3,+ width: 612,+ height: 792,+ text: "Sample text for page 3",+ textItems: [],+ boundingBoxes: undefined,+ },+ {+ pageNum: 4,+ width: 612,+ height: 792,+ text: "Sample text for page 4",+ textItems: [],+ },+ {+ pageNum: 5,+ width: 612,+ height: 792,+ text: "Sample text for page 5",+ textItems: [],+ boundingBoxes: undefined,+ },+ ];++ const mockParsedPagesWithBoundingBoxes = mockParsedPages.map((page) => ({+ ...page,+ boundingBoxes: mockBoundingBoxes,+ }));++ const mockOcrResults = [+ { text: "Hello World", bbox: [10, 20, 200, 40], confidence: 0.98 },+ { text: "Sample text", bbox: [10, 50, 180, 70], confidence: 0.85 },+ { text: "Page footer", bbox: [10, 750, 300, 770], confidence: 0.76 },+ ];++ const mockTextItems = mockOcrResults+ .filter((r) => r.confidence > 0.1) // Filter low confidence+ .filter((r) => {+ // Filter out OCR text that already exists in native PDF text+ const ocrText = r.text.trim().toLowerCase();+ return ocrText.length > 0;+ })+ .map((r) => ({+ str: r.text,+ x: r.bbox[0],+ y: r.bbox[1],+ width: r.bbox[2] - r.bbox[0],+ height: r.bbox[3] - r.bbox[1],+ w: r.bbox[2] - r.bbox[0],+ h: r.bbox[3] - r.bbox[1],+ fontName: "OCR",+ fontSize: r.bbox[3] - r.bbox[1],+ }));++ const mockTextItemsJson = mockOcrResults.map((r) => ({+ text: r.text,+ x: r.bbox[0],+ y: r.bbox[1],+ width: r.bbox[2] - r.bbox[0],+ height: r.bbox[3] - r.bbox[1],+ fontName: "OCR",+ fontSize: r.bbox[3] - r.bbox[1],+ }));++ const mockParsedPagesOcr = [+ {+ pageNum: 1,+ width: 612,+ height: 792,+ text: "Sample text for page 1",+ textItems: mockTextItems,+ boundingBoxes: undefined,+ },+ {+ pageNum: 2,+ width: 612,+ height: 792,+ text: "Sample text for page 2",+ textItems: mockTextItems,+ },+ {+ pageNum: 3,+ width: 612,+ height: 792,+ text: "Sample text for page 3",+ textItems: mockTextItems,+ boundingBoxes: undefined,+ },+ {+ pageNum: 4,+ width: 612,+ height: 792,+ text: "Sample text for page 4",+ textItems: mockTextItems,+ },+ {+ pageNum: 5,+ width: 612,+ height: 792,+ text: "Sample text for page 5",+ textItems: mockTextItems,+ boundingBoxes: undefined,+ },+ ];++ const mockParsedPagesWithBoundingBoxesOcr = mockParsedPagesOcr.map((page) => ({+ ...page,+ boundingBoxes: mockBoundingBoxes,+ }));++ const mockParseResultJson = {+ pages: [+ {+ page: 1,+ width: 612,+ height: 792,+ text: "Sample text for page 1",+ textItems: [],+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ {+ page: 2,+ width: 612,+ height: 792,+ text: "Sample text for page 2",+ textItems: [],+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ {+ page: 3,+ width: 612,+ height: 792,+ text: "Sample text for page 3",+ textItems: [],+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ {+ page: 4,+ width: 612,+ height: 792,+ text: "Sample text for page 4",+ textItems: [],+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ {+ page: 5,+ width: 612,+ height: 792,+ text: "Sample text for page 5",+ textItems: [],+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ ],+ };++ const mockParseResultJsonOcr = {+ pages: [+ {+ page: 1,+ width: 612,+ height: 792,+ text: "Sample text for page 1",+ textItems: mockTextItemsJson,+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ {+ page: 2,+ width: 612,+ height: 792,+ text: "Sample text for page 2",+ textItems: mockTextItemsJson,+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ {+ page: 3,+ width: 612,+ height: 792,+ text: "Sample text for page 3",+ textItems: mockTextItemsJson,+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ {+ page: 4,+ width: 612,+ height: 792,+ text: "Sample text for page 4",+ textItems: mockTextItemsJson,+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ {+ page: 5,+ width: 612,+ height: 792,+ text: "Sample text for page 5",+ textItems: mockTextItemsJson,+ boundingBoxes: [{ x1: 0, y1: 0, x2: 300, y2: 400 }],+ },+ ],+ };++ return {+ mockPdfConvertedResult,+ mockPdfDocument,+ mockPages,+ mockParsedPages,+ mockBoundingBoxes,+ mockOcrResults,+ mockParseResultJson,+ mockParsedPagesOcr,+ mockParseResultJsonOcr,+ mockTextItems,+ mockParsedPagesWithBoundingBoxes,+ mockParsedPagesWithBoundingBoxesOcr,+ };+});++import { LiteParse } from "./parser";+import { LiteParseConfig } from "./types";++vi.mock("../conversion/convertToPdf.js", async () => {+ const actual = await vi.importActual<typeof import("../conversion/convertToPdf.js")>(+ "../conversion/convertToPdf.js"+ );+ return {+ ...actual,+ convertToPdf: vi.fn(async () => {+ return mockPdfConvertedResult;+ }),+ cleanupConversionFiles: vi.fn(async () => {}),+ };+});++vi.mock("../engines/pdf/pdfjs.js", async () => {+ const actual =+ await vi.importActual<typeof import("../engines/pdf/pdfjs.js")>("../engines/pdf/pdfjs.js");+ return {+ ...actual,+ PdfJsEngine: vi.fn(+ class {+ constructor() {}++ loadDocument = vi.fn().mockResolvedValue(mockPdfDocument);+ extractAllPages = vi.fn().mockResolvedValue(mockPages);+ renderPageImage = vi.fn(async () => Buffer.from(new Uint8Array([1, 2, 3, 4, 5])));+ close = vi.fn(async () => {});+ }+ ),+ };+});++vi.mock("../engines/ocr/tesseract.js", async () => {+ const actual = await vi.importActual<typeof import("../engines/ocr/tesseract.js")>(+ "../engines/ocr/tesseract.js"+ );+ return {+ ...actual,+ TesseractEngine: vi.fn(+ class {+ constructor() {}++ recognize = vi.fn().mockResolvedValue(mockOcrResults);+ }+ ),+ };+});++vi.mock("../engines/ocr/http-simple.js", async () => {+ const actual = await vi.importActual<typeof import("../engines/ocr/http-simple.js")>(+ "../engines/ocr/http-simple.js"+ );+ return {+ ...actual,+ HttpOcrEngine: vi.fn(+ class {+ private url;++ constructor(url: string) {+ this.url = url;+ if (this.url != ("http://localhost:8000" as string)) {+ // works as an assertion that the URL is being passed to the class+ // at init time+ throw new Error("cannot accept a URL that is not http://localhost:8000");+ }+ }++ recognize = vi.fn().mockResolvedValue(mockOcrResults);+ }+ ),+ };+});++vi.mock("../processing/grid.js", async () => {+ const actual =+ await vi.importActual<typeof import("../processing/grid.js")>("../processing/grid.js");+ return {+ ...actual,+ projectPagesToGrid: vi+ .fn()+ // these get modified, so we need to pass a deep copy of the array instead of the original one+ .mockImplementationOnce(() => structuredClone(mockParsedPages)) // no ocr - text+ .mockImplementationOnce(() => structuredClone(mockParsedPages)) // no ocr - json+ .mockImplementationOnce(() => structuredClone(mockParsedPagesOcr)) // ocr (tesseract) - text+ .mockImplementationOnce(() => structuredClone(mockParsedPagesOcr)) // ocr (tesseract) - json+ .mockImplementationOnce(() => structuredClone(mockParsedPagesOcr)) // ocr (http) - text+ .mockImplementationOnce(() => structuredClone(mockParsedPagesOcr)) // ocr (http) - json+ .mockImplementationOnce(() => structuredClone(mockParsedPages)), //no bounding boxes+ };+});++vi.mock("../processing/bbox.js", async () => {+ const actual =+ await vi.importActual<typeof import("../processing/bbox.js")>("../processing/bbox.js");+ return {+ ...actual,+ buildBoundingBoxes: vi.fn().mockReturnValue(mockBoundingBoxes),+ };+});++describe("Parse tests", () => {+ it("test parse without OCR and text format", async () => {+ const config: Partial<LiteParseConfig> = { ocrEnabled: false, outputFormat: "text" };+ const liteparse = new LiteParse(config);+ expect(liteparse.getConfig().ocrEnabled).toBeFalsy();+ expect(liteparse.getConfig().outputFormat).toBe("text");+ const result = await liteparse.parse("/tmp/test.docx");+ expect(result.json).toBeUndefined();+ expect(result.text).toBe(mockParsedPages.map((p) => p.text).join("\n\n"));+ expect(result.pages).toStrictEqual(mockParsedPagesWithBoundingBoxes);+ });++ it("test parse without OCR and json format", async () => {+ const config: Partial<LiteParseConfig> = { ocrEnabled: false, outputFormat: "json" };+ const liteparse = new LiteParse(config);+ expect(liteparse.getConfig().ocrEnabled).toBeFalsy();+ expect(liteparse.getConfig().outputFormat).toBe("json");+ const result = await liteparse.parse("/tmp/test.docx");+ expect(result.json).toBeDefined();+ expect(result.json).toStrictEqual(mockParseResultJson);+ expect(result.text).toBe(mockParsedPages.map((p) => p.text).join("\n\n"));+ expect(result.pages).toStrictEqual(mockParsedPagesWithBoundingBoxes);+ });++ it("test parse with OCR (tesseract) and text format", async () => {+ const config: Partial<LiteParseConfig> = {+ ocrEnabled: true,+ outputFormat: "text",+ ocrServerUrl: undefined,+ };+ const liteparse = new LiteParse(config);+ expect(liteparse.getConfig().ocrEnabled).toBeTruthy();+ expect(liteparse.getConfig().outputFormat).toBe("text");+ expect(liteparse.getConfig().ocrServerUrl).toBeUndefined();+ const result = await liteparse.parse("/tmp/test.docx");+ console.log(result.pages.at(0)?.textItems);+ expect(+ result.pages.filter((page) => {+ return page.textItems.length > 0;+ }).length+ ).toBeGreaterThan(0);+ expect(result.json).toBeUndefined();+ expect(result.text).toBe(mockParsedPages.map((p) => p.text).join("\n\n"));+ expect(result.pages).toStrictEqual(mockParsedPagesWithBoundingBoxesOcr);+ });++ it("test parse with OCR (tesseract) and json format", async () => {+ const config: Partial<LiteParseConfig> = {+ ocrEnabled: true,+ outputFormat: "json",+ ocrServerUrl: undefined,+ };+ const liteparse = new LiteParse(config);+ expect(liteparse.getConfig().ocrEnabled).toBeTruthy();+ expect(liteparse.getConfig().outputFormat).toBe("json");+ expect(liteparse.getConfig().ocrServerUrl).toBeUndefined();+ const result = await liteparse.parse("/tmp/test.docx");+ expect(+ result.pages.filter((page) => {+ return page.textItems.length > 0;+ }).length+ ).toBe(result.pages.length);+ expect(result.pages.at(0)!.textItems).toStrictEqual(mockTextItems);+ expect(result.json).toBeDefined();+ expect(result.json).toStrictEqual(mockParseResultJsonOcr);+ expect(result.text).toBe(mockParsedPages.map((p) => p.text).join("\n\n"));+ expect(result.pages).toStrictEqual(mockParsedPagesWithBoundingBoxesOcr);+ });++ it("test parse with OCR (http) and text format", async () => {+ const config: Partial<LiteParseConfig> = {+ ocrEnabled: true,+ outputFormat: "text",+ ocrServerUrl: "http://localhost:8000",+ };+ const liteparse = new LiteParse(config);+ expect(liteparse.getConfig().ocrEnabled).toBeTruthy();+ expect(liteparse.getConfig().outputFormat).toBe("text");+ expect(liteparse.getConfig().ocrServerUrl).toBe("http://localhost:8000");+ const result = await liteparse.parse("/tmp/test.docx");+ console.log(result.pages.at(0)?.textItems);+ expect(+ result.pages.filter((page) => {+ return page.textItems.length > 0;+ }).length+ ).toBe(result.pages.length);+ expect(result.pages.at(0)!.textItems).toStrictEqual(mockTextItems);+ expect(result.json).toBeUndefined();+ expect(result.text).toBe(mockParsedPages.map((p) => p.text).join("\n\n"));+ expect(result.pages).toStrictEqual(mockParsedPagesWithBoundingBoxesOcr);+ });++ it("test parse with OCR (http) and json format", async () => {+ const config: Partial<LiteParseConfig> = {+ ocrEnabled: true,+ outputFormat: "json",+ ocrServerUrl: "http://localhost:8000",+ };+ const liteparse = new LiteParse(config);+ expect(liteparse.getConfig().ocrEnabled).toBeTruthy();+ expect(liteparse.getConfig().outputFormat).toBe("json");+ expect(liteparse.getConfig().ocrServerUrl).toBe("http://localhost:8000");+ const result = await liteparse.parse("/tmp/test.docx");+ expect(+ result.pages.filter((page) => {+ return page.textItems.length > 0;+ }).length+ ).toBe(result.pages.length);+ expect(result.pages.at(0)!.textItems).toStrictEqual(mockTextItems);+ expect(result.json).toBeDefined();+ expect(result.json).toStrictEqual(mockParseResultJsonOcr);+ expect(result.text).toBe(mockParsedPages.map((p) => p.text).join("\n\n"));+ expect(result.pages).toStrictEqual(mockParsedPagesWithBoundingBoxesOcr);+ });++ it("test without bounding boxes", async () => {+ const config: Partial<LiteParseConfig> = {+ ocrEnabled: false,+ outputFormat: "text",+ preciseBoundingBox: false,+ };+ const liteparse = new LiteParse(config);+ expect(liteparse.getConfig().preciseBoundingBox).toBeFalsy();+ const result = await liteparse.parse("/tmp/test.docx");+ console.log(result.pages.at(0)!.boundingBoxes);+ expect(+ result.pages.filter((page) => {+ return typeof page.boundingBoxes != "undefined";+ }).length+ ).toBe(0);+ });+});