A fast, helpful, and open-source document parser
added dedicated browser test, delete old one
Logan Markewich1451ce9513cd
diff --git a/.github/workflows/browser-compat.yml b/.github/workflows/browser-compat.ymldeleted file mode 100644index eda7518..0000000--- a/.github/workflows/browser-compat.yml+++ /dev/null@@ -1,39 +0,0 @@-name: Browser Compatibility--on:- push:- branches: [main]- pull_request:- branches: [main]--jobs:- browser-bundle:- runs-on: ubuntu-latest- name: Verify browser bundle builds-- steps:- - uses: actions/checkout@v4-- - uses: actions/setup-node@v4- with:- node-version: 22- cache: "npm"-- - run: npm ci-- - name: Build browser bundle with Vite- run: npx vite build --config scripts/browser-compat/vite.config.ts-- - name: Verify bundle was produced- run: |- if [ ! -d dist-browser-compat ]; then- echo "❌ dist-browser-compat directory not found"- exit 1- fi- JS_FILES=$(find dist-browser-compat -name "*.js" | wc -l)- if [ "$JS_FILES" -eq 0 ]; then- echo "❌ No JS files in dist-browser-compat"- exit 1- fi- echo "✅ Browser bundle produced ($JS_FILES JS file(s))"- du -sh dist-browser-compat/diff --git a/.github/workflows/ci-wasm.yml b/.github/workflows/ci-wasm.ymlindex dd5544a..38d1c49 100644--- a/.github/workflows/ci-wasm.yml+++ b/.github/workflows/ci-wasm.yml@@ -6,12 +6,14 @@ on:paths:- 'crates/**'- 'packages/wasm/**'+ - 'scripts/browser-compat/**'- '.github/workflows/ci-wasm.yml'pull_request:branches: [main]paths:- 'crates/**'- 'packages/wasm/**'+ - 'scripts/browser-compat/**'- '.github/workflows/ci-wasm.yml'env:@@ -63,3 +65,29 @@ jobs:name: wasm-pkgpath: packages/wasm/pkg/if-no-files-found: error++ browser-test:+ runs-on: ubuntu-latest+ needs: build+ name: Browser PDF parse test++ steps:+ - uses: actions/checkout@v4++ - uses: actions/setup-node@v4+ with:+ node-version: 20++ - name: Download WASM artifact+ uses: actions/download-artifact@v4+ with:+ name: wasm-pkg+ path: packages/wasm/pkg/++ - name: Install Playwright+ run: |+ npm install playwright@latest+ npx playwright install --with-deps chromium++ - name: Run browser parse test+ run: node scripts/browser-compat/wasm-test.mjsdiff --git a/scripts/browser-compat/entry.ts b/scripts/browser-compat/entry.tsdeleted file mode 100644index a744375..0000000--- a/scripts/browser-compat/entry.ts+++ /dev/null@@ -1,10 +0,0 @@-// Minimal entry point that exercises the LiteParse browser import path.-// If this bundles successfully with Vite, the browser compat config works.-import { LiteParse } from "../../src/lib.js";-export type { ParseResult, LiteParseConfig } from "../../src/lib.js";--// Instantiate to verify the constructor doesn't crash at bundle time-const parser = new LiteParse({ ocrEnabled: false, outputFormat: "text" });--// Export so Vite doesn't tree-shake it away-export { parser, LiteParse };diff --git a/scripts/browser-compat/stubs/convertToPdf.ts b/scripts/browser-compat/stubs/convertToPdf.tsdeleted file mode 100644index 468921b..0000000--- a/scripts/browser-compat/stubs/convertToPdf.ts+++ /dev/null@@ -1,18 +0,0 @@-// Browser stub — only PDF input is supported, no file conversion.-export const officeExtensions: string[] = [];-export const spreadsheetExtensions: string[] = [];-export const imageExtensions: string[] = [];-export const htmlExtensions: string[] = [];-export async function convertToPdf() {- throw new Error("File conversion is not supported in browser environments.");-}-export async function convertBufferToPdf() {- throw new Error("File conversion is not supported in browser environments.");-}-export async function cleanupConversionFiles() {}-export async function guessExtensionFromBuffer(data: Uint8Array) {- if (data[0] === 0x25 && data[1] === 0x50 && data[2] === 0x44 && data[3] === 0x46) {- return ".pdf";- }- return null;-}diff --git a/scripts/browser-compat/stubs/empty.ts b/scripts/browser-compat/stubs/empty.tsdeleted file mode 100644index 6396001..0000000--- a/scripts/browser-compat/stubs/empty.ts+++ /dev/null@@ -1,2 +0,0 @@-// Empty stub for Node built-in modules that aren't available in browsers.-export default {};diff --git a/scripts/browser-compat/stubs/file-type.ts b/scripts/browser-compat/stubs/file-type.tsdeleted file mode 100644index ac83313..0000000--- a/scripts/browser-compat/stubs/file-type.ts+++ /dev/null@@ -1,10 +0,0 @@-// Browser stub for file-type. Only PDF magic-byte detection is needed.-export async function fileTypeFromBuffer(buf: Uint8Array) {- if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) {- return { ext: "pdf", mime: "application/pdf" };- }- return undefined;-}-export async function fileTypeFromFile() {- return undefined;-}diff --git a/scripts/browser-compat/stubs/gridDebugLogger.ts b/scripts/browser-compat/stubs/gridDebugLogger.tsdeleted file mode 100644index 5b39123..0000000--- a/scripts/browser-compat/stubs/gridDebugLogger.ts+++ /dev/null@@ -1,36 +0,0 @@-// Browser stub — debug logging is a no-op.-export interface GridDebugConfig { enabled: boolean; }-export const DEFAULT_DEBUG_CONFIG = { enabled: false };-export interface RenderedSegment { lineIndex: number; gridCol: number; text: string; snap: string; }-export interface RenderTraceContext {}-export interface VisualizerPageData { pageNum: number; segments: RenderedSegment[]; rawLines: string[]; }--export class GridDebugLogger {- config = { enabled: false };- get enabled() { return false; }- get shouldVisualize() { return false; }- get visualizerPages(): VisualizerPageData[] { return []; }- get debugConfig() { return this.config; }- matchesBbox() { return false; }- setPage() {}- logBlock() {}- logFlowingBlock() {}- logStructuredBlock() {}- logFlowingLine() {}- logAnchors() {}- logSnapAssignment() {}- captureRender() {}- captureRawLines() {}- logRender() {}- logForwardAnchor() {}- logDuplicateResolution() {}- logBlockContext() {}- logRenderTrace() {}- logForwardAnchorMutation() {}- logLineComposition() {}- flushSync() {}- async flush() {}-}--export const NOOP_LOGGER = new GridDebugLogger();-export function createGridDebugLogger() { return NOOP_LOGGER; }diff --git a/scripts/browser-compat/stubs/gridVisualizer.ts b/scripts/browser-compat/stubs/gridVisualizer.tsdeleted file mode 100644index c490e05..0000000--- a/scripts/browser-compat/stubs/gridVisualizer.ts+++ /dev/null@@ -1,3 +0,0 @@-// Browser stub — grid visualization is a no-op.-export async function renderGridVisualization() {}-export async function renderAllVisualizations() { return []; }diff --git a/scripts/browser-compat/stubs/http-simple.ts b/scripts/browser-compat/stubs/http-simple.tsdeleted file mode 100644index 7b1400c..0000000--- a/scripts/browser-compat/stubs/http-simple.ts+++ /dev/null@@ -1,9 +0,0 @@-// Browser stub — HTTP OCR is not available, use Tesseract.js instead.-export class HttpOcrEngine {- name = "http-ocr";- constructor() {- throw new Error("HTTP OCR engine is not available in browser environments.");- }- async recognize() { throw new Error("Not available in browser."); }- async recognizeBatch() { throw new Error("Not available in browser."); }-}diff --git a/scripts/browser-compat/stubs/pdfium-renderer.ts b/scripts/browser-compat/stubs/pdfium-renderer.tsdeleted file mode 100644index 42c54a0..0000000--- a/scripts/browser-compat/stubs/pdfium-renderer.ts+++ /dev/null@@ -1,11 +0,0 @@-// Browser stub for PdfiumRenderer.-// A real browser app would replace this with a PDF.js canvas renderer.-// This stub is enough to verify the bundle builds.-export class PdfiumRenderer {- async init() {}- async loadDocument() {}- closeDocument() {}- async renderPageToBuffer() { return new Uint8Array(0); }- async extractImageBounds() { return []; }- async close() {}-}diff --git a/scripts/browser-compat/stubs/pdfjsImporter.ts b/scripts/browser-compat/stubs/pdfjsImporter.tsdeleted file mode 100644index fb3cb60..0000000--- a/scripts/browser-compat/stubs/pdfjsImporter.ts+++ /dev/null@@ -1,11 +0,0 @@-// Browser stub for pdfjsImporter.-// In a real browser app, this would configure PDF.js worker and asset URLs.-// @ts-expect-error vendored ESM build has no types-import * as pdfjs from "../../../src/vendor/pdfjs/pdf.mjs";--export async function importPdfJs() {- return {- fn: (pdfjs as { getDocument: unknown }).getDocument,- dir: new URL("../../../src/vendor/pdfjs", import.meta.url).href,- };-}diff --git a/scripts/browser-compat/vite.config.ts b/scripts/browser-compat/vite.config.tsdeleted file mode 100644index 3e9d6ab..0000000--- a/scripts/browser-compat/vite.config.ts+++ /dev/null@@ -1,100 +0,0 @@-/**- * Minimal Vite config that verifies LiteParse can be bundled for the browser.- * This mirrors the recommended configuration from the README.- *- * Run: npx vite build --config scripts/browser-compat/vite.config.ts- */-import { defineConfig, type Plugin } from "vite";-import { resolve, dirname } from "node:path";-import { fileURLToPath } from "node:url";--const root = resolve(dirname(fileURLToPath(import.meta.url)), "../..");-const here = (p: string) => resolve(root, p);--// Node-only source files that must be replaced with browser stubs.-const FILE_REDIRECTS: Array<{ match: RegExp; target: string }> = [- {- match: /\/engines\/pdf\/pdfium-renderer(\.js|\.ts)?$/,- target: here("scripts/browser-compat/stubs/pdfium-renderer.ts"),- },- {- match: /\/engines\/pdf\/pdfjsImporter(\.js|\.ts)?$/,- target: here("scripts/browser-compat/stubs/pdfjsImporter.ts"),- },- {- match: /\/engines\/ocr\/http-simple(\.js|\.ts)?$/,- target: here("scripts/browser-compat/stubs/http-simple.ts"),- },- {- match: /\/conversion\/convertToPdf(\.js|\.ts)?$/,- target: here("scripts/browser-compat/stubs/convertToPdf.ts"),- },- {- match: /\/processing\/gridDebugLogger(\.js|\.ts)?$/,- target: here("scripts/browser-compat/stubs/gridDebugLogger.ts"),- },- {- match: /\/processing\/gridVisualizer(\.js|\.ts)?$/,- target: here("scripts/browser-compat/stubs/gridVisualizer.ts"),- },-];--function liteparseNodeRedirects(): Plugin {- return {- name: "liteparse-node-redirects",- enforce: "pre",- async resolveId(source, importer) {- if (!importer) return null;- const importerDir = dirname(importer);- const absolutePath = source.startsWith(".")- ? resolve(importerDir, source)- : source;- for (const { match, target } of FILE_REDIRECTS) {- if (match.test(absolutePath) || match.test(source)) {- return target;- }- }- return null;- },- };-}--export default defineConfig({- root: here("scripts/browser-compat"),- build: {- outDir: resolve(root, "dist-browser-compat"),- emptyOutDir: true,- target: "es2022",- rollupOptions: {- input: here("scripts/browser-compat/entry.ts"),- },- },- server: {- fs: { allow: [root] },- },- optimizeDeps: {- include: ["tesseract.js"],- },- plugins: [liteparseNodeRedirects()],- resolve: {- alias: [- { find: "node:fs/promises", replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: "node:fs", replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: "node:url", replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: "node:path", replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: "node:os", replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: "node:child_process", replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: /^fs$/, replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: /^fs\/promises$/, replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: /^path$/, replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: /^os$/, replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: /^child_process$/, replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: "form-data", replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: "axios", replacement: here("scripts/browser-compat/stubs/empty.ts") },- { find: "file-type", replacement: here("scripts/browser-compat/stubs/file-type.ts") },- ],- },- define: {- "process.env.NODE_ENV": JSON.stringify("production"),- },-});diff --git a/scripts/browser-compat/wasm-test.html b/scripts/browser-compat/wasm-test.htmlnew file mode 100644index 0000000..88445f5--- /dev/null+++ b/scripts/browser-compat/wasm-test.html@@ -0,0 +1,53 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8" />+ <title>LiteParse WASM Browser Test</title>+</head>+<body>+ <div id="status">loading</div>+ <div id="result" style="display:none"></div>+ <div id="error" style="display:none"></div>++ <script type="module">+ const status = document.getElementById("status");+ const resultEl = document.getElementById("result");+ const errorEl = document.getElementById("error");++ try {+ status.textContent = "loading wasm";+ const { default: init, LiteParse } = await import("/packages/wasm/pkg/liteparse_wasm.js");+ await init();++ status.textContent = "creating parser";+ const parser = new LiteParse({ ocrEnabled: false, outputFormat: "text", quiet: true });++ status.textContent = "fetching pdf";+ const resp = await fetch("/demo/docs/apple-10k-2024.pdf");+ if (!resp.ok) throw new Error(`Failed to fetch PDF: ${resp.status}`);+ const bytes = new Uint8Array(await resp.arrayBuffer());++ status.textContent = "parsing";+ const result = await parser.parse(bytes);++ const pageCount = result.pages.length;+ const textLength = result.text.length;+ const hasText = textLength > 100;++ if (pageCount === 0) throw new Error("No pages parsed");+ if (!hasText) throw new Error(`Text too short: ${textLength} chars`);++ status.textContent = "done";+ resultEl.style.display = "block";+ resultEl.setAttribute("data-pages", pageCount);+ resultEl.setAttribute("data-text-length", textLength);+ resultEl.textContent = `OK: ${pageCount} pages, ${textLength} chars`;+ } catch (err) {+ status.textContent = "error";+ errorEl.style.display = "block";+ errorEl.textContent = err.message || String(err);+ console.error(err);+ }+ </script>+</body>+</html>diff --git a/scripts/browser-compat/wasm-test.mjs b/scripts/browser-compat/wasm-test.mjsnew file mode 100644index 0000000..01128df--- /dev/null+++ b/scripts/browser-compat/wasm-test.mjs@@ -0,0 +1,112 @@+/**+ * Playwright test that loads the LiteParse WASM module in a real browser+ * and verifies it can parse a PDF end-to-end.+ *+ * Usage: node scripts/browser-compat/wasm-test.mjs+ *+ * Requires: playwright (npx playwright install chromium)+ * Expects: packages/wasm/pkg/ to contain the built WASM files+ * demo/docs/apple-10k-2024.pdf to exist+ */++import { createServer } from "node:http";+import { readFile } from "node:fs/promises";+import { resolve, extname } from "node:path";+import { fileURLToPath } from "node:url";+import { chromium } from "playwright";++const ROOT = resolve(fileURLToPath(import.meta.url), "../../..");++const MIME_TYPES = {+ ".html": "text/html",+ ".js": "application/javascript",+ ".wasm": "application/wasm",+ ".pdf": "application/pdf",+ ".json": "application/json",+};++function startServer() {+ return new Promise((resolvePromise) => {+ const server = createServer(async (req, res) => {+ const url = new URL(req.url, "http://localhost");+ const filePath = resolve(ROOT, "." + url.pathname);++ // Basic security: don't serve outside ROOT+ if (!filePath.startsWith(ROOT)) {+ res.writeHead(403);+ res.end("Forbidden");+ return;+ }++ try {+ const data = await readFile(filePath);+ const ext = extname(filePath);+ res.writeHead(200, {+ "Content-Type": MIME_TYPES[ext] || "application/octet-stream",+ "Cross-Origin-Opener-Policy": "same-origin",+ "Cross-Origin-Embedder-Policy": "require-corp",+ });+ res.end(data);+ } catch {+ res.writeHead(404);+ res.end("Not found");+ }+ });++ server.listen(0, "127.0.0.1", () => {+ const port = server.address().port;+ resolvePromise({ server, port });+ });+ });+}++async function main() {+ const { server, port } = await startServer();+ const baseUrl = `http://127.0.0.1:${port}`;+ console.log(`Static server listening on ${baseUrl}`);++ let browser;+ try {+ browser = await chromium.launch();+ const page = await browser.newPage();++ // Collect console errors+ const errors = [];+ page.on("pageerror", (err) => errors.push(err.message));++ console.log("Navigating to test page...");+ await page.goto(`${baseUrl}/scripts/browser-compat/wasm-test.html`);++ // Wait for either #result or #error to appear (up to 120s for large PDFs)+ await page.waitForSelector("#result[style*='block'], #error[style*='block']", {+ timeout: 120_000,+ });++ const errorText = await page.locator("#error").textContent();+ if (errorText) {+ console.error(`FAIL: ${errorText}`);+ if (errors.length) console.error("Console errors:", errors);+ process.exit(1);+ }++ const resultText = await page.locator("#result").textContent();+ const pages = await page.locator("#result").getAttribute("data-pages");+ const textLength = await page.locator("#result").getAttribute("data-text-length");++ console.log(`PASS: ${resultText}`);+ console.log(` Pages: ${pages}`);+ console.log(` Text length: ${textLength}`);++ if (errors.length) {+ console.warn("Browser console errors (non-fatal):", errors);+ }+ } finally {+ if (browser) await browser.close();+ server.close();+ }+}++main().catch((err) => {+ console.error("Test runner failed:", err);+ process.exit(1);+});