Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Fix go-html-to-md on Windows (ENG-3398) (#2082)
tomf4664006b2a5
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.ymlindex d7d36244f..18730fa4b 100644--- a/.github/workflows/test-server-self-host.yml+++ b/.github/workflows/test-server-self-host.yml@@ -70,8 +70,8 @@ jobs:- name: Build go-html-to-mdrun: |go mod tidy- go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go- chmod +x html-to-markdown.so+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go+ chmod +x libhtml-to-markdown.soworking-directory: ./apps/api/sharedLibs/go-html-to-md- name: Build html-transformerrun: |diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 2202fcf27..8c76cd08b 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -87,8 +87,8 @@ jobs:- name: Build go-html-to-mdrun: |go mod tidy- go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go- chmod +x html-to-markdown.so+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go+ chmod +x libhtml-to-markdown.soworking-directory: ./apps/api/sharedLibs/go-html-to-md- name: Build html-transformerrun: |diff --git a/apps/api/Dockerfile b/apps/api/Dockerfileindex 5a8c62464..1297dc47d 100644--- a/apps/api/Dockerfile+++ b/apps/api/Dockerfile@@ -20,8 +20,8 @@ COPY sharedLibs/go-html-to-md /app/sharedLibs/go-html-to-md# Install Go dependencies and build parser libRUN cd /app/sharedLibs/go-html-to-md && \go mod download && \- go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go && \- chmod +x html-to-markdown.so+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go && \+ chmod +x libhtml-to-markdown.so# Install RustFROM rust:1-slim AS rust-base@@ -47,7 +47,7 @@ RUN cd /app/sharedLibs/crawler && \FROM baseCOPY --from=build /app/dist /app/distCOPY --from=prod-deps /app/node_modules /app/node_modules-COPY --from=go-base /app/sharedLibs/go-html-to-md/html-to-markdown.so /app/sharedLibs/go-html-to-md/html-to-markdown.so+COPY --from=go-base /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so /app/sharedLibs/go-html-to-md/libhtml-to-markdown.soCOPY --from=rust-base /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so /app/sharedLibs/html-transformer/target/release/libhtml_transformer.soCOPY --from=rust-base /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so /app/sharedLibs/pdf-parser/target/release/libpdf_parser.soCOPY --from=rust-base /app/sharedLibs/crawler/target/release/libcrawler.so /app/sharedLibs/crawler/target/release/libcrawler.sodiff --git a/apps/api/sharedLibs/go-html-to-md/.gitignore b/apps/api/sharedLibs/go-html-to-md/.gitignoreindex bdab47c67..9e40cf424 100644--- a/apps/api/sharedLibs/go-html-to-md/.gitignore+++ b/apps/api/sharedLibs/go-html-to-md/.gitignore@@ -1,2 +1,2 @@-html-to-markdown.so-html-to-markdown.h\ No newline at end of file+html-to-markdown.*+!html-to-markdown.go\ No newline at end of filediff --git a/apps/api/sharedLibs/go-html-to-md/README.md b/apps/api/sharedLibs/go-html-to-md/README.mdindex 4ad510c30..2994ccf35 100644--- a/apps/api/sharedLibs/go-html-to-md/README.md+++ b/apps/api/sharedLibs/go-html-to-md/README.md@@ -1,7 +1,12 @@-To build the go-html-to-md library, run the following command:+To build the `go-html-to-md` library, run the following command:```bash-cd apps/api/src/lib/go-html-to-md-go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go-chmod +x html-to-markdown.so-```\ No newline at end of file+cd apps/api/sharedLibs/go-html-to-md+go build -o <OUTPUT> -buildmode=c-shared html-to-markdown.go+```++Replace `<OUTPUT>` with the correct filename for your OS:++- Windows β `html-to-markdown.dll`+- Linux β `libhtml-to-markdown.so`+- macOS β `libhtml-to-markdown.dylib`diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 9f1fe14db..ebda6594b 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -1,126 +1,147 @@import { exec } from "child_process";import * as net from "net";+import { basename } from "path";+import { HTML_TO_MARKDOWN_PATH } from "./natives";function waitForPort(port: number, host: string): Promise<void> {- return new Promise((resolve) => {- const checkPort = () => {- const socket = new net.Socket();-- const onError = () => {- socket.destroy();- setTimeout(checkPort, 1000); // Try again in 1 second- };-- socket.once('error', onError);-- socket.connect(port, host, () => {- socket.destroy();- resolve();- });- };-- checkPort();- });+ return new Promise((resolve) => {+ const checkPort = () => {+ const socket = new net.Socket();++ const onError = () => {+ socket.destroy();+ setTimeout(checkPort, 1000); // Try again in 1 second+ };++ socket.once("error", onError);++ socket.connect(port, host, () => {+ socket.destroy();+ resolve();+ });+ };++ checkPort();+ });}if (process.argv.length < 3) {- console.error("Usage: pnpm harness <command...>");- console.error()- console.error("The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above.");- console.error("It also tears down the API and Worker after the command is run.");- process.exit(1);+ console.error("Usage: pnpm harness <command...>");+ console.error();+ console.error(+ "The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above.",+ );+ console.error(+ "It also tears down the API and Worker after the command is run.",+ );+ process.exit(1);}const command = process.argv.slice(2);function execForward(fancyName: string, command: string): Promise<void> {- return new Promise((resolve, reject) => {- const child = exec(command);- let stdoutBuffer = "";- let stderrBuffer = "";- child.stdout?.on("data", (data) => {- stdoutBuffer += data;- while(stdoutBuffer.includes("\n")) {- const split = stdoutBuffer.split("\n");- const line = split[0];- stdoutBuffer = split.slice(1).join("\n");- process.stdout.write(`[${fancyName}] ${line}\n`);- }- });- child.stderr?.on("data", (data) => {- stderrBuffer += data;- while(stderrBuffer.includes("\n")) {- const split = stderrBuffer.split("\n");- const line = split[0];- stderrBuffer = split.slice(1).join("\n");- process.stderr.write(`[${fancyName}] ${line}\n`);- }- });- child.on("close", (code) => {- if (code !== 0) {- reject(new Error(`Command ${JSON.stringify(command)} failed with code ${code}`));- } else {- resolve();- }- });+ return new Promise((resolve, reject) => {+ const child = exec(command);+ let stdoutBuffer = "";+ let stderrBuffer = "";+ child.stdout?.on("data", (data) => {+ stdoutBuffer += data;+ while (stdoutBuffer.includes("\n")) {+ const split = stdoutBuffer.split("\n");+ const line = split[0];+ stdoutBuffer = split.slice(1).join("\n");+ process.stdout.write(`[${fancyName}] ${line}\n`);+ }+ });+ child.stderr?.on("data", (data) => {+ stderrBuffer += data;+ while (stderrBuffer.includes("\n")) {+ const split = stderrBuffer.split("\n");+ const line = split[0];+ stderrBuffer = split.slice(1).join("\n");+ process.stderr.write(`[${fancyName}] ${line}\n`);+ }});+ child.on("close", (code) => {+ if (code !== 0) {+ reject(+ new Error(+ `Command ${JSON.stringify(command)} failed with code ${code}`,+ ),+ );+ } else {+ resolve();+ }+ });+ });}(async () => {- console.log("=== Installing dependencies and building all components...");- await Promise.all([- (async () => {- const install = execForward("api@install", "pnpm install");- await install;-- const build = execForward("api@build", "pnpm build");- await build;- })(),- execForward("sharedLibs/crawler@build", "cd sharedLibs/crawler && cargo build --release"),- execForward("sharedLibs/html-transformer@build", "cd sharedLibs/html-transformer && cargo build --release"),- execForward("sharedLibs/pdf-parser@build", "cd sharedLibs/pdf-parser && cargo build --release"),- (async () => {- const install = execForward("sharedLibs/go-html-to-md@install", "cd sharedLibs/go-html-to-md && go mod tidy");- await install;-- const build = execForward("sharedLibs/go-html-to-md@build", "cd sharedLibs/go-html-to-md && go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go");- await build;- })(),+ console.log("=== Installing dependencies and building all components...");+ await Promise.all([+ (async () => {+ const install = execForward("api@install", "pnpm install");+ await install;++ const build = execForward("api@build", "pnpm build");+ await build;+ })(),+ execForward(+ "sharedLibs/crawler@build",+ "cd sharedLibs/crawler && cargo build --release",+ ),+ execForward(+ "sharedLibs/html-transformer@build",+ "cd sharedLibs/html-transformer && cargo build --release",+ ),+ execForward(+ "sharedLibs/pdf-parser@build",+ "cd sharedLibs/pdf-parser && cargo build --release",+ ),+ (async () => {+ const install = execForward(+ "sharedLibs/go-html-to-md@install",+ "cd sharedLibs/go-html-to-md && go mod tidy",+ );+ await install;++ const build = execForward(+ "sharedLibs/go-html-to-md@build",+ `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,+ );+ await build;+ })(),+ ]);++ console.log("=== Starting API, Worker, and Index Worker...");++ const api = execForward("api", "pnpm start:production:nobuild");+ const worker = execForward("worker", "pnpm worker:production");+ const indexWorker = execForward(+ "index-worker",+ "pnpm index-worker:production",+ );++ try {+ await Promise.race([+ waitForPort(3002, "localhost"),+ new Promise((reject) =>+ setTimeout(() => reject(new Error("API did not start in time")), 10000),+ ),]);- console.log("=== Starting API, Worker, and Index Worker...");-- const api = execForward("api", "pnpm start:production:nobuild");- const worker = execForward("worker", "pnpm worker:production");- const indexWorker = execForward("index-worker", "pnpm index-worker:production");-- try {- await Promise.race([- waitForPort(3002, "localhost"),- new Promise((reject) => setTimeout(() => reject(new Error("API did not start in time")), 10000)),- ]);-- console.log("=== Running command...");-- const cmd = execForward("command", command.join(" "));-- await Promise.race([- cmd,- api,- worker,- indexWorker,- ]);- } finally {- console.log("=== Tearing down API, Worker, and Index Worker...");- exec("pkill -f 'queue-worker.js'");- exec("pkill -f 'index.js'");- exec("pkill -f 'index-worker.js'");- await Promise.all([- api,- worker,- indexWorker,- ]);- }-- console.log("=== Goodbye!");-})();\ No newline at end of file+ console.log("=== Running command...");++ const cmd = execForward("command", command.join(" "));++ await Promise.race([cmd, api, worker, indexWorker]);+ } finally {+ console.log("=== Tearing down API, Worker, and Index Worker...");+ exec("pkill -f 'queue-worker.js'");+ exec("pkill -f 'index.js'");+ exec("pkill -f 'index-worker.js'");+ await Promise.all([api, worker, indexWorker]);+ }++ console.log("=== Goodbye!");+})();diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.tsindex 862988dce..cf4c311d5 100644--- a/apps/api/src/lib/crawler.ts+++ b/apps/api/src/lib/crawler.ts@@ -1,34 +1,28 @@import koffi, { KoffiFunction } from "koffi";-import { join } from "path";import { stat } from "fs/promises";-import { platform } from "os";import type { DenialReason } from "../scraper/WebScraper/crawler";+import { CRAWLER_PATH } from "../natives";// TODO: add a timeout to the Rust transformer-const rustExecutablePath = join(- process.cwd(),- "sharedLibs/crawler/target/release/",- platform() === "darwin" ? "libcrawler.dylib" : "libcrawler.so"-);export type FilterLinksCall = {- links: string[],- limit: number | undefined,- max_depth: number,- base_url: string,- initial_url: string,- regex_on_full_url: boolean,- excludes: string[],- includes: string[],- allow_backward_crawling: boolean,- ignore_robots_txt: boolean,- robots_txt: string,-}+ links: string[];+ limit: number | undefined;+ max_depth: number;+ base_url: string;+ initial_url: string;+ regex_on_full_url: boolean;+ excludes: string[];+ includes: string[];+ allow_backward_crawling: boolean;+ ignore_robots_txt: boolean;+ robots_txt: string;+};export type FilterLinksResult = {- links: string[],- denial_reasons: Map<string, keyof typeof DenialReason>,-}+ links: string[];+ denial_reasons: Map<string, keyof typeof DenialReason>;+};class RustCrawler {private static instance: RustCrawler;@@ -38,19 +32,27 @@ class RustCrawler {private _processSitemap: KoffiFunction;private constructor() {- const lib = koffi.load(rustExecutablePath);+ const lib = koffi.load(CRAWLER_PATH);this._freeString = lib.func("free_string", "void", ["string"]);const cstn = "CString:" + crypto.randomUUID();- const freedResultString = koffi.disposable(cstn, "string", this._freeString);+ const freedResultString = koffi.disposable(+ cstn,+ "string",+ this._freeString,+ );this._filterLinks = lib.func("filter_links", freedResultString, ["string"]);- this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, ["string"]);- this._processSitemap = lib.func("process_sitemap", freedResultString, ["string"]);+ this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, [+ "string",+ ]);+ this._processSitemap = lib.func("process_sitemap", freedResultString, [+ "string",+ ]);}public static async getInstance(): Promise<RustCrawler> {if (!RustCrawler.instance) {try {- await stat(rustExecutablePath);+ await stat(CRAWLER_PATH);} catch (_) {throw Error("Rust crawler shared library not found");}@@ -61,30 +63,35 @@ class RustCrawler {public async filterLinks(call: FilterLinksCall): Promise<FilterLinksResult> {return new Promise<FilterLinksResult>((resolve, reject) => {- this._filterLinks.async(JSON.stringify(call), (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- return reject(new Error(res.split("RUSTFC:ERROR:")[1]));- }+ this._filterLinks.async(+ JSON.stringify(call),+ (err: Error, res: string) => {+ if (err) {+ reject(err);+ } else {+ if (res.startsWith("RUSTFC:ERROR:")) {+ return reject(new Error(res.split("RUSTFC:ERROR:")[1]));+ }- if (res === "RUSTFC:ERROR") {- return reject(new Error("Something went wrong on the Rust side."));- }+ if (res === "RUSTFC:ERROR") {+ return reject(+ new Error("Something went wrong on the Rust side."),+ );+ }- try {- const raw = JSON.parse(res);- const result: FilterLinksResult = {- links: raw.links,- denial_reasons: new Map(Object.entries(raw.denial_reasons)),+ try {+ const raw = JSON.parse(res);+ const result: FilterLinksResult = {+ links: raw.links,+ denial_reasons: new Map(Object.entries(raw.denial_reasons)),+ };+ resolve(result);+ } catch (e) {+ reject(e);}- resolve(result);- } catch (e) {- reject(e);}- }- });+ },+ );});}@@ -142,16 +149,16 @@ class RustCrawler {export async function filterLinks(call: FilterLinksCall,): Promise<FilterLinksResult> {- const converter = await RustCrawler.getInstance();- return await converter.filterLinks(call);+ const converter = await RustCrawler.getInstance();+ return await converter.filterLinks(call);}export async function parseSitemapXml(xmlContent: string): Promise<any> {- const converter = await RustCrawler.getInstance();- return await converter.parseSitemapXml(xmlContent);+ const converter = await RustCrawler.getInstance();+ return await converter.parseSitemapXml(xmlContent);}export async function processSitemap(xmlContent: string): Promise<any> {- const converter = await RustCrawler.getInstance();- return await converter.processSitemap(xmlContent);+ const converter = await RustCrawler.getInstance();+ return await converter.processSitemap(xmlContent);}diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.tsindex a474e34e2..5a21b9b1f 100644--- a/apps/api/src/lib/html-to-markdown.ts+++ b/apps/api/src/lib/html-to-markdown.ts@@ -1,38 +1,34 @@import koffi from "koffi";-import { join } from "path";import "../services/sentry";import * as Sentry from "@sentry/node";import dotenv from "dotenv";import { logger } from "./logger";import { stat } from "fs/promises";+import { HTML_TO_MARKDOWN_PATH } from "../natives";dotenv.config();// TODO: add a timeout to the Go parser-const goExecutablePath = join(- process.cwd(),- "sharedLibs",- "go-html-to-md",- "html-to-markdown.so",-);class GoMarkdownConverter {private static instance: GoMarkdownConverter;private convert: any;private free: any;-+private constructor() {- const lib = koffi.load(goExecutablePath);+ const lib = koffi.load(HTML_TO_MARKDOWN_PATH);this.free = lib.func("FreeCString", "void", ["string"]);const cstn = "CString:" + crypto.randomUUID();const freedResultString = koffi.disposable(cstn, "string", this.free);- this.convert = lib.func("ConvertHTMLToMarkdown", freedResultString, ["string"]);+ this.convert = lib.func("ConvertHTMLToMarkdown", freedResultString, [+ "string",+ ]);}public static async getInstance(): Promise<GoMarkdownConverter> {if (!GoMarkdownConverter.instance) {try {- await stat(goExecutablePath);+ await stat(HTML_TO_MARKDOWN_PATH);} catch (_) {throw Error("Go shared library not found");}@@ -83,7 +79,7 @@ export async function parseMarkdown(} else {logger.warn("Tried to use Go parser, but it doesn't exist in the file system.",- { goExecutablePath },+ { HTML_TO_MARKDOWN_PATH },);}}diff --git a/apps/api/src/lib/html-transformer.ts b/apps/api/src/lib/html-transformer.tsindex 4f05fe78c..247837e25 100644--- a/apps/api/src/lib/html-transformer.ts+++ b/apps/api/src/lib/html-transformer.ts@@ -1,22 +1,14 @@import koffi, { KoffiFunction } from "koffi";-import { join } from "path";import { stat } from "fs/promises";-import { platform } from "os";--// TODO: add a timeout to the Rust transformer-const rustExecutablePath = join(- process.cwd(),- "sharedLibs/html-transformer/target/release/",- platform() === "darwin" ? "libhtml_transformer.dylib" : "libhtml_transformer.so"-);+import { HTML_TRANSFORMER_PATH } from "../natives";type TransformHtmlOptions = {- html: string,- url: string,- include_tags: string[],- exclude_tags: string[],- only_main_content: boolean,- omce_signatures?: string[],+ html: string;+ url: string;+ include_tags: string[];+ exclude_tags: string[];+ only_main_content: boolean;+ omce_signatures?: string[];};class RustHTMLTransformer {@@ -31,23 +23,45 @@ class RustHTMLTransformer {private _extractAttributes: KoffiFunction;private constructor() {- const lib = koffi.load(rustExecutablePath);+ const lib = koffi.load(HTML_TRANSFORMER_PATH);this._freeString = lib.func("free_string", "void", ["string"]);const cstn = "CString:" + crypto.randomUUID();- const freedResultString = koffi.disposable(cstn, "string", this._freeString);- this._extractLinks = lib.func("extract_links", freedResultString, ["string"]);- this._extractImages = lib.func("extract_images", freedResultString, ["string", "string"]);- this._extractBaseHref = lib.func("extract_base_href", freedResultString, ["string", "string"]);- this._extractMetadata = lib.func("extract_metadata", freedResultString, ["string"]);- this._transformHtml = lib.func("transform_html", freedResultString, ["string"]);- this._getInnerJSON = lib.func("get_inner_json", freedResultString, ["string"]);- this._extractAttributes = lib.func("extract_attributes", freedResultString, ["string", "string"]);+ const freedResultString = koffi.disposable(+ cstn,+ "string",+ this._freeString,+ );+ this._extractLinks = lib.func("extract_links", freedResultString, [+ "string",+ ]);+ this._extractImages = lib.func("extract_images", freedResultString, [+ "string",+ "string",+ ]);+ this._extractBaseHref = lib.func("extract_base_href", freedResultString, [+ "string",+ "string",+ ]);+ this._extractMetadata = lib.func("extract_metadata", freedResultString, [+ "string",+ ]);+ this._transformHtml = lib.func("transform_html", freedResultString, [+ "string",+ ]);+ this._getInnerJSON = lib.func("get_inner_json", freedResultString, [+ "string",+ ]);+ this._extractAttributes = lib.func(+ "extract_attributes",+ freedResultString,+ ["string", "string"],+ );}public static async getInstance(): Promise<RustHTMLTransformer> {if (!RustHTMLTransformer.instance) {try {- await stat(rustExecutablePath);+ await stat(HTML_TRANSFORMER_PATH);} catch (_) {throw Error("Rust html-transformer shared library not found");}@@ -110,17 +124,27 @@ class RustHTMLTransformer {public async transformHtml(opts: TransformHtmlOptions): Promise<string> {return new Promise<string>((resolve, reject) => {- this._transformHtml.async(JSON.stringify(opts), (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res === "RUSTFC:ERROR" || res.startsWith("RUSTFC:ERROR:")) {- reject(new Error(res.startsWith("RUSTFC:ERROR:") ? ("Something went wrong on the Rust side. " + res.split("RUSTFC:ERROR:")[1]) : "Something went wrong on the Rust side."));+ this._transformHtml.async(+ JSON.stringify(opts),+ (err: Error, res: string) => {+ if (err) {+ reject(err);} else {- resolve(res);+ if (res === "RUSTFC:ERROR" || res.startsWith("RUSTFC:ERROR:")) {+ reject(+ new Error(+ res.startsWith("RUSTFC:ERROR:")+ ? "Something went wrong on the Rust side. " ++ res.split("RUSTFC:ERROR:")[1]+ : "Something went wrong on the Rust side.",+ ),+ );+ } else {+ resolve(res);+ }}- }- });+ },+ );});}@@ -140,19 +164,31 @@ class RustHTMLTransformer {});}- public async extractAttributes(html: string, options: string): Promise<string> {+ public async extractAttributes(+ html: string,+ options: string,+ ): Promise<string> {return new Promise<string>((resolve, reject) => {- this._extractAttributes.async(html, options, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- reject(new Error("Rust attribute extraction failed: " + res.split("RUSTFC:ERROR:")[1]));+ this._extractAttributes.async(+ html,+ options,+ (err: Error, res: string) => {+ if (err) {+ reject(err);} else {- resolve(res);+ if (res.startsWith("RUSTFC:ERROR:")) {+ reject(+ new Error(+ "Rust attribute extraction failed: " ++ res.split("RUSTFC:ERROR:")[1],+ ),+ );+ } else {+ resolve(res);+ }}- }- });+ },+ );});}}@@ -160,47 +196,47 @@ class RustHTMLTransformer {export async function extractLinks(html: string | null | undefined,): Promise<string[]> {- if (!html) {- return [];- }+ if (!html) {+ return [];+ }- const converter = await RustHTMLTransformer.getInstance();- return await converter.extractLinks(html);+ const converter = await RustHTMLTransformer.getInstance();+ return await converter.extractLinks(html);}export async function extractImages(html: string | null | undefined,- baseUrl: string = ''+ baseUrl: string = "",): Promise<string[]> {- if (!html) {- return [];- }+ if (!html) {+ return [];+ }- const converter = await RustHTMLTransformer.getInstance();- return await converter.extractImages(html, baseUrl);+ const converter = await RustHTMLTransformer.getInstance();+ return await converter.extractImages(html, baseUrl);}export async function extractBaseHref(html: string | null | undefined,- url: string+ url: string,): Promise<string> {- if (!html) {- return url;- }+ if (!html) {+ return url;+ }- const converter = await RustHTMLTransformer.getInstance();- return await converter.extractBaseHref(html, url);+ const converter = await RustHTMLTransformer.getInstance();+ return await converter.extractBaseHref(html, url);}export async function extractMetadata(- html: string | null | undefined,+ html: string | null | undefined,): Promise<any> {- if (!html) {- return [];- }+ if (!html) {+ return [];+ }- const converter = await RustHTMLTransformer.getInstance();- return await converter.extractMetadata(html);+ const converter = await RustHTMLTransformer.getInstance();+ return await converter.extractMetadata(html);}export async function transformHtml(@@ -210,9 +246,7 @@ export async function transformHtml(return await converter.transformHtml(opts);}-export async function getInnerJSON(- html: string,-): Promise<string> {+export async function getInnerJSON(html: string): Promise<string> {const converter = await RustHTMLTransformer.getInstance();return await converter.getInnerJSON(html);}@@ -230,7 +264,7 @@ export type AttributeResult = {export async function extractAttributesRust(html: string,- selectors: AttributeSelector[]+ selectors: AttributeSelector[],): Promise<AttributeResult[]> {if (!html || selectors.length === 0) {return [];@@ -243,6 +277,8 @@ export async function extractAttributesRust(try {return JSON.parse(resultJson);} catch (error) {- throw new Error(`Failed to parse Rust attribute extraction result: ${error}`);+ throw new Error(+ `Failed to parse Rust attribute extraction result: ${error}`,+ );}}diff --git a/apps/api/src/lib/pdf-parser.ts b/apps/api/src/lib/pdf-parser.tsindex 0c24637d6..b930b78a7 100644--- a/apps/api/src/lib/pdf-parser.ts+++ b/apps/api/src/lib/pdf-parser.ts@@ -2,79 +2,73 @@ import koffi, { KoffiFunction } from "koffi";import { join } from "path";import { stat } from "fs/promises";import { platform } from "os";+import { PDF_PARSER_PATH } from "../natives";// TODO: add a timeout to the Rust parser-const rustExecutablePath = join(- process.cwd(),- "sharedLibs/pdf-parser/target/release/",- platform() === "darwin" ? "libpdf_parser.dylib" : "libpdf_parser.so"-);export type PDFMetadata = {- numPages: number;- title?: string;-}+ numPages: number;+ title?: string;+};class RustPDFParser {- private static instance: RustPDFParser;- private _getPDFMetadata: KoffiFunction;+ private static instance: RustPDFParser;+ private _getPDFMetadata: KoffiFunction;++ private constructor() {+ const lib = koffi.load(PDF_PARSER_PATH);+ this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]);+ }- private constructor() {- const lib = koffi.load(rustExecutablePath);- this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]);+ public static async isParserAvailable(): Promise<boolean> {+ if (RustPDFParser.instance) {+ return true;}- public static async isParserAvailable(): Promise<boolean> {- if (RustPDFParser.instance) {- return true;- }+ try {+ await stat(PDF_PARSER_PATH);+ RustPDFParser.instance = new RustPDFParser();+ return true;+ } catch (_) {+ return false;+ }+ }- try {- await stat(rustExecutablePath);- RustPDFParser.instance = new RustPDFParser();- return true;- } catch (_) {- return false;- }+ public static async getInstance(): Promise<RustPDFParser> {+ if (!RustPDFParser.instance) {+ try {+ await stat(PDF_PARSER_PATH);+ } catch (_) {+ throw Error("Rust pdf-parser shared library not found");+ }+ RustPDFParser.instance = new RustPDFParser();}+ return RustPDFParser.instance;+ }- public static async getInstance(): Promise<RustPDFParser> {- if (!RustPDFParser.instance) {+ public async getPDFMetadata(path: string): Promise<PDFMetadata> {+ return new Promise<PDFMetadata>((resolve, reject) => {+ this._getPDFMetadata.async(path, (err: Error, res: string) => {+ if (err) {+ reject(err);+ } else {+ if (res.startsWith("RUSTFC:ERROR:")) {+ reject(new Error(res.replace("RUSTFC:ERROR:", "")));+ } else {try {- await stat(rustExecutablePath);- } catch (_) {- throw Error("Rust pdf-parser shared library not found");+ const metadata = JSON.parse(res) as PDFMetadata;+ resolve(metadata);+ } catch (e) {+ reject(new Error("Failed to parse PDF metadata."));}- RustPDFParser.instance = new RustPDFParser();+ }}- return RustPDFParser.instance;- }-- public async getPDFMetadata(path: string): Promise<PDFMetadata> {- return new Promise<PDFMetadata>((resolve, reject) => {- this._getPDFMetadata.async(path, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- reject(new Error(res.replace("RUSTFC:ERROR:", "")));- } else {- try {- const metadata = JSON.parse(res) as PDFMetadata;- resolve(metadata);- } catch (e) {- reject(new Error("Failed to parse PDF metadata."));- }- }- }- });- });- }+ });+ });+ }}-export async function getPDFMetadata(- path: string,-): Promise<PDFMetadata> {- const converter = await RustPDFParser.getInstance();- return await converter.getPDFMetadata(path);+export async function getPDFMetadata(path: string): Promise<PDFMetadata> {+ const converter = await RustPDFParser.getInstance();+ return await converter.getPDFMetadata(path);}diff --git a/apps/api/src/natives.ts b/apps/api/src/natives.tsnew file mode 100644index 000000000..560a14cb7--- /dev/null+++ b/apps/api/src/natives.ts@@ -0,0 +1,39 @@+import { platform } from "os";+import { join } from "path";++const currentPlatform = platform();+const isWindows = currentPlatform === "win32";++const EXTENSIONS = {+ win32: ".dll",+ darwin: ".dylib",+ default: ".so",+} as const;++function createNativePath(subPath: string, filename: string): string {+ const extension =+ EXTENSIONS[currentPlatform as keyof typeof EXTENSIONS] ??+ EXTENSIONS.default;+ const fullFilename = `${isWindows ? "" : "lib"}${filename}${extension}`;+ return join(process.cwd(), "sharedLibs", subPath, fullFilename);+}++export const HTML_TRANSFORMER_PATH = createNativePath(+ "html-transformer/target/release",+ "html_transformer",+);++export const PDF_PARSER_PATH = createNativePath(+ "pdf-parser/target/release",+ "pdf_parser",+);++export const CRAWLER_PATH = createNativePath(+ "crawler/target/release",+ "crawler",+);++export const HTML_TO_MARKDOWN_PATH = createNativePath(+ "go-html-to-md",+ "html-to-markdown",+);