Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Fix go-html-to-md on Windows (ENG-3398) (#2082)

tomf4664006b2a5
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml
index d7d36244f..18730fa4b 100644
--- a/.github/workflows/test-server-self-host.yml
+++ b/.github/workflows/test-server-self-host.yml
@@ -70,8 +70,8 @@ jobs:
- name: Build go-html-to-md
run: |
go mod tidy
- go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go
- chmod +x html-to-markdown.so
+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
+ chmod +x libhtml-to-markdown.so
working-directory: ./apps/api/sharedLibs/go-html-to-md
- name: Build html-transformer
run: |
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 2202fcf27..8c76cd08b 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -87,8 +87,8 @@ jobs:
- name: Build go-html-to-md
run: |
go mod tidy
- go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go
- chmod +x html-to-markdown.so
+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
+ chmod +x libhtml-to-markdown.so
working-directory: ./apps/api/sharedLibs/go-html-to-md
- name: Build html-transformer
run: |
diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
index 5a8c62464..1297dc47d 100644
--- a/apps/api/Dockerfile
+++ b/apps/api/Dockerfile
@@ -20,8 +20,8 @@ COPY sharedLibs/go-html-to-md /app/sharedLibs/go-html-to-md
# Install Go dependencies and build parser lib
RUN cd /app/sharedLibs/go-html-to-md && \
go mod download && \
- go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go && \
- chmod +x html-to-markdown.so
+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go && \
+ chmod +x libhtml-to-markdown.so
# Install Rust
FROM rust:1-slim AS rust-base
@@ -47,7 +47,7 @@ RUN cd /app/sharedLibs/crawler && \
FROM base
COPY --from=build /app/dist /app/dist
COPY --from=prod-deps /app/node_modules /app/node_modules
-COPY --from=go-base /app/sharedLibs/go-html-to-md/html-to-markdown.so /app/sharedLibs/go-html-to-md/html-to-markdown.so
+COPY --from=go-base /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so
COPY --from=rust-base /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so
COPY --from=rust-base /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so
COPY --from=rust-base /app/sharedLibs/crawler/target/release/libcrawler.so /app/sharedLibs/crawler/target/release/libcrawler.so
diff --git a/apps/api/sharedLibs/go-html-to-md/.gitignore b/apps/api/sharedLibs/go-html-to-md/.gitignore
index bdab47c67..9e40cf424 100644
--- a/apps/api/sharedLibs/go-html-to-md/.gitignore
+++ b/apps/api/sharedLibs/go-html-to-md/.gitignore
@@ -1,2 +1,2 @@
-html-to-markdown.so
-html-to-markdown.h
\ No newline at end of file
+html-to-markdown.*
+!html-to-markdown.go
\ No newline at end of file
diff --git a/apps/api/sharedLibs/go-html-to-md/README.md b/apps/api/sharedLibs/go-html-to-md/README.md
index 4ad510c30..2994ccf35 100644
--- a/apps/api/sharedLibs/go-html-to-md/README.md
+++ b/apps/api/sharedLibs/go-html-to-md/README.md
@@ -1,7 +1,12 @@
-To build the go-html-to-md library, run the following command:
+To build the `go-html-to-md` library, run the following command:
```bash
-cd apps/api/src/lib/go-html-to-md
-go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go
-chmod +x html-to-markdown.so
-```
\ No newline at end of file
+cd apps/api/sharedLibs/go-html-to-md
+go build -o <OUTPUT> -buildmode=c-shared html-to-markdown.go
+```
+
+Replace `<OUTPUT>` with the correct filename for your OS:
+
+- Windows β†’ `html-to-markdown.dll`
+- Linux β†’ `libhtml-to-markdown.so`
+- macOS β†’ `libhtml-to-markdown.dylib`
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 9f1fe14db..ebda6594b 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -1,126 +1,147 @@
import { exec } from "child_process";
import * as net from "net";
+import { basename } from "path";
+import { HTML_TO_MARKDOWN_PATH } from "./natives";
function waitForPort(port: number, host: string): Promise<void> {
- return new Promise((resolve) => {
- const checkPort = () => {
- const socket = new net.Socket();
-
- const onError = () => {
- socket.destroy();
- setTimeout(checkPort, 1000); // Try again in 1 second
- };
-
- socket.once('error', onError);
-
- socket.connect(port, host, () => {
- socket.destroy();
- resolve();
- });
- };
-
- checkPort();
- });
+ return new Promise((resolve) => {
+ const checkPort = () => {
+ const socket = new net.Socket();
+
+ const onError = () => {
+ socket.destroy();
+ setTimeout(checkPort, 1000); // Try again in 1 second
+ };
+
+ socket.once("error", onError);
+
+ socket.connect(port, host, () => {
+ socket.destroy();
+ resolve();
+ });
+ };
+
+ checkPort();
+ });
}
if (process.argv.length < 3) {
- console.error("Usage: pnpm harness <command...>");
- console.error()
- console.error("The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above.");
- console.error("It also tears down the API and Worker after the command is run.");
- process.exit(1);
+ console.error("Usage: pnpm harness <command...>");
+ console.error();
+ console.error(
+ "The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above.",
+ );
+ console.error(
+ "It also tears down the API and Worker after the command is run.",
+ );
+ process.exit(1);
}
const command = process.argv.slice(2);
function execForward(fancyName: string, command: string): Promise<void> {
- return new Promise((resolve, reject) => {
- const child = exec(command);
- let stdoutBuffer = "";
- let stderrBuffer = "";
- child.stdout?.on("data", (data) => {
- stdoutBuffer += data;
- while(stdoutBuffer.includes("\n")) {
- const split = stdoutBuffer.split("\n");
- const line = split[0];
- stdoutBuffer = split.slice(1).join("\n");
- process.stdout.write(`[${fancyName}] ${line}\n`);
- }
- });
- child.stderr?.on("data", (data) => {
- stderrBuffer += data;
- while(stderrBuffer.includes("\n")) {
- const split = stderrBuffer.split("\n");
- const line = split[0];
- stderrBuffer = split.slice(1).join("\n");
- process.stderr.write(`[${fancyName}] ${line}\n`);
- }
- });
- child.on("close", (code) => {
- if (code !== 0) {
- reject(new Error(`Command ${JSON.stringify(command)} failed with code ${code}`));
- } else {
- resolve();
- }
- });
+ return new Promise((resolve, reject) => {
+ const child = exec(command);
+ let stdoutBuffer = "";
+ let stderrBuffer = "";
+ child.stdout?.on("data", (data) => {
+ stdoutBuffer += data;
+ while (stdoutBuffer.includes("\n")) {
+ const split = stdoutBuffer.split("\n");
+ const line = split[0];
+ stdoutBuffer = split.slice(1).join("\n");
+ process.stdout.write(`[${fancyName}] ${line}\n`);
+ }
+ });
+ child.stderr?.on("data", (data) => {
+ stderrBuffer += data;
+ while (stderrBuffer.includes("\n")) {
+ const split = stderrBuffer.split("\n");
+ const line = split[0];
+ stderrBuffer = split.slice(1).join("\n");
+ process.stderr.write(`[${fancyName}] ${line}\n`);
+ }
});
+ child.on("close", (code) => {
+ if (code !== 0) {
+ reject(
+ new Error(
+ `Command ${JSON.stringify(command)} failed with code ${code}`,
+ ),
+ );
+ } else {
+ resolve();
+ }
+ });
+ });
}
(async () => {
- console.log("=== Installing dependencies and building all components...");
- await Promise.all([
- (async () => {
- const install = execForward("api@install", "pnpm install");
- await install;
-
- const build = execForward("api@build", "pnpm build");
- await build;
- })(),
- execForward("sharedLibs/crawler@build", "cd sharedLibs/crawler && cargo build --release"),
- execForward("sharedLibs/html-transformer@build", "cd sharedLibs/html-transformer && cargo build --release"),
- execForward("sharedLibs/pdf-parser@build", "cd sharedLibs/pdf-parser && cargo build --release"),
- (async () => {
- const install = execForward("sharedLibs/go-html-to-md@install", "cd sharedLibs/go-html-to-md && go mod tidy");
- await install;
-
- const build = execForward("sharedLibs/go-html-to-md@build", "cd sharedLibs/go-html-to-md && go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go");
- await build;
- })(),
+ console.log("=== Installing dependencies and building all components...");
+ await Promise.all([
+ (async () => {
+ const install = execForward("api@install", "pnpm install");
+ await install;
+
+ const build = execForward("api@build", "pnpm build");
+ await build;
+ })(),
+ execForward(
+ "sharedLibs/crawler@build",
+ "cd sharedLibs/crawler && cargo build --release",
+ ),
+ execForward(
+ "sharedLibs/html-transformer@build",
+ "cd sharedLibs/html-transformer && cargo build --release",
+ ),
+ execForward(
+ "sharedLibs/pdf-parser@build",
+ "cd sharedLibs/pdf-parser && cargo build --release",
+ ),
+ (async () => {
+ const install = execForward(
+ "sharedLibs/go-html-to-md@install",
+ "cd sharedLibs/go-html-to-md && go mod tidy",
+ );
+ await install;
+
+ const build = execForward(
+ "sharedLibs/go-html-to-md@build",
+ `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`,
+ );
+ await build;
+ })(),
+ ]);
+
+ console.log("=== Starting API, Worker, and Index Worker...");
+
+ const api = execForward("api", "pnpm start:production:nobuild");
+ const worker = execForward("worker", "pnpm worker:production");
+ const indexWorker = execForward(
+ "index-worker",
+ "pnpm index-worker:production",
+ );
+
+ try {
+ await Promise.race([
+ waitForPort(3002, "localhost"),
+ new Promise((reject) =>
+ setTimeout(() => reject(new Error("API did not start in time")), 10000),
+ ),
]);
- console.log("=== Starting API, Worker, and Index Worker...");
-
- const api = execForward("api", "pnpm start:production:nobuild");
- const worker = execForward("worker", "pnpm worker:production");
- const indexWorker = execForward("index-worker", "pnpm index-worker:production");
-
- try {
- await Promise.race([
- waitForPort(3002, "localhost"),
- new Promise((reject) => setTimeout(() => reject(new Error("API did not start in time")), 10000)),
- ]);
-
- console.log("=== Running command...");
-
- const cmd = execForward("command", command.join(" "));
-
- await Promise.race([
- cmd,
- api,
- worker,
- indexWorker,
- ]);
- } finally {
- console.log("=== Tearing down API, Worker, and Index Worker...");
- exec("pkill -f 'queue-worker.js'");
- exec("pkill -f 'index.js'");
- exec("pkill -f 'index-worker.js'");
- await Promise.all([
- api,
- worker,
- indexWorker,
- ]);
- }
-
- console.log("=== Goodbye!");
-})();
\ No newline at end of file
+ console.log("=== Running command...");
+
+ const cmd = execForward("command", command.join(" "));
+
+ await Promise.race([cmd, api, worker, indexWorker]);
+ } finally {
+ console.log("=== Tearing down API, Worker, and Index Worker...");
+ exec("pkill -f 'queue-worker.js'");
+ exec("pkill -f 'index.js'");
+ exec("pkill -f 'index-worker.js'");
+ await Promise.all([api, worker, indexWorker]);
+ }
+
+ console.log("=== Goodbye!");
+})();
diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.ts
index 862988dce..cf4c311d5 100644
--- a/apps/api/src/lib/crawler.ts
+++ b/apps/api/src/lib/crawler.ts
@@ -1,34 +1,28 @@
import koffi, { KoffiFunction } from "koffi";
-import { join } from "path";
import { stat } from "fs/promises";
-import { platform } from "os";
import type { DenialReason } from "../scraper/WebScraper/crawler";
+import { CRAWLER_PATH } from "../natives";
// TODO: add a timeout to the Rust transformer
-const rustExecutablePath = join(
- process.cwd(),
- "sharedLibs/crawler/target/release/",
- platform() === "darwin" ? "libcrawler.dylib" : "libcrawler.so"
-);
export type FilterLinksCall = {
- links: string[],
- limit: number | undefined,
- max_depth: number,
- base_url: string,
- initial_url: string,
- regex_on_full_url: boolean,
- excludes: string[],
- includes: string[],
- allow_backward_crawling: boolean,
- ignore_robots_txt: boolean,
- robots_txt: string,
-}
+ links: string[];
+ limit: number | undefined;
+ max_depth: number;
+ base_url: string;
+ initial_url: string;
+ regex_on_full_url: boolean;
+ excludes: string[];
+ includes: string[];
+ allow_backward_crawling: boolean;
+ ignore_robots_txt: boolean;
+ robots_txt: string;
+};
export type FilterLinksResult = {
- links: string[],
- denial_reasons: Map<string, keyof typeof DenialReason>,
-}
+ links: string[];
+ denial_reasons: Map<string, keyof typeof DenialReason>;
+};
class RustCrawler {
private static instance: RustCrawler;
@@ -38,19 +32,27 @@ class RustCrawler {
private _processSitemap: KoffiFunction;
private constructor() {
- const lib = koffi.load(rustExecutablePath);
+ const lib = koffi.load(CRAWLER_PATH);
this._freeString = lib.func("free_string", "void", ["string"]);
const cstn = "CString:" + crypto.randomUUID();
- const freedResultString = koffi.disposable(cstn, "string", this._freeString);
+ const freedResultString = koffi.disposable(
+ cstn,
+ "string",
+ this._freeString,
+ );
this._filterLinks = lib.func("filter_links", freedResultString, ["string"]);
- this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, ["string"]);
- this._processSitemap = lib.func("process_sitemap", freedResultString, ["string"]);
+ this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, [
+ "string",
+ ]);
+ this._processSitemap = lib.func("process_sitemap", freedResultString, [
+ "string",
+ ]);
}
public static async getInstance(): Promise<RustCrawler> {
if (!RustCrawler.instance) {
try {
- await stat(rustExecutablePath);
+ await stat(CRAWLER_PATH);
} catch (_) {
throw Error("Rust crawler shared library not found");
}
@@ -61,30 +63,35 @@ class RustCrawler {
public async filterLinks(call: FilterLinksCall): Promise<FilterLinksResult> {
return new Promise<FilterLinksResult>((resolve, reject) => {
- this._filterLinks.async(JSON.stringify(call), (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- return reject(new Error(res.split("RUSTFC:ERROR:")[1]));
- }
+ this._filterLinks.async(
+ JSON.stringify(call),
+ (err: Error, res: string) => {
+ if (err) {
+ reject(err);
+ } else {
+ if (res.startsWith("RUSTFC:ERROR:")) {
+ return reject(new Error(res.split("RUSTFC:ERROR:")[1]));
+ }
- if (res === "RUSTFC:ERROR") {
- return reject(new Error("Something went wrong on the Rust side."));
- }
+ if (res === "RUSTFC:ERROR") {
+ return reject(
+ new Error("Something went wrong on the Rust side."),
+ );
+ }
- try {
- const raw = JSON.parse(res);
- const result: FilterLinksResult = {
- links: raw.links,
- denial_reasons: new Map(Object.entries(raw.denial_reasons)),
+ try {
+ const raw = JSON.parse(res);
+ const result: FilterLinksResult = {
+ links: raw.links,
+ denial_reasons: new Map(Object.entries(raw.denial_reasons)),
+ };
+ resolve(result);
+ } catch (e) {
+ reject(e);
}
- resolve(result);
- } catch (e) {
- reject(e);
}
- }
- });
+ },
+ );
});
}
@@ -142,16 +149,16 @@ class RustCrawler {
export async function filterLinks(
call: FilterLinksCall,
): Promise<FilterLinksResult> {
- const converter = await RustCrawler.getInstance();
- return await converter.filterLinks(call);
+ const converter = await RustCrawler.getInstance();
+ return await converter.filterLinks(call);
}
export async function parseSitemapXml(xmlContent: string): Promise<any> {
- const converter = await RustCrawler.getInstance();
- return await converter.parseSitemapXml(xmlContent);
+ const converter = await RustCrawler.getInstance();
+ return await converter.parseSitemapXml(xmlContent);
}
export async function processSitemap(xmlContent: string): Promise<any> {
- const converter = await RustCrawler.getInstance();
- return await converter.processSitemap(xmlContent);
+ const converter = await RustCrawler.getInstance();
+ return await converter.processSitemap(xmlContent);
}
diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts
index a474e34e2..5a21b9b1f 100644
--- a/apps/api/src/lib/html-to-markdown.ts
+++ b/apps/api/src/lib/html-to-markdown.ts
@@ -1,38 +1,34 @@
import koffi from "koffi";
-import { join } from "path";
import "../services/sentry";
import * as Sentry from "@sentry/node";
import dotenv from "dotenv";
import { logger } from "./logger";
import { stat } from "fs/promises";
+import { HTML_TO_MARKDOWN_PATH } from "../natives";
dotenv.config();
// TODO: add a timeout to the Go parser
-const goExecutablePath = join(
- process.cwd(),
- "sharedLibs",
- "go-html-to-md",
- "html-to-markdown.so",
-);
class GoMarkdownConverter {
private static instance: GoMarkdownConverter;
private convert: any;
private free: any;
-
+
private constructor() {
- const lib = koffi.load(goExecutablePath);
+ const lib = koffi.load(HTML_TO_MARKDOWN_PATH);
this.free = lib.func("FreeCString", "void", ["string"]);
const cstn = "CString:" + crypto.randomUUID();
const freedResultString = koffi.disposable(cstn, "string", this.free);
- this.convert = lib.func("ConvertHTMLToMarkdown", freedResultString, ["string"]);
+ this.convert = lib.func("ConvertHTMLToMarkdown", freedResultString, [
+ "string",
+ ]);
}
public static async getInstance(): Promise<GoMarkdownConverter> {
if (!GoMarkdownConverter.instance) {
try {
- await stat(goExecutablePath);
+ await stat(HTML_TO_MARKDOWN_PATH);
} catch (_) {
throw Error("Go shared library not found");
}
@@ -83,7 +79,7 @@ export async function parseMarkdown(
} else {
logger.warn(
"Tried to use Go parser, but it doesn't exist in the file system.",
- { goExecutablePath },
+ { HTML_TO_MARKDOWN_PATH },
);
}
}
diff --git a/apps/api/src/lib/html-transformer.ts b/apps/api/src/lib/html-transformer.ts
index 4f05fe78c..247837e25 100644
--- a/apps/api/src/lib/html-transformer.ts
+++ b/apps/api/src/lib/html-transformer.ts
@@ -1,22 +1,14 @@
import koffi, { KoffiFunction } from "koffi";
-import { join } from "path";
import { stat } from "fs/promises";
-import { platform } from "os";
-
-// TODO: add a timeout to the Rust transformer
-const rustExecutablePath = join(
- process.cwd(),
- "sharedLibs/html-transformer/target/release/",
- platform() === "darwin" ? "libhtml_transformer.dylib" : "libhtml_transformer.so"
-);
+import { HTML_TRANSFORMER_PATH } from "../natives";
type TransformHtmlOptions = {
- html: string,
- url: string,
- include_tags: string[],
- exclude_tags: string[],
- only_main_content: boolean,
- omce_signatures?: string[],
+ html: string;
+ url: string;
+ include_tags: string[];
+ exclude_tags: string[];
+ only_main_content: boolean;
+ omce_signatures?: string[];
};
class RustHTMLTransformer {
@@ -31,23 +23,45 @@ class RustHTMLTransformer {
private _extractAttributes: KoffiFunction;
private constructor() {
- const lib = koffi.load(rustExecutablePath);
+ const lib = koffi.load(HTML_TRANSFORMER_PATH);
this._freeString = lib.func("free_string", "void", ["string"]);
const cstn = "CString:" + crypto.randomUUID();
- const freedResultString = koffi.disposable(cstn, "string", this._freeString);
- this._extractLinks = lib.func("extract_links", freedResultString, ["string"]);
- this._extractImages = lib.func("extract_images", freedResultString, ["string", "string"]);
- this._extractBaseHref = lib.func("extract_base_href", freedResultString, ["string", "string"]);
- this._extractMetadata = lib.func("extract_metadata", freedResultString, ["string"]);
- this._transformHtml = lib.func("transform_html", freedResultString, ["string"]);
- this._getInnerJSON = lib.func("get_inner_json", freedResultString, ["string"]);
- this._extractAttributes = lib.func("extract_attributes", freedResultString, ["string", "string"]);
+ const freedResultString = koffi.disposable(
+ cstn,
+ "string",
+ this._freeString,
+ );
+ this._extractLinks = lib.func("extract_links", freedResultString, [
+ "string",
+ ]);
+ this._extractImages = lib.func("extract_images", freedResultString, [
+ "string",
+ "string",
+ ]);
+ this._extractBaseHref = lib.func("extract_base_href", freedResultString, [
+ "string",
+ "string",
+ ]);
+ this._extractMetadata = lib.func("extract_metadata", freedResultString, [
+ "string",
+ ]);
+ this._transformHtml = lib.func("transform_html", freedResultString, [
+ "string",
+ ]);
+ this._getInnerJSON = lib.func("get_inner_json", freedResultString, [
+ "string",
+ ]);
+ this._extractAttributes = lib.func(
+ "extract_attributes",
+ freedResultString,
+ ["string", "string"],
+ );
}
public static async getInstance(): Promise<RustHTMLTransformer> {
if (!RustHTMLTransformer.instance) {
try {
- await stat(rustExecutablePath);
+ await stat(HTML_TRANSFORMER_PATH);
} catch (_) {
throw Error("Rust html-transformer shared library not found");
}
@@ -110,17 +124,27 @@ class RustHTMLTransformer {
public async transformHtml(opts: TransformHtmlOptions): Promise<string> {
return new Promise<string>((resolve, reject) => {
- this._transformHtml.async(JSON.stringify(opts), (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res === "RUSTFC:ERROR" || res.startsWith("RUSTFC:ERROR:")) {
- reject(new Error(res.startsWith("RUSTFC:ERROR:") ? ("Something went wrong on the Rust side. " + res.split("RUSTFC:ERROR:")[1]) : "Something went wrong on the Rust side."));
+ this._transformHtml.async(
+ JSON.stringify(opts),
+ (err: Error, res: string) => {
+ if (err) {
+ reject(err);
} else {
- resolve(res);
+ if (res === "RUSTFC:ERROR" || res.startsWith("RUSTFC:ERROR:")) {
+ reject(
+ new Error(
+ res.startsWith("RUSTFC:ERROR:")
+ ? "Something went wrong on the Rust side. " +
+ res.split("RUSTFC:ERROR:")[1]
+ : "Something went wrong on the Rust side.",
+ ),
+ );
+ } else {
+ resolve(res);
+ }
}
- }
- });
+ },
+ );
});
}
@@ -140,19 +164,31 @@ class RustHTMLTransformer {
});
}
- public async extractAttributes(html: string, options: string): Promise<string> {
+ public async extractAttributes(
+ html: string,
+ options: string,
+ ): Promise<string> {
return new Promise<string>((resolve, reject) => {
- this._extractAttributes.async(html, options, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- reject(new Error("Rust attribute extraction failed: " + res.split("RUSTFC:ERROR:")[1]));
+ this._extractAttributes.async(
+ html,
+ options,
+ (err: Error, res: string) => {
+ if (err) {
+ reject(err);
} else {
- resolve(res);
+ if (res.startsWith("RUSTFC:ERROR:")) {
+ reject(
+ new Error(
+ "Rust attribute extraction failed: " +
+ res.split("RUSTFC:ERROR:")[1],
+ ),
+ );
+ } else {
+ resolve(res);
+ }
}
- }
- });
+ },
+ );
});
}
}
@@ -160,47 +196,47 @@ class RustHTMLTransformer {
export async function extractLinks(
html: string | null | undefined,
): Promise<string[]> {
- if (!html) {
- return [];
- }
+ if (!html) {
+ return [];
+ }
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.extractLinks(html);
+ const converter = await RustHTMLTransformer.getInstance();
+ return await converter.extractLinks(html);
}
export async function extractImages(
html: string | null | undefined,
- baseUrl: string = ''
+ baseUrl: string = "",
): Promise<string[]> {
- if (!html) {
- return [];
- }
+ if (!html) {
+ return [];
+ }
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.extractImages(html, baseUrl);
+ const converter = await RustHTMLTransformer.getInstance();
+ return await converter.extractImages(html, baseUrl);
}
export async function extractBaseHref(
html: string | null | undefined,
- url: string
+ url: string,
): Promise<string> {
- if (!html) {
- return url;
- }
+ if (!html) {
+ return url;
+ }
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.extractBaseHref(html, url);
+ const converter = await RustHTMLTransformer.getInstance();
+ return await converter.extractBaseHref(html, url);
}
export async function extractMetadata(
- html: string | null | undefined,
+ html: string | null | undefined,
): Promise<any> {
- if (!html) {
- return [];
- }
+ if (!html) {
+ return [];
+ }
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.extractMetadata(html);
+ const converter = await RustHTMLTransformer.getInstance();
+ return await converter.extractMetadata(html);
}
export async function transformHtml(
@@ -210,9 +246,7 @@ export async function transformHtml(
return await converter.transformHtml(opts);
}
-export async function getInnerJSON(
- html: string,
-): Promise<string> {
+export async function getInnerJSON(html: string): Promise<string> {
const converter = await RustHTMLTransformer.getInstance();
return await converter.getInnerJSON(html);
}
@@ -230,7 +264,7 @@ export type AttributeResult = {
export async function extractAttributesRust(
html: string,
- selectors: AttributeSelector[]
+ selectors: AttributeSelector[],
): Promise<AttributeResult[]> {
if (!html || selectors.length === 0) {
return [];
@@ -243,6 +277,8 @@ export async function extractAttributesRust(
try {
return JSON.parse(resultJson);
} catch (error) {
- throw new Error(`Failed to parse Rust attribute extraction result: ${error}`);
+ throw new Error(
+ `Failed to parse Rust attribute extraction result: ${error}`,
+ );
}
}
diff --git a/apps/api/src/lib/pdf-parser.ts b/apps/api/src/lib/pdf-parser.ts
index 0c24637d6..b930b78a7 100644
--- a/apps/api/src/lib/pdf-parser.ts
+++ b/apps/api/src/lib/pdf-parser.ts
@@ -2,79 +2,73 @@ import koffi, { KoffiFunction } from "koffi";
import { join } from "path";
import { stat } from "fs/promises";
import { platform } from "os";
+import { PDF_PARSER_PATH } from "../natives";
// TODO: add a timeout to the Rust parser
-const rustExecutablePath = join(
- process.cwd(),
- "sharedLibs/pdf-parser/target/release/",
- platform() === "darwin" ? "libpdf_parser.dylib" : "libpdf_parser.so"
-);
export type PDFMetadata = {
- numPages: number;
- title?: string;
-}
+ numPages: number;
+ title?: string;
+};
class RustPDFParser {
- private static instance: RustPDFParser;
- private _getPDFMetadata: KoffiFunction;
+ private static instance: RustPDFParser;
+ private _getPDFMetadata: KoffiFunction;
+
+ private constructor() {
+ const lib = koffi.load(PDF_PARSER_PATH);
+ this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]);
+ }
- private constructor() {
- const lib = koffi.load(rustExecutablePath);
- this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]);
+ public static async isParserAvailable(): Promise<boolean> {
+ if (RustPDFParser.instance) {
+ return true;
}
- public static async isParserAvailable(): Promise<boolean> {
- if (RustPDFParser.instance) {
- return true;
- }
+ try {
+ await stat(PDF_PARSER_PATH);
+ RustPDFParser.instance = new RustPDFParser();
+ return true;
+ } catch (_) {
+ return false;
+ }
+ }
- try {
- await stat(rustExecutablePath);
- RustPDFParser.instance = new RustPDFParser();
- return true;
- } catch (_) {
- return false;
- }
+ public static async getInstance(): Promise<RustPDFParser> {
+ if (!RustPDFParser.instance) {
+ try {
+ await stat(PDF_PARSER_PATH);
+ } catch (_) {
+ throw Error("Rust pdf-parser shared library not found");
+ }
+ RustPDFParser.instance = new RustPDFParser();
}
+ return RustPDFParser.instance;
+ }
- public static async getInstance(): Promise<RustPDFParser> {
- if (!RustPDFParser.instance) {
+ public async getPDFMetadata(path: string): Promise<PDFMetadata> {
+ return new Promise<PDFMetadata>((resolve, reject) => {
+ this._getPDFMetadata.async(path, (err: Error, res: string) => {
+ if (err) {
+ reject(err);
+ } else {
+ if (res.startsWith("RUSTFC:ERROR:")) {
+ reject(new Error(res.replace("RUSTFC:ERROR:", "")));
+ } else {
try {
- await stat(rustExecutablePath);
- } catch (_) {
- throw Error("Rust pdf-parser shared library not found");
+ const metadata = JSON.parse(res) as PDFMetadata;
+ resolve(metadata);
+ } catch (e) {
+ reject(new Error("Failed to parse PDF metadata."));
}
- RustPDFParser.instance = new RustPDFParser();
+ }
}
- return RustPDFParser.instance;
- }
-
- public async getPDFMetadata(path: string): Promise<PDFMetadata> {
- return new Promise<PDFMetadata>((resolve, reject) => {
- this._getPDFMetadata.async(path, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- reject(new Error(res.replace("RUSTFC:ERROR:", "")));
- } else {
- try {
- const metadata = JSON.parse(res) as PDFMetadata;
- resolve(metadata);
- } catch (e) {
- reject(new Error("Failed to parse PDF metadata."));
- }
- }
- }
- });
- });
- }
+ });
+ });
+ }
}
-export async function getPDFMetadata(
- path: string,
-): Promise<PDFMetadata> {
- const converter = await RustPDFParser.getInstance();
- return await converter.getPDFMetadata(path);
+export async function getPDFMetadata(path: string): Promise<PDFMetadata> {
+ const converter = await RustPDFParser.getInstance();
+ return await converter.getPDFMetadata(path);
}
diff --git a/apps/api/src/natives.ts b/apps/api/src/natives.ts
new file mode 100644
index 000000000..560a14cb7
--- /dev/null
+++ b/apps/api/src/natives.ts
@@ -0,0 +1,39 @@
+import { platform } from "os";
+import { join } from "path";
+
+const currentPlatform = platform();
+const isWindows = currentPlatform === "win32";
+
+const EXTENSIONS = {
+ win32: ".dll",
+ darwin: ".dylib",
+ default: ".so",
+} as const;
+
+function createNativePath(subPath: string, filename: string): string {
+ const extension =
+ EXTENSIONS[currentPlatform as keyof typeof EXTENSIONS] ??
+ EXTENSIONS.default;
+ const fullFilename = `${isWindows ? "" : "lib"}${filename}${extension}`;
+ return join(process.cwd(), "sharedLibs", subPath, fullFilename);
+}
+
+export const HTML_TRANSFORMER_PATH = createNativePath(
+ "html-transformer/target/release",
+ "html_transformer",
+);
+
+export const PDF_PARSER_PATH = createNativePath(
+ "pdf-parser/target/release",
+ "pdf_parser",
+);
+
+export const CRAWLER_PATH = createNativePath(
+ "crawler/target/release",
+ "crawler",
+);
+
+export const HTML_TO_MARKDOWN_PATH = createNativePath(
+ "go-html-to-md",
+ "html-to-markdown",
+);