Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Convert all Rust natives to a single library using NAPI (ENG-3397) (#2105)

tom7569cd3a67a1
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml
index 18730fa4b..9c97c8f00 100644
--- a/.github/workflows/test-server-self-host.yml
+++ b/.github/workflows/test-server-self-host.yml
@@ -19,11 +19,11 @@ jobs:
name: Run tests
strategy:
matrix:
- ai: ["openai", "no-ai"]
- search: ["searxng", "google"]
- engine: ["playwright", "fetch"]
+ ai: ['openai', 'no-ai']
+ search: ['searxng', 'google']
+ engine: ['playwright', 'fetch']
# proxy: ["proxy", "no-proxy"]
- proxy: ["proxy"]
+ proxy: ['proxy']
fail-fast: false
runs-on: ubuntu-latest
services:
@@ -47,8 +47,8 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v4
with:
- node-version: "20"
- cache: "pnpm"
+ node-version: '20'
+ cache: 'pnpm'
cache-dependency-path: './apps/api/pnpm-lock.yaml'
- name: Install dependencies
run: pnpm install
@@ -65,29 +65,12 @@ jobs:
with:
go-version: '1.24'
cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
- - name: Set up Rust
- uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Build go-html-to-md
run: |
go mod tidy
go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
chmod +x libhtml-to-markdown.so
working-directory: ./apps/api/sharedLibs/go-html-to-md
- - name: Build html-transformer
- run: |
- cargo build --release
- chmod +x target/release/libhtml_transformer.so
- working-directory: ./apps/api/sharedLibs/html-transformer
- - name: Build pdf-parser
- run: |
- cargo build --release
- chmod +x target/release/libpdf_parser.so
- working-directory: ./apps/api/sharedLibs/pdf-parser
- - name: Build crawler
- run: |
- cargo build --release
- chmod +x target/release/libcrawler.so
- working-directory: ./apps/api/sharedLibs/crawler
- name: Set up SearXNG
if: matrix.search == 'searxng'
run: |
@@ -98,28 +81,28 @@ jobs:
formats: [html, json, csv]
server:
secret_key: 'fcsecret'" > searxng/settings.yml
-
+
docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng
pnpx wait-on tcp:3434 -t 30s
working-directory: ./
- name: Build the application
- run: npm run build
+ run: pnpm run build
working-directory: ./apps/api
- name: Start server
- run: npm run start:production:nobuild > api.log 2>&1 &
+ run: pnpm run start:production:nobuild > api.log 2>&1 &
env:
PORT: 3002
HOST: 0.0.0.0
working-directory: ./apps/api
- name: Start worker
- run: npm run worker:production > worker.log 2>&1 &
+ run: pnpm run worker:production > worker.log 2>&1 &
env:
PORT: 3005
HOST: 0.0.0.0
working-directory: ./apps/api
- name: Start playwright
if: matrix.engine == 'playwright'
- run: npm run dev > playwright.log 2>&1 &
+ run: pnpm run dev > playwright.log 2>&1 &
working-directory: ./apps/playwright-service-ts
env:
PORT: 3003
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 8c76cd08b..6278db813 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -71,8 +71,8 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v4
with:
- node-version: "20"
- cache: "pnpm"
+ node-version: '20'
+ cache: 'pnpm'
cache-dependency-path: './apps/api/pnpm-lock.yaml'
- name: Install dependencies
run: pnpm install
@@ -82,48 +82,31 @@ jobs:
with:
go-version: '1.24'
cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum
- - name: Set up Rust
- uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Build go-html-to-md
run: |
go mod tidy
go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
chmod +x libhtml-to-markdown.so
working-directory: ./apps/api/sharedLibs/go-html-to-md
- - name: Build html-transformer
- run: |
- cargo build --release
- chmod +x target/release/libhtml_transformer.so
- working-directory: ./apps/api/sharedLibs/html-transformer
- - name: Build pdf-parser
- run: |
- cargo build --release
- chmod +x target/release/libpdf_parser.so
- working-directory: ./apps/api/sharedLibs/pdf-parser
- - name: Build crawler
- run: |
- cargo build --release
- chmod +x target/release/libcrawler.so
- working-directory: ./apps/api/sharedLibs/crawler
- name: Build the application
- run: npm run build
+ run: pnpm run build
working-directory: ./apps/api
- name: Start the application
- run: npm run start:production:nobuild > api.log 2>&1 &
+ run: pnpm run start:production:nobuild > api.log 2>&1 &
env:
PORT: 3002
HOST: 0.0.0.0
working-directory: ./apps/api
id: start_app
- name: Start worker
- run: npm run worker:production > worker.log 2>&1 &
+ run: pnpm run worker:production > worker.log 2>&1 &
env:
PORT: 3005
HOST: 0.0.0.0
working-directory: ./apps/api
id: start_workers
- name: Start index worker
- run: npm run index-worker:production > index-worker.log 2>&1 &
+ run: pnpm run index-worker:production > index-worker.log 2>&1 &
working-directory: ./apps/api
id: start_index_worker
- name: Wait for API
@@ -156,4 +139,4 @@ jobs:
with:
name: Encrypted Logs
path: logs/logs.zip.gpg
- retention-days: 5
\ No newline at end of file
+ retention-days: 5
diff --git a/apps/api/.dockerignore b/apps/api/.dockerignore
index 679a66475..0f38d9cc0 100644
--- a/apps/api/.dockerignore
+++ b/apps/api/.dockerignore
@@ -1,4 +1,5 @@
-/node_modules/
-/dist/
+**/node_modules/
+**/dist/
+**/target/
.env
*.csv
diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
index 544e06385..bf96a1615 100644
--- a/apps/api/Dockerfile
+++ b/apps/api/Dockerfile
@@ -1,64 +1,77 @@
+# syntax=docker/dockerfile:1
FROM node:22-slim AS base
ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH"
+ENV CI=true
RUN corepack enable
-COPY . /app
+
+# Build Go shared library
+FROM golang:1.24 AS go-build
WORKDIR /app
+COPY sharedLibs/go-html-to-md ./sharedLibs/go-html-to-md
-FROM base AS prod-deps
-RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile --ignore-scripts
+RUN cd sharedLibs/go-html-to-md && \
+ go mod download && \
+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go
FROM base AS build
-RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install && pnpm run build
-
-# Install Go
-FROM golang:1.24 AS go-base
-COPY sharedLibs/go-html-to-md /app/sharedLibs/go-html-to-md
+WORKDIR /app
-# Install Go dependencies and build parser lib
-RUN cd /app/sharedLibs/go-html-to-md && \
- go mod download && \
- go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go && \
- chmod +x libhtml-to-markdown.so
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+ curl \
+ build-essential \
+ pkg-config \
+ && rm -rf /var/lib/apt/lists/*
# Install Rust
-FROM rust:1-slim AS rust-base
-COPY sharedLibs/html-transformer /app/sharedLibs/html-transformer
-COPY sharedLibs/pdf-parser /app/sharedLibs/pdf-parser
-COPY sharedLibs/crawler /app/sharedLibs/crawler
-
-# Install Rust dependencies and build transformer lib
-RUN cd /app/sharedLibs/html-transformer && \
- cargo build --release --locked && \
- chmod +x target/release/libhtml_transformer.so
-
-# Install Rust dependencies and build PDF parser lib
-RUN cd /app/sharedLibs/pdf-parser && \
- cargo build --release --locked && \
- chmod +x target/release/libpdf_parser.so
-
-# Install Rust dependencies and build crawler lib
-RUN cd /app/sharedLibs/crawler && \
- cargo build --release --locked && \
- chmod +x target/release/libcrawler.so
-
-FROM base
-COPY --from=build /app/dist /app/dist
-COPY --from=prod-deps /app/node_modules /app/node_modules
-COPY --from=go-base /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so
-COPY --from=rust-base /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so
-COPY --from=rust-base /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so
-COPY --from=rust-base /app/sharedLibs/crawler/target/release/libcrawler.so /app/sharedLibs/crawler/target/release/libcrawler.so
-
-# Install git
-RUN apt-get update && apt-get install -y git procps && rm -rf /var/lib/apt/lists/*
-
-# Start the server by default, this can be overwritten at runtime
+ENV RUSTUP_HOME=/usr/local/rustup \
+ CARGO_HOME=/usr/local/cargo \
+ PATH=/usr/local/cargo/bin:$PATH
+
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path \
+ && chmod -R a+w $RUSTUP_HOME $CARGO_HOME
+
+# Copy source files
+COPY pnpm-lock.yaml pnpm-workspace.yaml package.json ./
+COPY . .
+
+# Install dependencies
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
+ --mount=type=cache,target=/usr/local/cargo/registry \
+ --mount=type=cache,target=/app/native/target \
+ pnpm install --frozen-lockfile
+
+# Build the application
+RUN pnpm run build
+
+# Remove dev dependencies
+RUN pnpm prune --prod --ignore-scripts
+
+# Runtime stage
+FROM base AS runtime
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y \
+ git \
+ procps \
+ && rm -rf /var/lib/apt/lists/*
+
EXPOSE 8080
+WORKDIR /app
+
+# Copy built application
+COPY --from=build /app/node_modules ./node_modules
+COPY --from=build /app/dist ./dist
+COPY --from=build /app/native ./native
+
+# Copy Go shared library
+COPY --from=go-build /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so ./sharedLibs/go-html-to-md/
-# Make sure the entrypoint script has the correct line endings
-RUN sed -i 's/\r$//' /app/docker-entrypoint.sh
+# Copy and prepare entrypoint
+COPY docker-entrypoint.sh ./
+RUN sed -i 's/\r$//' ./docker-entrypoint.sh && chmod +x ./docker-entrypoint.sh
-ENTRYPOINT "/app/docker-entrypoint.sh"
+ENTRYPOINT ["./docker-entrypoint.sh"]
\ No newline at end of file
diff --git a/apps/api/native/.cargo/config.toml b/apps/api/native/.cargo/config.toml
new file mode 100644
index 000000000..ac2b23f81
--- /dev/null
+++ b/apps/api/native/.cargo/config.toml
@@ -0,0 +1,2 @@
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-C", "target-feature=+crt-static"]
diff --git a/apps/api/native/.editorconfig b/apps/api/native/.editorconfig
new file mode 100644
index 000000000..182fda23a
--- /dev/null
+++ b/apps/api/native/.editorconfig
@@ -0,0 +1,15 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors or IDEs
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = false
diff --git a/apps/api/native/.gitattributes b/apps/api/native/.gitattributes
new file mode 100644
index 000000000..e310f6263
--- /dev/null
+++ b/apps/api/native/.gitattributes
@@ -0,0 +1,18 @@
+# Auto detect text files and perform LF normalization
+* text=auto
+
+
+*.ts text eol=lf merge=union
+*.tsx text eol=lf merge=union
+*.rs text eol=lf merge=union
+*.js text eol=lf merge=union
+*.json text eol=lf merge=union
+*.debug text eol=lf merge=union
+
+# Generated codes
+index.js linguist-detectable=false
+index.d.ts linguist-detectable=false
+firecrawl-rs.wasi-browser.js linguist-detectable=false
+firecrawl-rs.wasi.cjs linguist-detectable=false
+wasi-worker-browser.mjs linguist-detectable=false
+wasi-worker.mjs linguist-detectable=false
diff --git a/apps/api/native/.gitignore b/apps/api/native/.gitignore
new file mode 100644
index 000000000..cf7f86697
--- /dev/null
+++ b/apps/api/native/.gitignore
@@ -0,0 +1,120 @@
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# Rust
+/target
+Cargo.lock
+
+*.node
+.pnp.*
+/npm
+
+index.js
+index.d.ts
+browser.js
diff --git a/apps/api/native/.prettierignore b/apps/api/native/.prettierignore
new file mode 100644
index 000000000..901e6b62c
--- /dev/null
+++ b/apps/api/native/.prettierignore
@@ -0,0 +1,8 @@
+target
+.yarn
+index.js
+package-template.wasi-browser.js
+package-template.wasi.cjs
+wasi-worker-browser.mjs
+wasi-worker.mjs
+.yarnrc.yml
\ No newline at end of file
diff --git a/apps/api/native/.taplo.toml b/apps/api/native/.taplo.toml
new file mode 100644
index 000000000..b2d27a70b
--- /dev/null
+++ b/apps/api/native/.taplo.toml
@@ -0,0 +1,7 @@
+exclude = ["node_modules/**/*.toml"]
+
+# https://taplo.tamasfe.dev/configuration/formatter-options.html
+[formatting]
+align_entries = true
+indent_tables = true
+reorder_keys = true
diff --git a/apps/api/native/.yarnrc.yml b/apps/api/native/.yarnrc.yml
new file mode 100644
index 000000000..52d58b687
--- /dev/null
+++ b/apps/api/native/.yarnrc.yml
@@ -0,0 +1,5 @@
+nodeLinker: node-modules
+
+npmAuditRegistry: "https://registry.npmjs.org"
+
+yarnPath: .yarn/releases/yarn-4.9.4.cjs
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
new file mode 100644
index 000000000..e08132429
--- /dev/null
+++ b/apps/api/native/Cargo.toml
@@ -0,0 +1,30 @@
+
+[package]
+authors = ["Firecrawl"]
+edition = "2021"
+name = "firecrawl_rs"
+version = "0.1.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+kuchikiki = "0.8.2"
+lol_html = "2.6.0"
+lopdf = "0.38.0"
+napi = { version = "3.0.0", features = ["serde-json"] }
+napi-derive = "3.0.0"
+nodesig = { git = "https://github.com/firecrawl/nodesig" }
+regex = "1.11.2"
+roxmltree = "0.20.0"
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = "1.0.143"
+texting_robots = "0.2.2"
+url = "2.5.7"
+
+[build-dependencies]
+napi-build = "2"
+
+[profile.release]
+lto = true
+strip = "symbols"
diff --git a/apps/api/native/README.md b/apps/api/native/README.md
new file mode 100644
index 000000000..92523e06a
--- /dev/null
+++ b/apps/api/native/README.md
@@ -0,0 +1,87 @@
+# `@napi-rs/package-template`
+
+![https://github.com/napi-rs/package-template/actions](https://github.com/napi-rs/package-template/workflows/CI/badge.svg)
+
+> Template project for writing node packages with napi-rs.
+
+# Usage
+
+1. Click **Use this template**.
+2. **Clone** your project.
+3. Run `yarn install` to install dependencies.
+4. Run `yarn napi rename -n [@your-scope/package-name] -b [binary-name]` command under the project folder to rename your package.
+
+## Install this test package
+
+```bash
+yarn add @napi-rs/package-template
+```
+
+## Ability
+
+### Build
+
+After `yarn build/npm run build` command, you can see `package-template.[darwin|win32|linux].node` file in project root. This is the native addon built from [lib.rs](./src/lib.rs).
+
+### Test
+
+With [ava](https://github.com/avajs/ava), run `yarn test/npm run test` to testing native addon. You can also switch to another testing framework if you want.
+
+### CI
+
+With GitHub Actions, each commit and pull request will be built and tested automatically in [`node@20`, `@node22`] x [`macOS`, `Linux`, `Windows`] matrix. You will never be afraid of the native addon broken in these platforms.
+
+### Release
+
+Release native package is very difficult in old days. Native packages may ask developers who use it to install `build toolchain` like `gcc/llvm`, `node-gyp` or something more.
+
+With `GitHub actions`, we can easily prebuild a `binary` for major platforms. And with `N-API`, we should never be afraid of **ABI Compatible**.
+
+The other problem is how to deliver prebuild `binary` to users. Downloading it in `postinstall` script is a common way that most packages do it right now. The problem with this solution is it introduced many other packages to download binary that has not been used by `runtime codes`. The other problem is some users may not easily download the binary from `GitHub/CDN` if they are behind a private network (But in most cases, they have a private NPM mirror).
+
+In this package, we choose a better way to solve this problem. We release different `npm packages` for different platforms. And add it to `optionalDependencies` before releasing the `Major` package to npm.
+
+`NPM` will choose which native package should download from `registry` automatically. You can see [npm](./npm) dir for details. And you can also run `yarn add @napi-rs/package-template` to see how it works.
+
+## Develop requirements
+
+- Install the latest `Rust`
+- Install `Node.js@10+` which fully supported `Node-API`
+- Install `[email protected]`
+
+## Test in local
+
+- yarn
+- yarn build
+- yarn test
+
+And you will see:
+
+```bash
+$ ava --verbose
+
+ ✔ sync function from native code
+ ✔ sleep function from native code (201ms)
+ ─
+
+ 2 tests passed
+✨ Done in 1.12s.
+```
+
+## Release package
+
+Ensure you have set your **NPM_TOKEN** in the `GitHub` project setting.
+
+In `Settings -> Secrets`, add **NPM_TOKEN** into it.
+
+When you want to release the package:
+
+```bash
+npm version [<newversion> | major | minor | patch | premajor | preminor | prepatch | prerelease [--preid=<prerelease-id>] | from-git]
+
+git push
+```
+
+GitHub actions will do the rest job for you.
+
+> WARN: Don't run `npm publish` manually.
diff --git a/apps/api/native/build.rs b/apps/api/native/build.rs
new file mode 100644
index 000000000..bbfc9e4b9
--- /dev/null
+++ b/apps/api/native/build.rs
@@ -0,0 +1,3 @@
+fn main() {
+ napi_build::setup();
+}
diff --git a/apps/api/native/package.json b/apps/api/native/package.json
new file mode 100644
index 000000000..99b35f0bc
--- /dev/null
+++ b/apps/api/native/package.json
@@ -0,0 +1,74 @@
+{
+ "name": "@mendable/firecrawl-rs",
+ "version": "1.0.0",
+ "description": "Template project for writing node package with napi-rs",
+ "main": "index.js",
+ "repository": {
+ "type": "git",
+ "url": "git+ssh://[email protected]/napi-rs/package-template.git"
+ },
+ "license": "AGPL",
+ "browser": "browser.js",
+ "keywords": [
+ "napi-rs",
+ "NAPI",
+ "N-API",
+ "Rust",
+ "node-addon",
+ "node-addon-api"
+ ],
+ "files": [
+ "index.d.ts",
+ "index.js",
+ "browser.js"
+ ],
+ "napi": {
+ "binaryName": "firecrawl-rs",
+ "targets": [
+ "x86_64-pc-windows-msvc",
+ "x86_64-apple-darwin",
+ "x86_64-unknown-linux-gnu",
+ "aarch64-apple-darwin"
+ ]
+ },
+ "engines": {
+ "node": ">= 6.14.2 < 7 || >= 8.11.2 < 9 || >= 9.11.0 < 10 || >= 10.0.0"
+ },
+ "publishConfig": {
+ "registry": "https://registry.npmjs.org/",
+ "access": "public"
+ },
+ "scripts": {
+ "artifacts": "napi artifacts",
+ "build": "napi build --platform --release",
+ "build:debug": "napi build --platform",
+ "format": "run-p format:prettier format:rs format:toml",
+ "format:prettier": "prettier . -w",
+ "format:toml": "taplo format",
+ "format:rs": "cargo fmt",
+ "lint": "oxlint .",
+ "prepublishOnly": "napi prepublish -t npm",
+ "version": "napi version",
+ "install": "pnpm build"
+ },
+ "devDependencies": {
+ "@emnapi/core": "^1.5.0",
+ "@emnapi/runtime": "^1.5.0",
+ "@napi-rs/cli": "^3.1.5",
+ "@oxc-node/core": "^0.0.32",
+ "@taplo/cli": "^0.7.0",
+ "@tybys/wasm-util": "^0.10.0",
+ "chalk": "^5.6.0",
+ "npm-run-all2": "^8.0.4",
+ "oxlint": "^1.14.0",
+ "prettier": "^3.6.2",
+ "typescript": "^5.9.2"
+ },
+ "prettier": {
+ "printWidth": 120,
+ "semi": false,
+ "trailingComma": "all",
+ "singleQuote": true,
+ "arrowParens": "always"
+ }
+}
diff --git a/apps/api/native/rustfmt.toml b/apps/api/native/rustfmt.toml
new file mode 100644
index 000000000..b196eaa2d
--- /dev/null
+++ b/apps/api/native/rustfmt.toml
@@ -0,0 +1 @@
+tab_spaces = 2
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
new file mode 100644
index 000000000..8a4c7aa66
--- /dev/null
+++ b/apps/api/native/src/crawler.rs
@@ -0,0 +1,640 @@
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+use std::{collections::HashMap, usize};
+use texting_robots::Robot;
+use url::Url;
+
+#[derive(Deserialize)]
+#[napi(object)]
+pub struct FilterLinksCall {
+ pub links: Vec<String>,
+ pub limit: Option<i64>,
+ pub max_depth: u32,
+ pub base_url: String,
+ pub initial_url: String,
+ pub regex_on_full_url: bool,
+ pub excludes: Vec<String>,
+ pub includes: Vec<String>,
+ pub allow_backward_crawling: bool,
+ pub ignore_robots_txt: bool,
+ pub robots_txt: String,
+}
+
+#[derive(Serialize)]
+#[napi(object)]
+pub struct FilterLinksResult {
+ pub links: Vec<String>,
+ pub denial_reasons: HashMap<String, String>,
+}
+
+#[derive(Serialize, Debug)]
+#[napi(object)]
+pub struct SitemapUrl {
+ pub loc: Vec<String>,
+}
+
+#[derive(Serialize, Debug)]
+#[napi(object)]
+pub struct SitemapEntry {
+ pub loc: Vec<String>,
+}
+
+#[derive(Serialize, Debug)]
+#[napi(object)]
+pub struct SitemapUrlset {
+ pub url: Vec<SitemapUrl>,
+}
+
+#[derive(Serialize, Debug)]
+#[napi(object)]
+pub struct SitemapIndex {
+ pub sitemap: Vec<SitemapEntry>,
+}
+
+#[derive(Serialize, Debug)]
+#[napi(object)]
+pub struct ParsedSitemap {
+ pub urlset: Option<SitemapUrlset>,
+ pub sitemapindex: Option<SitemapIndex>,
+}
+
+#[derive(Serialize, Debug)]
+#[napi(object)]
+pub struct SitemapInstruction {
+ pub action: String,
+ pub urls: Vec<String>,
+ pub count: u32,
+}
+
+#[derive(Serialize, Debug)]
+#[napi(object)]
+pub struct SitemapProcessingResult {
+ pub instructions: Vec<SitemapInstruction>,
+ pub total_count: u32,
+}
+
+fn _is_file(url: &Url) -> bool {
+ let file_extensions = vec![
+ ".png", ".jpg", ".jpeg", ".gif", ".css", ".js", ".ico", ".svg", ".tiff", ".zip", ".exe",
+ ".dmg", ".mp4", ".mp3", ".wav", ".pptx", ".xlsx", ".avi", ".flv", ".woff", ".ttf", ".woff2",
+ ".webp", ".inc",
+ ];
+ let url_without_query = url.path().to_lowercase();
+ file_extensions
+ .iter()
+ .any(|ext| url_without_query.ends_with(ext))
+}
+
+fn _get_url_depth(path: &str) -> u32 {
+ path
+ .split('/')
+ .filter(|x| *x != "" && *x != "index.php" && *x != "index.html")
+ .count() as u32
+}
+
+fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult, String> {
+ let mut denial_reasons = HashMap::new();
+
+ let limit = data
+ .limit
+ .and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) })
+ .unwrap_or(usize::MAX);
+
+ if limit == 0 {
+ return Ok(FilterLinksResult {
+ links: Vec::with_capacity(0),
+ denial_reasons,
+ });
+ }
+
+ let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {}", e))?;
+ let initial_url = Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {}", e))?;
+
+ let excludes_regex = data
+ .excludes
+ .iter()
+ .map(|exclude| Regex::new(exclude))
+ .collect::<Vec<std::result::Result<Regex, regex::Error>>>();
+ let excludes_regex = excludes_regex
+ .into_iter()
+ .filter_map(|x| x.ok())
+ .collect::<Vec<Regex>>();
+
+ let includes_regex = data
+ .includes
+ .iter()
+ .map(|include| Regex::new(include))
+ .collect::<Vec<std::result::Result<Regex, regex::Error>>>();
+ let includes_regex = includes_regex
+ .into_iter()
+ .filter_map(|x| x.ok())
+ .collect::<Vec<Regex>>();
+
+ let links = data
+ .links
+ .into_iter()
+ .filter(|link| {
+ let url = match base_url.join(link) {
+ Ok(x) => x,
+ Err(_) => {
+ denial_reasons.insert(link.clone(), "URL_PARSE_ERROR".to_string());
+ return false;
+ }
+ };
+
+ let path = url.path();
+ let depth = _get_url_depth(path);
+ if depth > data.max_depth {
+ denial_reasons.insert(link.clone(), "DEPTH_LIMIT".to_string());
+ return false;
+ }
+
+ let exinc_path = if data.regex_on_full_url {
+ url.as_str()
+ } else {
+ url.path()
+ };
+
+ if !excludes_regex.is_empty()
+ && excludes_regex
+ .iter()
+ .any(|regex| regex.is_match(exinc_path))
+ {
+ denial_reasons.insert(link.clone(), "EXCLUDE_PATTERN".to_string());
+ return false;
+ }
+
+ if !includes_regex.is_empty()
+ && !includes_regex
+ .iter()
+ .any(|regex| regex.is_match(exinc_path))
+ {
+ denial_reasons.insert(link.clone(), "INCLUDE_PATTERN".to_string());
+ return false;
+ }
+
+ if !data.allow_backward_crawling {
+ if !url.path().starts_with(initial_url.path()) {
+ denial_reasons.insert(link.clone(), "BACKWARD_CRAWLING".to_string());
+ return false;
+ }
+ }
+
+ if !data.ignore_robots_txt {
+ match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {
+ Ok(robot) => {
+ let allowed = robot.allowed(url.as_str());
+ if !allowed {
+ match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {
+ Ok(robot_alt) => {
+ let allowed_alt = robot_alt.allowed(url.as_str());
+ if !allowed_alt {
+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
+ return false;
+ }
+ }
+ Err(_) => {
+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
+ return false;
+ }
+ }
+ }
+ }
+ Err(_) => {}
+ }
+ }
+
+ if _is_file(&url) {
+ denial_reasons.insert(link.clone(), "FILE_TYPE".to_string());
+ return false;
+ }
+
+ true
+ })
+ .take(limit)
+ .collect::<Vec<_>>();
+
+ Ok(FilterLinksResult {
+ links,
+ denial_reasons,
+ })
+}
+
+/// Filter links based on crawling rules and constraints.
+#[napi]
+pub fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {
+ _filter_links(data)
+ .map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {}", e)))
+}
+
+fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, String> {
+ let doc = roxmltree::Document::parse_with_options(
+ xml_content,
+ roxmltree::ParsingOptions {
+ allow_dtd: true,
+ ..Default::default()
+ },
+ ).map_err(|e| format!("XML parsing error: {}", e))?;
+ let root = doc.root_element();
+
+ match root.tag_name().name() {
+ "sitemapindex" => {
+ let mut sitemaps = Vec::new();
+
+ for sitemap_node in root
+ .children()
+ .filter(|n| n.is_element() && n.tag_name().name() == "sitemap")
+ {
+ if let Some(loc_node) = sitemap_node
+ .children()
+ .find(|n| n.is_element() && n.tag_name().name() == "loc")
+ {
+ if let Some(loc_text) = loc_node.text() {
+ sitemaps.push(SitemapEntry {
+ loc: vec![loc_text.to_string()],
+ });
+ }
+ }
+ }
+
+ Ok(ParsedSitemap {
+ urlset: None,
+ sitemapindex: Some(SitemapIndex { sitemap: sitemaps }),
+ })
+ }
+ "urlset" => {
+ let mut urls = Vec::new();
+
+ for url_node in root
+ .children()
+ .filter(|n| n.is_element() && n.tag_name().name() == "url")
+ {
+ if let Some(loc_node) = url_node
+ .children()
+ .find(|n| n.is_element() && n.tag_name().name() == "loc")
+ {
+ if let Some(loc_text) = loc_node.text() {
+ urls.push(SitemapUrl {
+ loc: vec![loc_text.to_string()],
+ });
+ }
+ }
+ }
+
+ Ok(ParsedSitemap {
+ urlset: Some(SitemapUrlset { url: urls }),
+ sitemapindex: None,
+ })
+ }
+ _ => Err("Invalid sitemap format: root element must be 'sitemapindex' or 'urlset'".to_string()),
+ }
+}
+
+/// Parse XML sitemap content into structured data.
+#[napi]
+pub fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {
+ _parse_sitemap_xml(&xml_content).map_err(|e| {
+ Error::new(
+ Status::GenericFailure,
+ format!("Parse sitemap XML error: {}", e),
+ )
+ })
+}
+
+fn _process_sitemap(
+ xml_content: &str,
+) -> std::result::Result<SitemapProcessingResult, String> {
+ let parsed = _parse_sitemap_xml(xml_content)?;
+ let mut instructions = Vec::new();
+ let mut total_count: u32 = 0;
+
+ if let Some(sitemapindex) = parsed.sitemapindex {
+ let sitemap_urls: Vec<String> = sitemapindex
+ .sitemap
+ .iter()
+ .filter_map(|sitemap| {
+ if !sitemap.loc.is_empty() {
+ Some(sitemap.loc[0].trim().to_string())
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ if !sitemap_urls.is_empty() {
+ instructions.push(SitemapInstruction {
+ action: "recurse".to_string(),
+ urls: sitemap_urls.clone(),
+ count: sitemap_urls.len() as u32,
+ });
+ total_count += sitemap_urls.len() as u32;
+ }
+ } else if let Some(urlset) = parsed.urlset {
+ let mut xml_sitemaps = Vec::new();
+ let mut valid_urls = Vec::new();
+
+ for url_entry in urlset.url {
+ if !url_entry.loc.is_empty() {
+ let url = url_entry.loc[0].trim();
+ if url.to_lowercase().ends_with(".xml") {
+ xml_sitemaps.push(url.to_string());
+ } else if let Ok(parsed_url) = Url::parse(url) {
+ if !_is_file(&parsed_url) {
+ valid_urls.push(url.to_string());
+ }
+ }
+ }
+ }
+
+ if !xml_sitemaps.is_empty() {
+ instructions.push(SitemapInstruction {
+ action: "recurse".to_string(),
+ urls: xml_sitemaps.clone(),
+ count: xml_sitemaps.len() as u32,
+ });
+ total_count += xml_sitemaps.len() as u32;
+ }
+
+ if !valid_urls.is_empty() {
+ instructions.push(SitemapInstruction {
+ action: "process".to_string(),
+ urls: valid_urls.clone(),
+ count: valid_urls.len() as u32,
+ });
+ total_count += valid_urls.len() as u32;
+ }
+ }
+
+ Ok(SitemapProcessingResult {
+ instructions,
+ total_count,
+ })
+}
+
+/// Process sitemap XML and extract crawling instructions.
+#[napi]
+pub fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResult> {
+ _process_sitemap(&xml_content).map_err(|e| {
+ Error::new(
+ Status::GenericFailure,
+ format!("Process sitemap error: {}", e),
+ )
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_parse_sitemap_xml_urlset() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>https://example.com/page1</loc>
+ </url>
+ <url>
+ <loc>https://example.com/page2</loc>
+ </url>
+</urlset>"#;
+
+ let result = _parse_sitemap_xml(xml_content).unwrap();
+ assert!(result.urlset.is_some());
+ let urlset = result.urlset.unwrap();
+ assert_eq!(urlset.url.len(), 2);
+ assert_eq!(urlset.url[0].loc[0], "https://example.com/page1");
+ assert_eq!(urlset.url[1].loc[0], "https://example.com/page2");
+ }
+
+ #[test]
+ fn test_parse_sitemap_xml_sitemapindex() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <sitemap>
+ <loc>https://example.com/sitemap1.xml</loc>
+ </sitemap>
+ <sitemap>
+ <loc>https://example.com/sitemap2.xml</loc>
+ </sitemap>
+</sitemapindex>"#;
+
+ let result = _parse_sitemap_xml(xml_content).unwrap();
+ assert!(result.sitemapindex.is_some());
+ let sitemapindex = result.sitemapindex.unwrap();
+ assert_eq!(sitemapindex.sitemap.len(), 2);
+ assert_eq!(
+ sitemapindex.sitemap[0].loc[0],
+ "https://example.com/sitemap1.xml"
+ );
+ assert_eq!(
+ sitemapindex.sitemap[1].loc[0],
+ "https://example.com/sitemap2.xml"
+ );
+ }
+
+ #[test]
+ fn test_parse_sitemap_xml_invalid_root() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<invalid xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>https://example.com/page1</loc>
+ </url>
+</invalid>"#;
+
+ let result = _parse_sitemap_xml(xml_content);
+ assert!(result.is_err());
+ assert!(result
+ .unwrap_err()
+ .to_string()
+ .contains("Invalid sitemap format"));
+ }
+
+ #[test]
+ fn test_parse_sitemap_xml_malformed() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>https://example.com/page1</loc>
+ </url>
+</urlset"#; // Missing closing >
+
+ let result = _parse_sitemap_xml(xml_content);
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn test_process_sitemap_urlset() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>https://example.com/page1</loc>
+ </url>
+ <url>
+ <loc>https://example.com/sitemap2.xml</loc>
+ </url>
+ <url>
+ <loc>https://example.com/image.png</loc>
+ </url>
+</urlset>"#;
+
+ let result = _process_sitemap(xml_content).unwrap();
+ assert_eq!(result.instructions.len(), 2);
+
+ let recurse_instruction = result
+ .instructions
+ .iter()
+ .find(|i| i.action == "recurse")
+ .unwrap();
+ assert_eq!(recurse_instruction.urls.len(), 1);
+ assert_eq!(
+ recurse_instruction.urls[0],
+ "https://example.com/sitemap2.xml"
+ );
+
+ let process_instruction = result
+ .instructions
+ .iter()
+ .find(|i| i.action == "process")
+ .unwrap();
+ assert_eq!(process_instruction.urls.len(), 1);
+ assert_eq!(process_instruction.urls[0], "https://example.com/page1");
+ }
+
+ #[test]
+ fn test_process_sitemap_sitemapindex() {
+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <sitemap>
+ <loc>https://example.com/sitemap1.xml</loc>
+ </sitemap>
+ <sitemap>
+ <loc>https://example.com/sitemap2.xml</loc>
+ </sitemap>
+</sitemapindex>"#;
+
+ let result = _process_sitemap(xml_content).unwrap();
+ assert_eq!(result.instructions.len(), 1);
+ assert_eq!(result.instructions[0].action, "recurse");
+ assert_eq!(result.instructions[0].urls.len(), 2);
+ assert_eq!(
+ result.instructions[0].urls[0],
+ "https://example.com/sitemap1.xml"
+ );
+ assert_eq!(
+ result.instructions[0].urls[1],
+ "https://example.com/sitemap2.xml"
+ );
+ }
+
+ #[test]
+ fn test_filter_links_normal_robots_txt() {
+ let data = FilterLinksCall {
+ links: vec![
+ "https://example.com/allowed".to_string(),
+ "https://example.com/disallowed".to_string(),
+ ],
+ limit: Some(10),
+ includes: vec![],
+ excludes: vec![],
+ ignore_robots_txt: false,
+ robots_txt: "User-agent: *\nDisallow: /disallowed".to_string(),
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ };
+
+ let result = _filter_links(data).unwrap();
+ assert_eq!(result.links.len(), 1);
+ assert_eq!(result.links[0], "https://example.com/allowed");
+ assert!(result
+ .denial_reasons
+ .contains_key("https://example.com/disallowed"));
+ assert_eq!(
+ result
+ .denial_reasons
+ .get("https://example.com/disallowed")
+ .unwrap(),
+ "ROBOTS_TXT"
+ );
+ }
+
+ #[test]
+ fn test_filter_links_malformed_robots_txt() {
+ let data = FilterLinksCall {
+ links: vec!["https://example.com/test".to_string()],
+ limit: Some(10),
+ includes: vec![],
+ excludes: vec![],
+ ignore_robots_txt: false,
+ robots_txt: "Invalid robots.txt content with \x00 null bytes and malformed syntax"
+ .to_string(),
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ };
+
+ let result = _filter_links(data);
+ assert!(result.is_ok());
+ let result = result.unwrap();
+ assert_eq!(result.links.len(), 1);
+ assert_eq!(result.links[0], "https://example.com/test");
+ }
+
+ #[test]
+ fn test_filter_links_non_utf8_robots_txt() {
+ let mut non_utf8_bytes = vec![0xFF, 0xFE];
+ non_utf8_bytes.extend_from_slice(b"User-agent: *\nDisallow: /blocked");
+ let non_utf8_string = String::from_utf8_lossy(&non_utf8_bytes).to_string();
+
+ let data = FilterLinksCall {
+ links: vec!["https://example.com/allowed".to_string()],
+ limit: Some(10),
+ includes: vec![],
+ excludes: vec![],
+ ignore_robots_txt: false,
+ robots_txt: non_utf8_string,
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ };
+
+ let result = _filter_links(data);
+ assert!(result.is_ok());
+ let result = result.unwrap();
+ assert_eq!(result.links.len(), 1);
+ assert_eq!(result.links[0], "https://example.com/allowed");
+ }
+
+ #[test]
+ fn test_filter_links_char_boundary_issue() {
+ let problematic_content = "User-agent: *\nDisallow: /\u{a0}test";
+
+ let data = FilterLinksCall {
+ links: vec!["https://example.com/test".to_string()],
+ limit: Some(10),
+ includes: vec![],
+ excludes: vec![],
+ ignore_robots_txt: false,
+ robots_txt: problematic_content.to_string(),
+ max_depth: 10,
+ base_url: "https://example.com".to_string(),
+ initial_url: "https://example.com".to_string(),
+ regex_on_full_url: false,
+ allow_backward_crawling: true,
+ };
+
+ let result = _filter_links(data);
+ assert!(result.is_ok());
+ let result = result.unwrap();
+ assert_eq!(result.links.len(), 1);
+ assert_eq!(result.links[0], "https://example.com/test");
+ }
+}
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rs
new file mode 100644
index 000000000..3d2372c36
--- /dev/null
+++ b/apps/api/native/src/html.rs
@@ -0,0 +1,810 @@
+use std::collections::{HashMap, HashSet};
+
+use kuchikiki::{iter::NodeEdge, parse_html, traits::TendrilSink, NodeRef};
+use napi_derive::napi;
+use nodesig::{get_node_signature, SignatureMode};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use url::Url;
+
+use crate::utils::to_napi_err;
+
+fn _extract_base_href_from_document(
+ document: &NodeRef,
+ url: &Url,
+) -> Result<String, Box<dyn std::error::Error>> {
+ if let Some(base) = document
+ .select("base[href]")
+ .map_err(|_| "Failed to select base href".to_string())?
+ .next()
+ .and_then(|base| base.attributes.borrow().get("href").map(|x| x.to_string()))
+ {
+ if let Ok(base) = url.join(&base) {
+ return Ok(base.to_string());
+ }
+ }
+
+ Ok(url.to_string())
+}
+
+fn _extract_base_href(html: &str, url: &str) -> Result<String, Box<dyn std::error::Error>> {
+ let document = parse_html().one(html);
+ let url = Url::parse(url)?;
+ _extract_base_href_from_document(&document, &url)
+}
+
+/// Extract the base href from HTML document.
+#[napi]
+pub fn extract_base_href(html: String, url: String) -> napi::Result<String> {
+ _extract_base_href(&html, &url).map_err(to_napi_err)
+}
+
+/// Extract all links from HTML document.
+#[napi]
+pub fn extract_links(html: Option<String>) -> napi::Result<Vec<String>> {
+ let html = match html {
+ Some(h) => h,
+ None => return Ok(Vec::new()),
+ };
+
+ let document = parse_html().one(html.as_str());
+
+ let anchors: Vec<_> = document
+ .select("a[href]")
+ .map_err(|_| to_napi_err("Failed to select links"))?
+ .collect();
+
+ let mut out: Vec<String> = Vec::new();
+
+ for anchor in anchors {
+ let mut href = match anchor.attributes.borrow().get("href") {
+ Some(x) => x.to_string(),
+ None => continue,
+ };
+
+ if href.starts_with("http:/") && !href.starts_with("http://") {
+ href = format!("http://{}", &href[6..]);
+ } else if href.starts_with("https:/") && !href.starts_with("https://") {
+ href = format!("https://{}", &href[7..]);
+ }
+
+ out.push(href);
+ }
+
+ Ok(out)
+}
+
+macro_rules! insert_meta_name {
+ ($out:ident, $document:ident, $metaName:expr, $outName:expr) => {
+ if let Some(x) = $document
+ .select(&format!("meta[name=\"{}\"]", $metaName))
+ .map_err(|_| "Failed to select meta name")?
+ .next()
+ .and_then(|description| {
+ description
+ .attributes
+ .borrow()
+ .get("content")
+ .map(|x| x.to_string())
+ })
+ {
+ $out.insert(($outName).to_string(), Value::String(x));
+ }
+ };
+}
+
+macro_rules! insert_meta_property {
+ ($out:ident, $document:ident, $metaName:expr, $outName:expr) => {
+ if let Some(x) = $document
+ .select(&format!("meta[property=\"{}\"]", $metaName))
+ .map_err(|_| "Failed to select meta property")?
+ .next()
+ .and_then(|description| {
+ description
+ .attributes
+ .borrow()
+ .get("content")
+ .map(|x| x.to_string())
+ })
+ {
+ $out.insert(($outName).to_string(), Value::String(x));
+ }
+ };
+}
+
+fn _extract_metadata(html: &str) -> Result<HashMap<String, Value>, Box<dyn std::error::Error>> {
+ let document = parse_html().one(html);
+ let mut out = HashMap::<String, Value>::new();
+
+ if let Some(title) = document
+ .select("title")
+ .map_err(|_| "Failed to select title")?
+ .next()
+ {
+ out.insert("title".to_string(), Value::String(title.text_contents()));
+ }
+
+ if let Some(favicon_link) = document
+ .select("link[rel=\"icon\"]")
+ .map_err(|_| "Failed to select favicon")?
+ .next()
+ .and_then(|x| x.attributes.borrow().get("href").map(|x| x.to_string()))
+ .or_else(|| {
+ document
+ .select("link[rel*=\"icon\"]")
+ .ok()
+ .and_then(|mut x| {
+ x.next()
+ .and_then(|x| x.attributes.borrow().get("href").map(|x| x.to_string()))
+ })
+ })
+ {
+ out.insert("favicon".to_string(), Value::String(favicon_link));
+ }
+
+ if let Some(lang) = document
+ .select("html[lang]")
+ .map_err(|_| "Failed to select lang")?
+ .next()
+ .and_then(|x| x.attributes.borrow().get("lang").map(|x| x.to_string()))
+ {
+ out.insert("language".to_string(), Value::String(lang));
+ }
+
+ insert_meta_property!(out, document, "og:title", "ogTitle");
+ insert_meta_property!(out, document, "og:description", "ogDescription");
+ insert_meta_property!(out, document, "og:url", "ogUrl");
+ insert_meta_property!(out, document, "og:image", "ogImage");
+ insert_meta_property!(out, document, "og:audio", "ogAudio");
+ insert_meta_property!(out, document, "og:determiner", "ogDeterminer");
+ insert_meta_property!(out, document, "og:locale", "ogLocale");
+
+ for meta in document
+ .select("meta[property=\"og:locale:alternate\"]")
+ .map_err(|_| "Failed to select og locale alternate")?
+ {
+ let attrs = meta.attributes.borrow();
+
+ if let Some(content) = attrs.get("content") {
+ if let Some(v) = out.get_mut("ogLocaleAlternate") {
+ match v {
+ Value::Array(x) => x.push(Value::String(content.to_string())),
+ _ => unreachable!(),
+ }
+ } else {
+ out.insert(
+ "ogLocaleAlternate".to_string(),
+ Value::Array(vec![Value::String(content.to_string())]),
+ );
+ }
+ }
+ }
+
+ insert_meta_property!(out, document, "og:site_name", "ogSiteName");
+ insert_meta_property!(out, document, "og:video", "ogVideo");
+ insert_meta_name!(out, document, "article:section", "articleSection");
+ insert_meta_name!(out, document, "article:tag", "articleTag");
+ insert_meta_property!(out, document, "article:published_time", "publishedTime");
+ insert_meta_property!(out, document, "article:modified_time", "modifiedTime");
+ insert_meta_name!(out, document, "dcterms.keywords", "dcTermsKeywords");
+ insert_meta_name!(out, document, "dc.description", "dcDescription");
+ insert_meta_name!(out, document, "dc.subject", "dcSubject");
+ insert_meta_name!(out, document, "dcterms.subject", "dcTermsSubject");
+ insert_meta_name!(out, document, "dcterms.audience", "dcTermsAudience");
+ insert_meta_name!(out, document, "dc.type", "dcType");
+ insert_meta_name!(out, document, "dcterms.type", "dcTermsType");
+ insert_meta_name!(out, document, "dc.date", "dcDate");
+ insert_meta_name!(out, document, "dc.date.created", "dcDateCreated");
+ insert_meta_name!(out, document, "dcterms.created", "dcTermsCreated");
+
+ for meta in document
+ .select("meta")
+ .map_err(|_| "Failed to select meta")?
+ {
+ let meta = meta.as_node().as_element().unwrap();
+ let attrs = meta.attributes.borrow();
+
+ if let Some(name) = attrs
+ .get("name")
+ .or_else(|| attrs.get("property"))
+ .or_else(|| attrs.get("itemprop"))
+ {
+ if let Some(content) = attrs.get("content") {
+ if let Some(v) = out.get(name) {
+ match v {
+ Value::String(existing) => {
+ if name == "description" {
+ out.insert(
+ name.to_string(),
+ Value::String(format!("{}, {}", existing, content)),
+ );
+ } else if name != "title" {
+ out.insert(
+ name.to_string(),
+ Value::Array(vec![
+ Value::String(existing.clone()),
+ Value::String(content.to_string()),
+ ]),
+ );
+ }
+ }
+ Value::Array(existing_array) => {
+ if name == "description" {
+ let mut values: Vec<String> = existing_array
+ .iter()
+ .filter_map(|v| match v {
+ Value::String(s) => Some(s.clone()),
+ _ => None,
+ })
+ .collect();
+ values.push(content.to_string());
+ out.insert(name.to_string(), Value::String(values.join(", ")));
+ } else {
+ match out.get_mut(name) {
+ Some(Value::Array(x)) => x.push(Value::String(content.to_string())),
+ _ => unreachable!(),
+ }
+ }
+ }
+ _ => unreachable!(),
+ }
+ } else {
+ out.insert(name.to_string(), Value::String(content.to_string()));
+ }
+ }
+ }
+ }
+
+ Ok(out)
+}
+
+/// Extract metadata from HTML document.
+#[napi]
+pub fn extract_metadata(html: Option<String>) -> napi::Result<HashMap<String, Value>> {
+ let html = match html {
+ Some(h) => h,
+ None => return Ok(HashMap::new()),
+ };
+
+ _extract_metadata(&html).map_err(to_napi_err)
+}
+
+const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [
+ "header",
+ "footer",
+ "nav",
+ "aside",
+ ".header",
+ ".top",
+ ".navbar",
+ "#header",
+ ".footer",
+ ".bottom",
+ "#footer",
+ ".sidebar",
+ ".side",
+ ".aside",
+ "#sidebar",
+ ".modal",
+ ".popup",
+ "#modal",
+ ".overlay",
+ ".ad",
+ ".ads",
+ ".advert",
+ "#ad",
+ ".lang-selector",
+ ".language",
+ "#language-selector",
+ ".social",
+ ".social-media",
+ ".social-links",
+ "#social",
+ ".menu",
+ ".navigation",
+ "#nav",
+ ".breadcrumbs",
+ "#breadcrumbs",
+ ".share",
+ "#share",
+ ".widget",
+ "#widget",
+ ".cookie",
+ "#cookie",
+ ".fc-decoration",
+];
+
+const FORCE_INCLUDE_MAIN_TAGS: [&str; 13] = [
+ "#main",
+ ".swoogo-cols",
+ ".swoogo-text",
+ ".swoogo-table-div",
+ ".swoogo-space",
+ ".swoogo-alert",
+ ".swoogo-sponsors",
+ ".swoogo-title",
+ ".swoogo-tabs",
+ ".swoogo-logo",
+ ".swoogo-image",
+ ".swoogo-button",
+ ".swoogo-agenda",
+];
+
+#[derive(Deserialize, Serialize)]
+#[napi(object)]
+pub struct TransformHtmlOptions {
+ pub html: String,
+ pub url: String,
+ #[serde(default)]
+ pub include_tags: Vec<String>,
+ #[serde(default)]
+ pub exclude_tags: Vec<String>,
+ pub only_main_content: bool,
+ pub omce_signatures: Option<Vec<String>>,
+}
+
+struct ImageSource {
+ url: String,
+ size: f64,
+ is_x: bool,
+}
+
+fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn std::error::Error>> {
+ let mut document = parse_html().one(opts.html.as_ref());
+ let url = Url::parse(&_extract_base_href_from_document(
+ &document,
+ &Url::parse(&opts.url)?,
+ )?)?;
+
+ if !opts.include_tags.is_empty() {
+ let new_document = parse_html().one("<div></div>");
+ let root = new_document
+ .select_first("div")
+ .map_err(|_| "Failed to select root element")?;
+
+ for x in opts.include_tags.iter() {
+ let matching_nodes: Vec<_> = document
+ .select(x)
+ .map_err(|_| "Failed to include_tags tags")?
+ .collect();
+ for tag in matching_nodes {
+ root.as_node().append(tag.as_node().clone());
+ }
+ }
+
+ document = new_document;
+ }
+
+ while let Ok(x) = document.select_first("head") {
+ x.as_node().detach();
+ }
+ while let Ok(x) = document.select_first("meta") {
+ x.as_node().detach();
+ }
+ while let Ok(x) = document.select_first("noscript") {
+ x.as_node().detach();
+ }
+ while let Ok(x) = document.select_first("style") {
+ x.as_node().detach();
+ }
+ while let Ok(x) = document.select_first("script") {
+ x.as_node().detach();
+ }
+
+ // OMCE first
+ if opts.only_main_content {
+ if let Some(signatures) = opts.omce_signatures.as_ref() {
+ let mut nodes_to_drop: Vec<NodeRef> = Vec::new();
+
+ let modes = signatures
+ .iter()
+ .map(|x| Into::<SignatureMode>::into(x.split(':').nth(1).unwrap().to_string()))
+ .collect::<HashSet<_>>();
+
+ for mode in modes {
+ let matcher = format!(":{}:", Into::<String>::into(mode.clone()));
+ let signatures = signatures
+ .iter()
+ .filter(|x| x.contains(&matcher))
+ .cloned()
+ .collect::<HashSet<_>>();
+
+ for edge in document.traverse() {
+ match edge {
+ NodeEdge::Start(_) => {}
+ NodeEdge::End(node) => {
+ if node.as_element().is_none() {
+ continue;
+ }
+ if node.text_contents().trim().is_empty() {
+ continue;
+ }
+
+ let signature = get_node_signature(&node, mode.clone());
+ if signatures.contains(&signature) {
+ nodes_to_drop.push(node);
+ }
+ }
+ }
+ }
+ }
+
+ for node in nodes_to_drop {
+ node.detach();
+ }
+ }
+ }
+
+ for x in opts.exclude_tags.iter() {
+ while let Ok(x) = document.select_first(x) {
+ x.as_node().detach();
+ }
+ }
+
+ if opts.only_main_content {
+ for x in EXCLUDE_NON_MAIN_TAGS.iter() {
+ let x: Vec<_> = document
+ .select(x)
+ .map_err(|_| "Failed to select tags")?
+ .collect();
+ for tag in x {
+ if !FORCE_INCLUDE_MAIN_TAGS.iter().any(|x| {
+ tag
+ .as_node()
+ .select(x)
+ .is_ok_and(|mut x| x.next().is_some())
+ }) {
+ tag.as_node().detach();
+ }
+ }
+ }
+ }
+
+ let srcset_images: Vec<_> = document
+ .select("img[srcset]")
+ .map_err(|_| "Failed to select srcset images")?
+ .collect();
+ for img in srcset_images {
+ let mut sizes: Vec<ImageSource> = img
+ .attributes
+ .borrow()
+ .get("srcset")
+ .ok_or("Failed to get srcset")?
+ .split(',')
+ .filter_map(|x| {
+ let tok: Vec<&str> = x.trim().split(' ').collect();
+ let last_token = tok[tok.len() - 1];
+ let (last_token, last_token_used) = if tok.len() > 1
+ && !last_token.is_empty()
+ && (last_token.ends_with('x') || last_token.ends_with('w'))
+ {
+ (last_token, true)
+ } else {
+ ("1x", false)
+ };
+
+ if let Some((last_index, _)) = last_token.char_indices().last() {
+ if let Ok(parsed_size) = last_token[..last_index].parse() {
+ Some(ImageSource {
+ url: if last_token_used {
+ tok[0..tok.len() - 1].join(" ")
+ } else {
+ tok.join(" ")
+ },
+ size: parsed_size,
+ is_x: last_token.ends_with('x'),
+ })
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ if sizes.iter().all(|x| x.is_x) {
+ if let Some(src) = img.attributes.borrow().get("src").map(|x| x.to_string()) {
+ sizes.push(ImageSource {
+ url: src,
+ size: 1.0,
+ is_x: true,
+ });
+ }
+ }
+
+ sizes.sort_by(|a, b| {
+ b.size
+ .partial_cmp(&a.size)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ });
+
+ if let Some(biggest) = sizes.first() {
+ img
+ .attributes
+ .borrow_mut()
+ .insert("src", biggest.url.clone());
+ }
+ }
+
+ let src_images: Vec<_> = document
+ .select("img[src]")
+ .map_err(|_| "Failed to select src images")?
+ .collect();
+ for img in src_images {
+ let old = img
+ .attributes
+ .borrow()
+ .get("src")
+ .map(|x| x.to_string())
+ .ok_or("Failed to get src")?;
+ if let Ok(new) = url.join(&old) {
+ img.attributes.borrow_mut().insert("src", new.to_string());
+ }
+ }
+
+ let href_anchors: Vec<_> = document
+ .select("a[href]")
+ .map_err(|_| "Failed to select href anchors")?
+ .collect();
+ for anchor in href_anchors {
+ let old = anchor
+ .attributes
+ .borrow()
+ .get("href")
+ .map(|x| x.to_string())
+ .ok_or("Failed to get href")?;
+ if let Ok(new) = url.join(&old) {
+ anchor
+ .attributes
+ .borrow_mut()
+ .insert("href", new.to_string());
+ }
+ }
+
+ Ok(document.to_string())
+}
+
+/// Transform and clean HTML content based on provided options.
+#[napi]
+pub fn transform_html(opts: TransformHtmlOptions) -> napi::Result<String> {
+ _transform_html_inner(opts).map_err(to_napi_err)
+}
+
+fn _get_inner_json(html: &str) -> Result<String, ()> {
+ Ok(parse_html().one(html).select_first("body")?.text_contents())
+}
+
+/// Extract inner text content from HTML body.
+#[napi]
+pub fn get_inner_json(html: String) -> napi::Result<String> {
+ _get_inner_json(&html).map_err(|_| to_napi_err("Failed to get inner JSON"))
+}
+
+#[derive(Deserialize, Serialize)]
+#[napi(object)]
+pub struct AttributeSelector {
+ pub selector: String,
+ pub attribute: String,
+}
+
+#[derive(Deserialize, Serialize)]
+#[napi(object)]
+pub struct ExtractAttributesOptions {
+ pub selectors: Vec<AttributeSelector>,
+}
+
+#[derive(Serialize)]
+#[napi(object)]
+pub struct ExtractedAttributeResult {
+ pub selector: String,
+ pub attribute: String,
+ pub values: Vec<String>,
+}
+
+fn _extract_attributes(
+ html: &str,
+ options: &ExtractAttributesOptions,
+) -> Result<Vec<ExtractedAttributeResult>, Box<dyn std::error::Error>> {
+ let document = parse_html().one(html);
+ let mut results = Vec::new();
+
+ for selector_config in &options.selectors {
+ let mut values = Vec::new();
+
+ let elements: Vec<_> = match document.select(&selector_config.selector).map_err(|_| {
+ format!(
+ "Failed to select with selector: {}",
+ selector_config.selector
+ )
+ }) {
+ Ok(x) => x.collect(),
+ Err(_) => Vec::new(), // invalid selector => empty list
+ };
+
+ for element in elements {
+ if let Some(attr_value) = element
+ .attributes
+ .borrow()
+ .get(selector_config.attribute.as_str())
+ {
+ values.push(attr_value.to_string());
+ continue;
+ }
+
+ if !selector_config.attribute.starts_with("data-") {
+ let data_attr = format!("data-{}", selector_config.attribute);
+ if let Some(attr_value) = element.attributes.borrow().get(data_attr.as_str()) {
+ values.push(attr_value.to_string());
+ }
+ }
+ }
+
+ results.push(ExtractedAttributeResult {
+ selector: selector_config.selector.clone(),
+ attribute: selector_config.attribute.clone(),
+ values,
+ });
+ }
+
+ Ok(results)
+}
+
+/// Extract specified attributes from HTML elements matching selectors.
+#[napi]
+pub fn extract_attributes(
+ html: String,
+ options: ExtractAttributesOptions,
+) -> napi::Result<Vec<ExtractedAttributeResult>> {
+ _extract_attributes(&html, &options).map_err(to_napi_err)
+}
+
+fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn std::error::Error>> {
+ let document = parse_html().one(html);
+ let base_url = Url::parse(base_url)?;
+ let base_href = _extract_base_href_from_document(&document, &base_url)?;
+ let base_href_url = Url::parse(&base_href)?;
+ let mut images = HashSet::<String>::new();
+
+ let resolve_image_url = |src: &str| -> Result<String, Box<dyn std::error::Error>> {
+ if src.starts_with("data:") || src.starts_with("blob:") {
+ return Ok(src.to_string());
+ }
+ if src.starts_with("http://") || src.starts_with("https://") {
+ return Ok(src.to_string());
+ }
+ if src.starts_with("//") {
+ let resolved = base_url.join(src)?;
+ return Ok(resolved.to_string());
+ }
+ let resolved = base_href_url.join(src)?;
+ Ok(resolved.to_string())
+ };
+
+ // <img>
+ let img_elements: Vec<_> = match document
+ .select("img")
+ .map_err(|_| "Failed to select img tags")
+ {
+ Ok(x) => x.collect(),
+ Err(e) => return Err(e.into()),
+ };
+
+ for img in img_elements {
+ let attrs = img.attributes.borrow();
+
+ if let Some(src) = attrs.get("src") {
+ if let Ok(resolved) = resolve_image_url(src) {
+ images.insert(resolved);
+ }
+ }
+
+ if let Some(data_src) = attrs.get("data-src") {
+ if let Ok(resolved) = resolve_image_url(data_src) {
+ images.insert(resolved);
+ }
+ }
+
+ if let Some(srcset) = attrs.get("srcset") {
+ for part in srcset.split(',') {
+ if let Some(url) = part.trim().split_whitespace().next() {
+ if !url.is_empty() {
+ if let Ok(resolved) = resolve_image_url(url) {
+ images.insert(resolved);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // <picture><source>
+ let source_elements: Vec<_> = match document
+ .select("picture source")
+ .map_err(|_| "Failed to select picture source")
+ {
+ Ok(x) => x.collect(),
+ Err(_) => Vec::new(),
+ };
+
+ for source in source_elements {
+ if let Some(srcset) = source.attributes.borrow().get("srcset") {
+ for part in srcset.split(',') {
+ if let Some(url) = part.trim().split_whitespace().next() {
+ if !url.is_empty() {
+ if let Ok(resolved) = resolve_image_url(url) {
+ images.insert(resolved);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // OG/Twitter images
+ let meta_selectors = [
+ "meta[property=\"og:image\"]",
+ "meta[property=\"og:image:url\"]",
+ "meta[property=\"og:image:secure_url\"]",
+ "meta[name=\"twitter:image\"]",
+ "meta[name=\"twitter:image:src\"]",
+ "meta[itemprop=\"image\"]",
+ ];
+
+ for selector in &meta_selectors {
+ if let Ok(elements) = document.select(selector) {
+ for element in elements {
+ if let Some(content) = element.attributes.borrow().get("content") {
+ if let Ok(resolved) = resolve_image_url(content) {
+ images.insert(resolved);
+ }
+ }
+ }
+ }
+ }
+
+ // icons
+ let link_selectors = [
+ "link[rel*=\"icon\"]",
+ "link[rel*=\"apple-touch-icon\"]",
+ "link[rel*=\"image_src\"]",
+ ];
+
+ for selector in &link_selectors {
+ if let Ok(elements) = document.select(selector) {
+ for element in elements {
+ if let Some(href) = element.attributes.borrow().get("href") {
+ if let Ok(resolved) = resolve_image_url(href) {
+ images.insert(resolved);
+ }
+ }
+ }
+ }
+ }
+
+ // <video poster="">
+ if let Ok(video_elements) = document.select("video[poster]") {
+ for video in video_elements {
+ if let Some(poster) = video.attributes.borrow().get("poster") {
+ if let Ok(resolved) = resolve_image_url(poster) {
+ images.insert(resolved);
+ }
+ }
+ }
+ }
+
+ let filtered_images: Vec<String> = images
+ .into_iter()
+ .filter(|url| !url.to_lowercase().starts_with("javascript:"))
+ .filter(|url| !url.is_empty())
+ .filter(|url| url.starts_with("data:") || url.starts_with("blob:") || Url::parse(url).is_ok())
+ .collect();
+
+ Ok(filtered_images)
+}
+
+/// Extract all image URLs from HTML document.
+#[napi]
+pub fn extract_images(html: String, base_url: String) -> napi::Result<Vec<String>> {
+ _extract_images(&html, &base_url).map_err(to_napi_err)
+}
diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rs
new file mode 100644
index 000000000..06cfebea6
--- /dev/null
+++ b/apps/api/native/src/lib.rs
@@ -0,0 +1,14 @@
+#![deny(clippy::all)]
+
+pub use crate::crawler::*;
+pub use crate::html::*;
+pub use crate::pdf::*;
+pub use crate::utils::*;
+
+mod crawler;
+mod html;
+mod pdf;
+mod utils;
+
+pub use napi::bindgen_prelude::*;
+pub use serde::{Deserialize, Serialize};
diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rs
new file mode 100644
index 000000000..8b144d042
--- /dev/null
+++ b/apps/api/native/src/pdf.rs
@@ -0,0 +1,64 @@
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+use serde::Serialize;
+
+#[derive(Debug, Clone, Serialize)]
+#[napi(object)]
+pub struct PDFMetadata {
+ pub num_pages: i32,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub title: Option<String>,
+}
+
+fn _get_pdf_metadata(path: &str) -> std::result::Result<PDFMetadata, String> {
+ let doc = match lopdf::Document::load(path) {
+ Ok(x) => x,
+ Err(_) => {
+ return Err("Failed to load PDF".to_string());
+ }
+ };
+
+ let num_pages = doc.get_pages().len() as i32;
+
+ let title = doc
+ .trailer
+ .get(b"Info")
+ .and_then(|info| {
+ info
+ .as_dict()
+ .and_then(|info| info.get(b"Title"))
+ .and_then(|title| {
+ title
+ .as_str()
+ .map(|s| String::from_utf8_lossy(s).to_string())
+ })
+ })
+ .ok()
+ .or_else(|| {
+ doc.objects.iter().find_map(|(_i, obj)| {
+ obj
+ .as_dict()
+ .and_then(|obj| obj.get(b"Title"))
+ .and_then(|title| title.as_str())
+ .map(|s| String::from_utf8_lossy(s).to_string())
+ .ok()
+ })
+ })
+ .map(|x| {
+ x.trim_matches(|x| char::is_whitespace(x) || x == '\0')
+ .to_string()
+ });
+
+ Ok(PDFMetadata { num_pages, title })
+}
+
+/// Extract metadata from PDF file.
+#[napi]
+pub fn get_pdf_metadata(path: String) -> Result<PDFMetadata> {
+ _get_pdf_metadata(&path).map_err(|e| {
+ Error::new(
+ Status::GenericFailure,
+ format!("Failed to get PDF metadata: {}", e),
+ )
+ })
+}
diff --git a/apps/api/native/src/utils.rs b/apps/api/native/src/utils.rs
new file mode 100644
index 000000000..3d78d06c5
--- /dev/null
+++ b/apps/api/native/src/utils.rs
@@ -0,0 +1,5 @@
+use napi::bindgen_prelude::*;
+
+pub fn to_napi_err<E: std::fmt::Display>(error: E) -> Error {
+ Error::new(Status::GenericFailure, error.to_string())
+}
diff --git a/apps/api/native/tsconfig.json b/apps/api/native/tsconfig.json
new file mode 100644
index 000000000..5056788c0
--- /dev/null
+++ b/apps/api/native/tsconfig.json
@@ -0,0 +1,14 @@
+{
+ "compilerOptions": {
+ "target": "ESNext",
+ "strict": true,
+ "moduleResolution": "node",
+ "module": "CommonJS",
+ "noUnusedLocals": true,
+ "noUnusedParameters": true,
+ "esModuleInterop": true,
+ "allowSyntheticDefaultImports": true
+ },
+ "include": ["."],
+ "exclude": ["node_modules", "bench", "__test__"]
+}
diff --git a/apps/api/native/wasi-worker-browser.mjs b/apps/api/native/wasi-worker-browser.mjs
new file mode 100644
index 000000000..8b1b17221
--- /dev/null
+++ b/apps/api/native/wasi-worker-browser.mjs
@@ -0,0 +1,32 @@
+import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'
+
+const handler = new MessageHandler({
+ onLoad({ wasmModule, wasmMemory }) {
+ const wasi = new WASI({
+ print: function () {
+ // eslint-disable-next-line no-console
+ console.log.apply(console, arguments)
+ },
+ printErr: function() {
+ // eslint-disable-next-line no-console
+ console.error.apply(console, arguments)
+ },
+ })
+ return instantiateNapiModuleSync(wasmModule, {
+ childThread: true,
+ wasi,
+ overwriteImports(importObject) {
+ importObject.env = {
+ ...importObject.env,
+ ...importObject.napi,
+ ...importObject.emnapi,
+ memory: wasmMemory,
+ }
+ },
+ })
+ },
+})
+
+globalThis.onmessage = function (e) {
+ handler.handle(e)
+}
diff --git a/apps/api/package.json b/apps/api/package.json
index a5b3a7a7e..be164054e 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -159,7 +159,8 @@
"ws": "^8.18.0",
"x402-express": "^0.5.3",
"xml2js": "^0.6.2",
- "zod": "^3.24.2"
+ "zod": "^3.24.2",
+ "@mendable/firecrawl-rs": "workspace:*"
},
"nodemonConfig": {
"ignore": [
@@ -179,5 +180,6 @@
},
"lint-staged": {
"*.{js,jsx,ts,tsx,json,css,md}": "prettier --write"
- }
+ },
+ "packageManager": "[email protected]"
}
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 715614fe4..b04f60409 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -3,6 +3,7 @@ lockfileVersion: '9.0'
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
+ injectWorkspacePackages: true
overrides:
brace-expansion@>=2.0.0 <=2.0.1: 2.0.2
@@ -56,6 +57,9 @@ importers:
'@google-cloud/storage':
specifier: ^7.16.0
version: 7.16.0([email protected])
+ '@mendable/firecrawl-rs':
+ specifier: workspace:*
+ version: link:native
'@nangohq/node':
specifier: ^0.40.8
version: 0.40.8
@@ -391,6 +395,42 @@ importers:
specifier: ^5.8.3
version: 5.8.3
+ native:
+ devDependencies:
+ '@emnapi/core':
+ specifier: ^1.5.0
+ version: 1.5.0
+ '@emnapi/runtime':
+ specifier: ^1.5.0
+ version: 1.5.0
+ '@napi-rs/cli':
+ specifier: ^3.1.5
+ version: 3.1.5(@emnapi/[email protected])(@types/[email protected])
+ '@oxc-node/core':
+ specifier: ^0.0.32
+ version: 0.0.32
+ '@taplo/cli':
+ specifier: ^0.7.0
+ version: 0.7.0
+ '@tybys/wasm-util':
+ specifier: ^0.10.0
+ version: 0.10.0
+ chalk:
+ specifier: ^5.6.0
+ version: 5.6.0
+ npm-run-all2:
+ specifier: ^8.0.4
+ version: 8.0.4
+ oxlint:
+ specifier: ^1.14.0
+ version: 1.14.0
+ prettier:
+ specifier: ^3.6.2
+ version: 3.6.2
+ typescript:
+ specifier: ^5.9.2
+ version: 5.9.2
+
packages:
'@adraffy/[email protected]':
@@ -983,6 +1023,15 @@ packages:
peerDependencies:
'@noble/ciphers': ^1.0.0
+ '@emnapi/[email protected]':
+ resolution: {integrity: sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==}
+
+ '@emnapi/[email protected]':
+ resolution: {integrity: sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==}
+
+ '@emnapi/[email protected]':
+ resolution: {integrity: sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==}
+
'@esbuild/[email protected]':
resolution: {integrity: sha512-urAvrUedIqEiFR3FYSLTWQgLu5tb+m0qZw0NBEasUeo6wuqatkMDaRT+1uABiGXEu5vqgPd7FGE1BhsAIy9QVA==}
engines: {node: '>=18'}
@@ -1180,6 +1229,136 @@ packages:
engines: {node: '>=6'}
hasBin: true
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-E+KExNurKcUJJdxmjglTl141EwxWyAHplvsYJQgSwXf8qiNWkTxTuCCqmhFEmbIXd4zLaGMfQFJ6WrZ7fSeV3g==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-j1a5VstaK5KQy8Mu8cHmuQvN1Zc62TbLhjJxwHvKPPKEoowSF6h/0UdOpA9DNdWZ+9Inq73+puRq1df6OJ8Sag==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-NyDSjPqhSvpZEMZrLCYUquWNl+XC/moEcVFqS55IEYIYsY0a1cUCevSqk7ctOlnm/RaSBU5psFryNlxcmGrjaA==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-yeQN3AXjCm7+Hmq5L6Dm2wEDeBRdAZuyZ4I7tWSSanbxDzqM0KqzoDbKM7p4ebllAYdoQuPJS6N71/3L281i6w==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-xUjteYtavH7HwDMzq4Cn2X4Qsh5NozoDHCJTdoXg9HfZ4w3R6mxV1B9tL7DGJX2eq/zqtsFjhm0/RJIMGlh3ag==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-Oau4yL24d2B5IL4ma4UpbQigkVhzPDXLoqy1ggK4gnHg/stmkffJE4oOXHXF3uz0UEpywG68KcyXsyYpA1Re/Q==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-lGPVU3yO9ZNqA7vTYz26jny41lE7yoQansmqdMLBEfqaGsmdg7V3W9mK9Pvb5IL4EVZ9GnSDGMO/cJXud5dMaw==}
+ engines: {node: '>=18'}
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-hqOvBZj/MhQCpHUuD3MVq18SSoDNHy7wEnQ8mtvs71K8OPZVXJinOzcvQna33dNYLYE4LkA9BlhAhK6MJcsVbw==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-7exgBm52WXZRczsydCVftozFTrrwbG5ySE0GqUd2zLNSBXyIucs2Wnm7ZKLe/aUu6NUg9dg7Q80QIHCdZJiY4A==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-zXvzAGxPQTNk/SbT3carAD4Iqi6A2JS2qtcqQjsL22uvD+JfQzUrDEtPjLL7PLn8zlSNyPdY02IiQjzoL9TStA==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-MuxVZ1en1g5oGamXV3DWP89GEkdD54alcfhHd7InUW5BifAdKQEK9SLFa/5hlWbvuhMPlobF0WAx7Okq988Jxg==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-KOZqa3QNr3f0pMnufzL7K+nweFFCCBs6LCXZzXDrVGTyssjLeudn5ySktZYv1XiSqobyHRYYK0c6QsOxJEhXKA==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-TkMUY+A2p2EYVY3GCTItYGvqT6LiLzHBnqsU1rJbrpXUijFfM6zvUx0R4civofVwFCmJZcKqOVwwWAjplKkhxA==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-nwous24r31M+WyDEHV+qckXkepvihxhnyIaod2MG7eCE6G0Zm/HUF6jgN8GXgf4U7AU6SLseKdanY195cwvU6w==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
+ '@inquirer/[email protected]':
+ resolution: {integrity: sha512-lg9Whz8onIHRthWaN1Q9EGLa/0LFJjyM8mEUbL1eTi6yMGvBf8gvyDLtxSXztQsxMvhxxNpJYrwa1YHdq+w4Jw==}
+ engines: {node: '>=18'}
+ peerDependencies:
+ '@types/node': '>=18'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+
'@ioredis/[email protected]':
resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
@@ -1404,6 +1583,342 @@ packages:
resolution: {integrity: sha512-w3btGjeTsYG32hPfbDfziYQRfqGWML3AK88ZE6sOu6qZHLbx7WWf1X/TsTogSxJ2BhnP4UoVQ4F935/G7ZWztA==}
engines: {node: '>=18.0'}
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-Wn6ZPw27qJiEWglGjkaAa70AHuLtyPya6FvjINYJ5U20uvbRhoB0Ta2+bFTAFfUb9R+wvuFvog9JQdy65OmFAQ==}
+ engines: {node: '>= 16'}
+ hasBin: true
+ peerDependencies:
+ '@emnapi/runtime': ^1.1.0
+ emnapi: ^1.1.0
+ peerDependenciesMeta:
+ '@emnapi/runtime':
+ optional: true
+ emnapi:
+ optional: true
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-ENPfLe4937bsKVTDA6zdABx4pq9w0tHqRrJHyaGxgaPq03a2Bd1unD5XSKjXJjebsABJ+MjAv1A2OvCgK9yehg==}
+ peerDependencies:
+ '@napi-rs/cross-toolchain-arm64-target-aarch64': ^1.0.3
+ '@napi-rs/cross-toolchain-arm64-target-armv7': ^1.0.3
+ '@napi-rs/cross-toolchain-arm64-target-ppc64le': ^1.0.3
+ '@napi-rs/cross-toolchain-arm64-target-s390x': ^1.0.3
+ '@napi-rs/cross-toolchain-arm64-target-x86_64': ^1.0.3
+ '@napi-rs/cross-toolchain-x64-target-aarch64': ^1.0.3
+ '@napi-rs/cross-toolchain-x64-target-armv7': ^1.0.3
+ '@napi-rs/cross-toolchain-x64-target-ppc64le': ^1.0.3
+ '@napi-rs/cross-toolchain-x64-target-s390x': ^1.0.3
+ '@napi-rs/cross-toolchain-x64-target-x86_64': ^1.0.3
+ peerDependenciesMeta:
+ '@napi-rs/cross-toolchain-arm64-target-aarch64':
+ optional: true
+ '@napi-rs/cross-toolchain-arm64-target-armv7':
+ optional: true
+ '@napi-rs/cross-toolchain-arm64-target-ppc64le':
+ optional: true
+ '@napi-rs/cross-toolchain-arm64-target-s390x':
+ optional: true
+ '@napi-rs/cross-toolchain-arm64-target-x86_64':
+ optional: true
+ '@napi-rs/cross-toolchain-x64-target-aarch64':
+ optional: true
+ '@napi-rs/cross-toolchain-x64-target-armv7':
+ optional: true
+ '@napi-rs/cross-toolchain-x64-target-ppc64le':
+ optional: true
+ '@napi-rs/cross-toolchain-x64-target-s390x':
+ optional: true
+ '@napi-rs/cross-toolchain-x64-target-x86_64':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-Up4gpyw2SacmyKWWEib06GhiDdF+H+CCU0LAV8pnM4aJIDqKKd5LHSlBht83Jut6frkB0vwEPmAkv4NjQ5u//Q==}
+ engines: {node: '>= 10'}
+ cpu: [arm]
+ os: [android]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-uwa8sLlWEzkAM0MWyoZJg0JTD3BkPknvejAFG2acUA1raXM8jLrqujWCdOStisXhqQjZ2nDMp3FV6cs//zjfuQ==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [android]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-0Y0TQLQ2xAjVabrMDem1NhIssOZzF/y/dqetc6OT8mD3xMTDtF8u5BqZoX3MyPc9FzpsZw4ksol+w7DsxHrpMA==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-vR2IUyJY3En+V1wJkwmbGWcYiT8pHloTAWdW4pG24+51GIq+intst6Uf6D/r46citObGZrlX0QvMarOkQeHWpw==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [darwin]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-XpnYQC5SVovO35tF0xGkbHYjsS6kqyNCjuaLQ2dbEblFRr5cAZVvsJ/9h7zj/5FluJPJRDojVNxGyRhTp4z2lw==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [freebsd]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-ic1ZZMoRfRMwtSwxkyw4zIlbDZGC6davC9r+2oX6x9QiF247BRqqT94qGeL5ZP4Vtz0Hyy7TEViWhx5j6Bpzvw==}
+ engines: {node: '>= 10'}
+ cpu: [arm]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-asEp7FPd7C1Yi6DQb45a3KPHKOFBSfGuJWXcAd4/bL2Fjetb2n/KK2z14yfW8YC/Fv6x3rBM0VAZKmJuz4tysg==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-yWjcPDgJ2nIL3KNvi4536dlT/CcCWO0DUyEOlBs/SacG7BeD6IjGh6yYzd3/X1Y3JItCbZoDoLUH8iB1lTXo3w==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-0XRhKuIU/9ZjT4WDIG/qnX7Xz7mSQHYZo9Gb3MP2gcvBgr6BA4zywQ9k3gmQaPn9ECE+CZg2V7DV7kT+x2pUMQ==}
+ engines: {node: '>= 10'}
+ cpu: [ppc64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-QrqDIPEUUB23GCpyQj/QFyMlr8SGxxyExeZz9OWFnHfb70kXdTLWrHS/hEI1Ru+lSbQ/6xRqeoGyQ4Aqdg+/RA==}
+ engines: {node: '>= 10'}
+ cpu: [riscv64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-k8RVM5aMhW86E9H0QXdquwojew4H3SwPxbRVbl49/COJQWCUjGi79X6mYruMnMPEznZinUiT1jgKbFo2A00NdA==}
+ engines: {node: '>= 10'}
+ cpu: [s390x]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-6rMtBgnIq2Wcl1rQdZsnM+rtCcVCbws1nF8S2NzaUsVaZv8bjrPiAa0lwg4Eqnn1d9lgwqT+cZgm5m+//K08Kw==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-eiadGBKi7Vd0bCArBUOO/qqRYPHt/VQVvGyYvDFt6C2ZSIjlD+HuOl+2oS1sjf4CFjK4eDIog6EdXnL0NE6iyQ==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-+VyHHlr68dvey6fXc2hehw9gHVFIW3TtGF1XkcbAu65qVXsA9D/T+uuoRVqhE+JCyFHFrO0ixRbZDRK1XJt1sA==}
+ engines: {node: '>=14.0.0'}
+ cpu: [wasm32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-eewnqvIyyhHi3KaZtBOJXohLvwwN27gfS2G/YDWdfHlbz1jrmfeHAmzMsP5qv8vGB+T80TMHNkro4kYjeh6Deg==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-OeacFVRCJOKNU/a0ephUfYZ2Yt+NvaHze/4TgOwJ0J0P4P7X1mHzN+ig9Iyd74aQDXYqc7kaCXA2dpAOcH87Cg==}
+ engines: {node: '>= 10'}
+ cpu: [ia32]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-T4I1SamdSmtyZgDXGAGP+y5LEK5vxHUFwe8mz6D4R7Sa5/WCxTcCIgPJ9BD7RkpO17lzhlaM2vmVvMy96Lvk9Q==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-zS5LuN1OBPAyZpda2ZZgYOEDC+xecUdAGnrvbYzjnLXkrq/OBC3B9qcRvlxbDR3k5H/gVfvef1/jyUqPknqjbg==}
+ engines: {node: '>= 10'}
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-h2Ryndraj/YiKgMV/r5by1cDusluYIRT0CaE0/PekQ4u+Wpy2iUVqvzVU98ZPnhXaNeYxEvVJHNGafpOfaD0TA==}
+ engines: {node: '>= 10'}
+ cpu: [arm]
+ os: [android]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-DJFyQHr1ZxNZorm/gzc1qBNLF/FcKzcH0V0Vwan5P+o0aE2keQIGEjJ09FudkF9v6uOuJjHCVDdK6S6uHtShAw==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [android]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-Zz2sXRzjIX4e532zD6xm2SjXEym6MkvfCvL2RMpG2+UwNVDVscHNcz3d47Pf3sysP2e2af7fBB3TIoK2f6trPw==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-EI+CptIMNweT0ms9S3mkP/q+J6FNZ1Q6pvpJOEcWglRfyfQpLqjlC0O+dptruTPE8VamKYuqdjxfqD8hifZDOA==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [darwin]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-J0PIqX+pl6lBIAckL/c87gpodLbjZB1OtIK+RDscKC9NLdpVv6VGOxzUV/fYev/hctcE8EfkLbgFOfpmVQPg2g==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [freebsd]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-SLgIQo3f3EjkZ82ZwvrEgFvMdDAhsxCYjyoSuWfHCz0U16qx3SuGCp8+FYOPYCECHN3ZlGjXnoAIt9ERd0dEUg==}
+ engines: {node: '>= 10'}
+ cpu: [arm]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-d014cdle52EGaH6GpYTQOP9Py7glMO1zz/+ynJPjjzYFSxvdYx0byrjumZk2UQdIyGZiJO2MEFpCkEEKFSgPYA==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-L/y1/26q9L/uBqiW/JdOb/Dc94egFvNALUZV2WCGKQXc6UByPBMgdiEyW2dtoYxYYYYc+AKD+jr+wQPcvX2vrQ==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-EPE1K/80RQvPbLRJDJs1QmCIcH+7WRi0F73+oTe1582y9RtfGRuzAkzeBuAGRXAQEjRQw/RjtNqr6UTJ+8UuWQ==}
+ engines: {node: '>= 10'}
+ cpu: [ppc64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-B2jhWiB1ffw1nQBqLUP1h4+J1ovAxBOoe5N2IqDMOc63fsPZKNqF1PvO/dIem8z7LL4U4bsfmhy3gBfu547oNQ==}
+ engines: {node: '>= 10'}
+ cpu: [s390x]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-tbZDHnb9617lTnsDMGo/eAMZxnsQFnaRe+MszRqHguKfMwkisc9CCJnks/r1o84u5fECI+J/HOrKXgczq/3Oww==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-dV6cODlzbO8u6Anmv2N/ilQHq/AWz0xyltuXoLU3yUyXbZcnWYZuB2rL8OBGPmqNcD+x9NdScBNXh7vWN0naSQ==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-jIa9nb2HzOrfH0F8QQ9g3WE4aMH5vSI5/1NYVNm9ysCmNjCCtMXCAhlI3WKCdm/DwHf0zLqdrrtDFXODcNaqMw==}
+ engines: {node: '>=14.0.0'}
+ cpu: [wasm32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-vfpG71OB0ijtjemp3WTdmBKJm9R70KM8vsSExMsIQtV0lVzP07oM1CW6JbNRPXNLhRoue9ofYLiUDk8bE0Hckg==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-hGPyPW60YSpOSgzfy68DLBHgi6HxkAM+L59ZZZPMQ0TOXjQg+p2EW87+TjZfJOkSpbYiEkULwa/f4a2hcVjsqQ==}
+ engines: {node: '>= 10'}
+ cpu: [ia32]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-L6Ed1DxXK9YSCMyvpR8MiNAyKNkQLjsHsHK9E0qnHa8NzLFqzDKhvs5LfnWxM2kJ+F7m/e5n9zPm24kHb3LsVw==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-7cmzIu+Vbupriudo7UudoMRH2OA3cTw67vva8MxeoAe5S7vPFI7z0vp0pMXiA25S8IUJefImQ90FeJjl8fjEaQ==}
+ engines: {node: '>= 10'}
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-rZxtMsLwjdXkMUGC3WwsPwLNVqVqnTJT6MNIB6e+5fhMcSCPP0AOsNWuMQ5mdCq6HNjs/ZeWAEchpqeprqBD2Q==}
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-lr07E/l571Gft5v4aA1dI8koJEmF1F0UigBbsqg9OWNzg80H3lDPO+auv85y3T/NHE3GirDk7x/D3sLO57vayw==}
+ engines: {node: '>= 10'}
+ cpu: [arm]
+ os: [android]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-WDR7S+aRLV6LtBJAg5fmjKkTZIdrEnnQxgdsb7Cf8pYiMWBHLU+LC49OUVppQ2YSPY0+GeYm9yuZWW3kLjJ7Bg==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [android]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-qWTI+EEkiN0oIn/N2gQo7+TVYil+AJ20jjuzD2vATS6uIjVz+Updeqmszi7zq7rdFTLp6Ea3/z4kDKIfZwmR9g==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-bA6hubqtHROR5UI3tToAF/c6TDmaAgF0SWgo4rADHtQ4wdn0JeogvOk50gs2TYVhKPE2ZD2+qqt7oBKB+sxW3A==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [darwin]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-90+KLBkD9hZEjPQW1MDfwSt5J1L46EUKacpCZWyRuL6iIEO5CgWU0V/JnEgFsDOGyyYtiTvHc5bUdUTWd4I9Vg==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [freebsd]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-rG0QlS65x9K/u3HrKafDf8cFKj5wV2JHGfl8abWgKew0GVPyp6vfsDweOwHbWAjcHtp2LHi6JHoW80/MTHm52Q==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-jAasbIvjZXCgX0TCuEFQr+4D6Lla/3AAVx2LmDuMjgG4xoIXzjKWl7c4chuaD+TI+prWT0X6LJcdzFT+ROKGHQ==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-Plgk5rPqqK2nocBGajkMVbGm010Z7dnUgq0wtnYRZbzWWxwWcXfZMPa8EYxrK4eE8SzpI7VlZP1tdVsdjgGwMw==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-GW7AzGuWxtQkyHknHWYFdR0CHmW6is8rG2Rf4V6GNmMpmwtXt/ItWYWtBe4zqJWycMNazpfZKSw/BpT7/MVCXQ==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [linux]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-/nQVSTrqSsn7YdAc2R7Ips/tnw5SPUcl3D7QrXCNGPqjbatIspnaexvaOYNyKMU6xPu+pc0BTnKVmqhlJJCPLA==}
+ engines: {node: '>=14.0.0'}
+ cpu: [wasm32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-PFi7oJIBu5w7Qzh3dwFea3sHRO3pojMsaEnUIy22QvsW+UJfNQwJCryVrpoUt8m4QyZXI+saEq/0r4GwdoHYFQ==}
+ engines: {node: '>= 10'}
+ cpu: [arm64]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-gXkuYzxQsgkj05Zaq+KQTkHIN83dFAwMcTKa2aQcpYPRImFm2AQzEyLtpXmyCWzJ0F9ZYAOmbSyrNew8/us6bw==}
+ engines: {node: '>= 10'}
+ cpu: [ia32]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-rEAf05nol3e3eei2sRButmgXP+6ATgm0/38MKhz9Isne82T4rPIMYsCIFj0kOisaGeVwoi2fnm7O9oWp5YVnYQ==}
+ engines: {node: '>= 10'}
+ cpu: [x64]
+ os: [win32]
+
+ '@napi-rs/[email protected]':
+ resolution: {integrity: sha512-enkZYyuCdo+9jneCPE/0fjIta4wWnvVN9hBo2HuiMpRF0q3lzv1J6b/cl7i0mxZUKhBrV3aCKDBQnCOhwKbPmQ==}
+ engines: {node: '>= 10'}
+
resolution: {integrity: sha512-rONPWMC7PeExE077uLE4oqWrZ1IvAfz3oH9LibVAcVCopJiA9R62uavnbEzdkVmJYI6M6Zgkbeb07+tWjlq2XA==}
engines: {node: ^14.21.3 || >=16}
@@ -1447,6 +1962,58 @@ packages:
resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}
engines: {node: ^14.21.3 || >=16}
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==}
+ engines: {node: '>= 20'}
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-oNXsh2ywth5aowwIa7RKtawnkdH6LgU1ztfP9AIUCQCvzysB+WeU8o2kyyosDPwBZutPpjZDKPQGIzzrfTWweQ==}
+ engines: {node: '>= 20'}
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-hoYicJZaqISMAI3JfaDr1qMNi48OctWuOih1m80bkYow/ayPw6Jj52tqWJ6GEoFTk1gBqfanSoI1iY99Z5+ekQ==}
+ engines: {node: '>= 20'}
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-j1nQNU1ZxNFx2ZtKmL4sMrs4egy5h65OMDmSbVyuCzjOcwsHq6EaYjOTGXPQxgfiN8dJ4CriYHk6zF050WEULg==}
+ engines: {node: '>= 20'}
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA==}
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-q9iQGlZlxAVNRN2jDNskJW/Cafy7/XE52wjZ5TTvyhyOD904Cvx//DNyoO3J/MXJ0ve3rPoNWKEg5iZrisQSuw==}
+ engines: {node: '>= 20'}
+ peerDependencies:
+ '@octokit/core': '>=6'
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==}
+ engines: {node: '>= 20'}
+ peerDependencies:
+ '@octokit/core': '>=6'
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-kJVUQk6/dx/gRNLWUnAWKFs1kVPn5O5CYZyssyEoNYaFedqZxsfYs7DwI3d67hGz4qOwaJ1dpm07hOAD1BXx6g==}
+ engines: {node: '>= 20'}
+ peerDependencies:
+ '@octokit/core': '>=6'
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-KRA7VTGdVyJlh0cP5Tf94hTiYVVqmt2f3I6mnimmaVz4UG3gQV/k4mDJlJv3X67iX6rmN7gSHCF8ssqeMnmhZg==}
+ engines: {node: '>= 20'}
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-V6jhKokg35vk098iBqp2FBKunk3kMTXlmq+PtbV9Gl3TfskWlebSofU9uunVKhUN7xl+0+i5vt0TGTG8/p/7HA==}
+ engines: {node: '>= 20'}
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-z6tmTu9BTnw51jYGulxrlernpsQYXpui1RK21vmXn8yF5bp6iX16yfTtJYGK5Mh1qDkvDOmp2n8sRMcQmR8jiA==}
+ engines: {node: '>= 20'}
+
+ '@octokit/[email protected]':
+ resolution: {integrity: sha512-1y6DgTy8Jomcpu33N+p5w58l6xyt55Ar2I91RPiIA0xCJBXyUAhXCcmZaDWSANiha7R9a6qJJ2CRomGPZ6f46g==}
+
'@one-ini/[email protected]':
resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==}
@@ -2091,6 +2658,134 @@ packages:
peerDependencies:
'@opentelemetry/api': ^1.1.0
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-Ykkz+xYJ1Hd+vjVSXljyIFRW/ZRMp2tQfPy9T2jj9F2pm5CAchqJHSXEbB3FbhdUXusUqhWLmu70JdVhS4SBcQ==}
+ cpu: [arm]
+ os: [android]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-csPEZPGTlPtkoMnDeG+hrYX2AJIh8dPKW7TW5E0MdNMIkJ8aFIymBH2QF+xSMsFl9Gpbl5ZxYLkRIrth+Mhzhg==}
+ cpu: [arm64]
+ os: [android]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-wlURG+gBge3ovdBtMQ/KvyFsLVXnQ4h5vlIxBYMCTkwCzwvfXbcSOLLZMxypOeXPlH/cdVqOigvh9XBbc3KAfg==}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-tFlUz54cemNp6N7+suP+uwCXErzol7ibMslxOHbXtGWwRVckV3Bb1rQyCajFSQYCr/du1ZdDCVsTfuPmXQlejg==}
+ cpu: [x64]
+ os: [darwin]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-yqikvHq0VbnPasRwOkDiMe5dhLZexvZSBzNroBVGbMWNaHBvkP+GP4EFZ3Y5pcNPC3og3xP4J6+N8bbbEZk3vg==}
+ cpu: [x64]
+ os: [freebsd]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-Qbv937NEH4gNNC1W3Lx6G4Mh5mETMYSf2jbTxCzcfng74BVorpQd0pKB72bvEA0RQtIkXoEfCW24VeyR3yA84Q==}
+ cpu: [arm]
+ os: [linux]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-w+6SmMWEaVap5RC9axCvg1ZOFdQsNLlLYrYrKX5siZvcVs6jPNyg3O9rbO85tBFPvuAvSLr66mBJwW70MQJv0w==}
+ cpu: [arm64]
+ os: [linux]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-AZeVCYaECJbQaj6bxCZK77ApbAQUKNOq4wWoRIkVZnn9/ay5wqHLmkvWdOku0aw2WkGiteLp5nCXSDZjw1g0/A==}
+ cpu: [arm64]
+ os: [linux]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-sz8r3BPLWZC5vZ/5UCb2MhcEFT8bRIeo3IFnksh6K9umAyjgByxnBZP/EyL4twcjQqbrDC64Ox0Pm5vtTIiFWw==}
+ cpu: [ppc64]
+ os: [linux]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-YLpDVEcsvj03z6SqqcxA1J+ocTR+pP6dv0oIrGXaATwQtx0C+nt6WrVESaXjJpQDAc/5I+LkVDlimeAJb4iLNQ==}
+ cpu: [s390x]
+ os: [linux]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-FCBaQq4Y08AOLvl83AVzGeRAp/7RI49sDb44sE3/XzxdAVHYyhp0wA84h/NBi8Cj8XhSJdr6KosskHl1cVB0Qg==}
+ cpu: [x64]
+ os: [linux]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-QKvj6d7VGKK3udW8TXfB0NYNjCloKxZznl58eaoFMoO+bOwuOWJX/8tN37FjkghH4Sb38vRK24+BdqebAHjBdA==}
+ cpu: [x64]
+ os: [linux]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-k8wqQZ04J4l44v6iB7VFVNc+tz0BBJV2nxSwhLTHpciAFZnOG4BJUqw+2OsObUreR3z5zarebkVFyEJ3QaUH5g==}
+ cpu: [arm64]
+ os: [openharmony]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-O8Bj67iC0lm0xSlpmHseU+tQytrqdhDFUc0MzYmPSvs0i60u3CNGiw/BWuO6xDpBfhp/IQkWSGBrecZZt1hMFg==}
+ engines: {node: '>=14.0.0'}
+ cpu: [wasm32]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-mGFu9B3E/jBqvy6FzTZHjUZUOOst1dnN/LIX1Bi8wbSrB+1TdOBykOcNJ3F+BppdaTCWqKb6ieLU6BvvnxboaA==}
+ cpu: [arm64]
+ os: [win32]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-JEk4P/c/x6T4MHPBtty7IdCx/hxqhdhLrrJ0pIQ8Ue1Kkx74Aq6NDPg6TWtHAxmn+4cw6c1TLzGPXW+1dwbFcA==}
+ cpu: [ia32]
+ os: [win32]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-6G0YmfxjD3Ec8G619VXfJ3luryJb1Fq649H8ZcfLO8810WLwtzZiAXZCsCLs8lPsLTEpTeFR+AWuc8v4IRHekg==}
+ cpu: [x64]
+ os: [win32]
+
+ '@oxc-node/[email protected]':
+ resolution: {integrity: sha512-2lbEquSd7qU5SZwbu2ngxs/vaa5sgRB5FE6TSPIPp6wfy7+11M0OrzD3g9TxH5CcsawecF2pC8nNpRVjBgBhOg==}
+
+ '@oxlint/[email protected]':
+ resolution: {integrity: sha512-rcTw0QWeOc6IeVp+Up7WtcwdS9l4j7TOq4tihF0Ud/fl+VUVdvDCPuZ9QTnLXJhwMXiyQRWdxRyI6XBwf80ncQ==}
+ cpu: [arm64]
+ os: [darwin]
+
+ '@oxlint/[email protected]':
+ resolution: {integrity: sha512-TWFSEmyl2/DN4HoXNwQl0y/y3EXFJDctfv5MiDtVOV1GJKX80cGSIxMxXb08Q3CCWqteqEijmfSMo5TG8X1H/A==}
+ cpu: [x64]
+ os: [darwin]
+
+ '@oxlint/[email protected]':
+ resolution: {integrity: sha512-N1FqdKfwhVWPpMElv8qlGqdEefTbDYaRVhdGWOjs/2f7FESa5vX0cvA7ToqzkoXyXZI5DqByWiPML33njK30Kg==}
+ cpu: [arm64]
+ os: [linux]
+
+ '@oxlint/[email protected]':
+ resolution: {integrity: sha512-v/BPuiateLBb7Gz1STb69EWjkgKdlPQ1NM56z+QQur21ly2hiMkBX2n0zEhqfu9PQVRUizu6AlsYuzcPY/zsIQ==}
+ cpu: [arm64]
+ os: [linux]
+
+ '@oxlint/[email protected]':
+ resolution: {integrity: sha512-gUTp8KIrSYt97dn+tRRC3LKnH4xlHKCwrPwiDcGmLbCxojuN9/H5mnIhPKEfwNuZNdoKGS/ABuq3neVyvRCRtQ==}
+ cpu: [x64]
+ os: [linux]
+
+ '@oxlint/[email protected]':
+ resolution: {integrity: sha512-DpN6cW2HPjYXeENG0JBbmubO8LtfKt6qJqEMBw9gUevbyBaX+k+Jn7sYgh6S23wGOkzmTNphBsf/7ulj4nIVYA==}
+ cpu: [x64]
+ os: [linux]
+
+ '@oxlint/[email protected]':
+ resolution: {integrity: sha512-oXxJksnUTUMgJ0NvjKS1mrCXAy1ttPgIVacRSlxQ+1XHy+aJDMM7I8fsCtoKoEcTIpPaD98eqUqlLYs0H2MGjA==}
+ cpu: [arm64]
+ os: [win32]
+
+ '@oxlint/[email protected]':
+ resolution: {integrity: sha512-iRYy2rhTQKFztyx0jtNMRBnFpzsRwFdjWQ7sKKzJpmbijA3Tw3DCqlGT7QRgoVRF0+X/ccNGvvsrgMohPVfLeQ==}
+ cpu: [x64]
+ os: [win32]
+
'@paralleldrive/[email protected]':
resolution: {integrity: sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==}
@@ -2638,6 +3333,10 @@ packages:
peerDependencies:
react: ^18 || ^19
+ '@taplo/[email protected]':
+ resolution: {integrity: sha512-Ck3zFhQhIhi02Hl6T4ZmJsXdnJE+wXcJz5f8klxd4keRYgenMnip3JDPMGDRLbnC/2iGd8P0sBIQqI3KxfVjBg==}
+ hasBin: true
+
'@tootallnate/[email protected]':
resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==}
engines: {node: '>= 10'}
@@ -2657,6 +3356,9 @@ packages:
'@tsconfig/[email protected]':
resolution: {integrity: sha512-0IKu9GHYF1NGTJiYgfWwqnOQSlnE9V9R7YohHNNf0/fj/SyOZWzdd06JFr0fLpg1Mqw0kGbYg8w5xdkSqLKM9g==}
+ '@tybys/[email protected]':
+ resolution: {integrity: sha512-VyyPYFlOMNylG45GoAe0xDoLwWuowvf92F9kySqzYh8vmYm7D2u4iUJKa1tOUpS70Ku13ASrOkS4ScXFsTaCNQ==}
+
resolution: {integrity: sha512-AX+AbjH/rH5ezX1fbK8onC/a+HyQHo7QGmvoxAE42n22OsciAxvZoZNEr22tbXs8WfP1nIsBjKDpgPm3HjOZbA==}
@@ -3161,6 +3863,9 @@ packages:
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
+ resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==}
+
resolution: {integrity: sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==}
@@ -3298,10 +4003,6 @@ packages:
resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
engines: {node: '>=10'}
- resolution: {integrity: sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==}
- engines: {node: ^12.17.0 || ^14.13 || >=16.0.0}
-
resolution: {integrity: sha512-46QrSQFyVSEyYAgQ22hQ+zDa60YHA4fBstHmtSApj1Y5vKtG27fWowW03jCk5KcbXEWPZUIR894aARCA/G1kfQ==}
engines: {node: ^12.17.0 || ^14.13 || >=16.0.0}
@@ -3310,6 +4011,9 @@ packages:
resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==}
engines: {node: '>=10'}
+ resolution: {integrity: sha512-bNFETTG/pM5ryzQ9Ad0lJOTa6HWD/YsScAR3EnCPZRPlQh77JocYktSHOUHelyhm8IARL+o4c4F1bP5KVOjiRA==}
+
resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==}
@@ -3343,6 +4047,15 @@ packages:
resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==}
+ engines: {node: '>= 12'}
+
+ resolution: {integrity: sha512-CXkMQxU6s9GklO/1f714dkKBMu1lopS1WFF0B8o4AxPykR1hpozxSiUZ5ZUeBjfPgCWqbcNOtZVFhB8Lkfp1+Q==}
+ peerDependencies:
+ typanion: '*'
+
resolution: {integrity: sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==}
@@ -3793,6 +4506,9 @@ packages:
resolution: {integrity: sha512-X13Q/ZSc+vsO1q600bvNK4bxgXMkHcf//RxCmYDaRY5DAcT+eoXjY5hoAPGMdRnWQjvyLEcyauG3b6hz76LNqg==}
+ resolution: {integrity: sha512-E0iGnTtbDhkeczB0T+mxmoVlT4YNweEKBLq7oaU4p11mecdsZpNWOglI4895Vh4usbQ+LsJiuLuI2L0Vdmfm2w==}
+
resolution: {integrity: sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==}
@@ -3905,6 +4621,9 @@ packages:
resolution: {integrity: sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==}
engines: {node: '> 0.1.90'}
+ resolution: {integrity: sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg==}
+
resolution: {integrity: sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==}
@@ -3966,6 +4685,10 @@ packages:
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-YyZM99iHrqLKjmt4LJDj58KI+fYyufRLBSYcqycxf//KpBk9FoewoGX0450m9nB44qrZnovzC2oeP5hUibxc/g==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==}
@@ -4361,6 +5084,10 @@ packages:
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
+ resolution: {integrity: sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==}
+ engines: {node: '>=16'}
+
resolution: {integrity: sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==}
@@ -4601,6 +5328,10 @@ packages:
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
+ resolution: {integrity: sha512-lR4MXjGNgkJc7tkQ97kb2nuEMnNCyU//XYVH0MKTGcXEiSudQ5MKGKen3C5QubYy0vmq+JGitUg92uuywGEwIA==}
+ engines: {node: ^18.17.0 || >=20.5.0}
+
resolution: {integrity: sha512-NEdLrtrq1jUZyfjkr9OCz9EzCNhnRyWtt1PAnvnhwy6e8XETS0Dtc+ZNCO2gvuAoKsIn2+vCSowXTYE4CkgnAQ==}
engines: {node: '>=10.0.0'}
@@ -4718,6 +5449,10 @@ packages:
resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==}
+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+
resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==}
@@ -4825,6 +5560,10 @@ packages:
resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
engines: {node: '>= 0.6'}
+ resolution: {integrity: sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==}
+ engines: {node: '>= 0.10.0'}
+
resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==}
@@ -4941,6 +5680,10 @@ packages:
resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==}
hasBin: true
+ resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==}
+ engines: {node: ^18.17.0 || >=20.5.0}
+
resolution: {integrity: sha512-21t+ozMQDAL/UGgQVBbZ/xXvNO10++ZPuTmKRO8k9V3AClVRht49ahtDjfY8l1q6nSHOrE5ASfthzH3ol6R/hg==}
engines: {node: '>=20.17'}
@@ -5023,6 +5766,11 @@ packages:
resolution: {integrity: sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==}
engines: {node: ^18.17.0 || >=20.5.0}
+ resolution: {integrity: sha512-wdbB5My48XKp2ZfJUlhnLVihzeuA1hgBnqB2J9ahV77wLS+/YAJAlN8I+X3DIFIPZ3m5L7nplmlbhNiFDmXRDA==}
+ engines: {node: ^20.5.0 || >=22.0.0, npm: '>= 10'}
+ hasBin: true
+
resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==}
engines: {node: '>=8'}
@@ -5119,6 +5867,16 @@ packages:
typescript:
optional: true
+ resolution: {integrity: sha512-oo0nq3zF9hmgATGc9esoMahLuEESOodUxEDeHDA2K7tbYcSfcmReE9G2QNppnq9rOSQHLTwlMtzGAjjttYaufQ==}
+ engines: {node: '>=8.*'}
+ hasBin: true
+ peerDependencies:
+ oxlint-tsgolint: '>=0.1.5'
+ peerDependenciesMeta:
+ oxlint-tsgolint:
+ optional: true
+
resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==}
engines: {node: '>=6'}
@@ -5127,10 +5885,18 @@ packages:
resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==}
engines: {node: '>=10'}
+ resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==}
+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+
resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==}
+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+
resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
engines: {node: '>=6'}
@@ -5171,6 +5937,10 @@ packages:
resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==}
+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+
resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
engines: {node: '>=0.10.0'}
@@ -5226,6 +5996,10 @@ packages:
resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}
engines: {node: '>=8.6'}
+ resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==}
+ engines: {node: '>=12'}
+
resolution: {integrity: sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==}
engines: {node: '>=0.10'}
@@ -5249,8 +6023,8 @@ packages:
resolution: {integrity: sha512-dMACeu63HtRLmCG8VKdy4cShCPKaYDR4youZqoSWLxl5Gu99HUw8bw75thbPv9Nip+H+QYX8o3ZJbTdVZZ2TVg==}
hasBin: true
- resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==}
+ resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==}
engines: {node: '>= 6'}
@@ -5432,6 +6206,10 @@ packages:
resolution: {integrity: sha512-SEbJV7tohp3DAAILbEMPXavBjAnMN0tVnh4+9G8ihV4Pq3HYF9h8QNez9zkJ1ILkv9G2BjdzwctznGZXgu/HGw==}
engines: {node: ^18.17.0 || >=20.5.0}
+ resolution: {integrity: sha512-qpt8EwugBWDw2cgE2W+/3oxC+KTez2uSVR8JU9Q36TXPAGCaozfQUs59v4j4GFpWTaw0i6hAZSvOmu1J0uOEUg==}
+ engines: {node: ^18.17.0 || >=20.5.0}
+
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
@@ -5581,6 +6359,11 @@ packages:
engines: {node: '>=10'}
hasBin: true
+ resolution: {integrity: sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==}
+ engines: {node: '>=10'}
+ hasBin: true
+
resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==}
engines: {node: '>= 0.8.0'}
@@ -5615,6 +6398,10 @@ packages:
resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
engines: {node: '>=8'}
+ resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==}
engines: {node: '>= 4.0.0'}
@@ -5986,6 +6773,9 @@ packages:
resolution: {integrity: sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==}
+ resolution: {integrity: sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==}
+
resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==}
engines: {node: '>=4'}
@@ -6007,6 +6797,11 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
+ resolution: {integrity: sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==}
+ engines: {node: '>=14.17'}
+ hasBin: true
+
resolution: {integrity: sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA==}
@@ -6026,6 +6821,13 @@ packages:
resolution: {integrity: sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw==}
engines: {node: '>=20.18.1'}
+ resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==}
+ engines: {node: '>=18'}
+
+ resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==}
+
resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==}
engines: {node: '>= 4.0.0'}
@@ -6259,6 +7061,11 @@ packages:
engines: {node: '>= 8'}
hasBin: true
+ resolution: {integrity: sha512-JEdGzHwwkrbWoGOlIHqQ5gtprKGOenpDHpxE9zVR1bWbOtYRyPPHMe9FaP6x61CmNaTThSkb0DAJte5jD+DmzQ==}
+ engines: {node: ^18.17.0 || >=20.5.0}
+ hasBin: true
+
resolution: {integrity: sha512-qxSTKswC6llEMZKgCQdaWgDuMJQnhuvF5f2Nk3SNXc4byfQ+voo2mX1Px9dkNOuR8p0KAjfPG29PuYUSIb+vSA==}
engines: {node: '>= 12.0.0'}
@@ -6442,6 +7249,14 @@ packages:
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
engines: {node: '>=10'}
+ resolution: {integrity: sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==}
+ engines: {node: '>=12.20'}
+
+ resolution: {integrity: sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==}
peerDependencies:
@@ -6693,17 +7508,17 @@ snapshots:
'@aws-sdk/types': 3.679.0
'@aws-sdk/util-locate-window': 3.679.0
'@smithy/util-utf8': 2.3.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-crypto/[email protected]':
dependencies:
'@aws-crypto/util': 5.2.0
'@aws-sdk/types': 3.679.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-crypto/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-crypto/[email protected]':
dependencies:
@@ -6715,7 +7530,7 @@ snapshots:
dependencies:
'@aws-sdk/types': 3.679.0
'@smithy/util-utf8': 2.3.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
@@ -6759,7 +7574,7 @@ snapshots:
'@smithy/util-middleware': 3.0.8
'@smithy/util-retry': 3.0.8
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- aws-crt
@@ -6807,7 +7622,7 @@ snapshots:
'@smithy/util-utf8': 3.0.0
'@smithy/util-waiter': 3.1.7
'@types/uuid': 9.0.8
- tslib: 2.6.3
+ tslib: 2.8.1
uuid: 9.0.1
transitivePeerDependencies:
- aws-crt
@@ -6853,7 +7668,7 @@ snapshots:
'@smithy/util-middleware': 3.0.8
'@smithy/util-retry': 3.0.8
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- aws-crt
@@ -6896,7 +7711,7 @@ snapshots:
'@smithy/util-middleware': 3.0.8
'@smithy/util-retry': 3.0.8
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- aws-crt
@@ -6941,7 +7756,7 @@ snapshots:
'@smithy/util-middleware': 3.0.8
'@smithy/util-retry': 3.0.8
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- aws-crt
@@ -6957,7 +7772,7 @@ snapshots:
'@smithy/types': 3.6.0
'@smithy/util-middleware': 3.0.8
fast-xml-parser: 4.4.1
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
@@ -6965,7 +7780,7 @@ snapshots:
'@aws-sdk/types': 3.679.0
'@smithy/property-provider': 3.1.8
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- aws-crt
@@ -6975,7 +7790,7 @@ snapshots:
'@aws-sdk/types': 3.679.0
'@smithy/property-provider': 3.1.8
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
@@ -6988,7 +7803,7 @@ snapshots:
'@smithy/smithy-client': 3.4.2
'@smithy/types': 3.6.0
'@smithy/util-stream': 3.2.1
- tslib: 2.6.3
+ tslib: 2.8.1
dependencies:
@@ -7004,7 +7819,7 @@ snapshots:
'@smithy/property-provider': 3.1.8
'@smithy/shared-ini-file-loader': 3.1.9
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- '@aws-sdk/client-sso-oidc'
- aws-crt
@@ -7022,7 +7837,7 @@ snapshots:
'@smithy/property-provider': 3.1.8
'@smithy/shared-ini-file-loader': 3.1.9
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- '@aws-sdk/client-sso-oidc'
- '@aws-sdk/client-sts'
@@ -7035,7 +7850,7 @@ snapshots:
'@smithy/property-provider': 3.1.8
'@smithy/shared-ini-file-loader': 3.1.9
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
dependencies:
@@ -7046,7 +7861,7 @@ snapshots:
'@smithy/property-provider': 3.1.8
'@smithy/shared-ini-file-loader': 3.1.9
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- '@aws-sdk/client-sso-oidc'
- aws-crt
@@ -7058,7 +7873,7 @@ snapshots:
'@aws-sdk/types': 3.679.0
'@smithy/property-provider': 3.1.8
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
dependencies:
@@ -7078,7 +7893,7 @@ snapshots:
'@smithy/credential-provider-imds': 3.2.5
'@smithy/property-provider': 3.1.8
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
transitivePeerDependencies:
- '@aws-sdk/client-sso-oidc'
- aws-crt
@@ -7088,20 +7903,20 @@ snapshots:
'@aws-sdk/types': 3.679.0
'@smithy/protocol-http': 4.1.5
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
'@aws-sdk/types': 3.679.0
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
'@aws-sdk/types': 3.679.0
'@smithy/protocol-http': 4.1.5
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
@@ -7111,12 +7926,12 @@ snapshots:
'@smithy/core': 2.5.1
'@smithy/protocol-http': 4.1.5
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
'@smithy/protocol-http': 1.2.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
@@ -7125,12 +7940,12 @@ snapshots:
'@smithy/types': 3.6.0
'@smithy/util-config-provider': 3.0.0
'@smithy/util-middleware': 3.0.8
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
'@smithy/signature-v4': 1.1.0
- tslib: 2.6.3
+ tslib: 2.8.1
dependencies:
@@ -7139,30 +7954,30 @@ snapshots:
'@smithy/property-provider': 3.1.8
'@smithy/shared-ini-file-loader': 3.1.9
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
'@aws-sdk/types': 3.679.0
'@smithy/types': 3.6.0
'@smithy/util-endpoints': 2.1.4
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
'@aws-sdk/types': 3.679.0
'@smithy/types': 3.6.0
bowser: 2.11.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
@@ -7170,11 +7985,11 @@ snapshots:
'@aws-sdk/types': 3.679.0
'@smithy/node-config-provider': 3.1.9
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@aws-sdk/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
dependencies:
@@ -7548,6 +8363,19 @@ snapshots:
dependencies:
'@noble/ciphers': 1.3.0
+ '@emnapi/[email protected]':
+ dependencies:
+ '@emnapi/wasi-threads': 1.1.0
+ tslib: 2.8.1
+
+ '@emnapi/[email protected]':
+ dependencies:
+ tslib: 2.8.1
+
+ '@emnapi/[email protected]':
+ dependencies:
+ tslib: 2.8.1
+
'@esbuild/[email protected]':
optional: true
@@ -7688,6 +8516,129 @@ snapshots:
protobufjs: 7.5.3
yargs: 17.7.2
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/figures': 1.0.13
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ ansi-escapes: 4.3.2
+ yoctocolors-cjs: 2.1.3
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/figures': 1.0.13
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ ansi-escapes: 4.3.2
+ cli-width: 4.1.0
+ mute-stream: 2.0.0
+ signal-exit: 4.1.0
+ wrap-ansi: 6.2.0
+ yoctocolors-cjs: 2.1.3
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/external-editor': 1.0.1(@types/[email protected])
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ yoctocolors-cjs: 2.1.3
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ chardet: 2.1.0
+ iconv-lite: 0.6.3
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected]': {}
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ ansi-escapes: 4.3.2
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/checkbox': 4.2.2(@types/[email protected])
+ '@inquirer/confirm': 5.1.16(@types/[email protected])
+ '@inquirer/editor': 4.2.18(@types/[email protected])
+ '@inquirer/expand': 4.0.18(@types/[email protected])
+ '@inquirer/input': 4.2.2(@types/[email protected])
+ '@inquirer/number': 3.0.18(@types/[email protected])
+ '@inquirer/password': 4.0.18(@types/[email protected])
+ '@inquirer/rawlist': 4.1.6(@types/[email protected])
+ '@inquirer/search': 3.1.1(@types/[email protected])
+ '@inquirer/select': 4.3.2(@types/[email protected])
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ yoctocolors-cjs: 2.1.3
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/figures': 1.0.13
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ yoctocolors-cjs: 2.1.3
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ dependencies:
+ '@inquirer/core': 10.2.0(@types/[email protected])
+ '@inquirer/figures': 1.0.13
+ '@inquirer/type': 3.0.8(@types/[email protected])
+ ansi-escapes: 4.3.2
+ yoctocolors-cjs: 2.1.3
+ optionalDependencies:
+ '@types/node': 20.14.1
+
+ '@inquirer/[email protected](@types/[email protected])':
+ optionalDependencies:
+ '@types/node': 20.14.1
+
'@ioredis/[email protected]': {}
'@isaacs/[email protected]':
@@ -7860,7 +8811,7 @@ snapshots:
jest-regex-util: 29.6.3
jest-util: 29.7.0
micromatch: 4.0.8
- pirates: 4.0.6
+ pirates: 4.0.7
slash: 3.0.0
write-file-atomic: 4.0.2
transitivePeerDependencies:
@@ -8030,7 +8981,7 @@ snapshots:
'@ethereumjs/tx': 4.2.0
'@types/debug': 4.1.12
debug: 4.4.1
- semver: 7.6.2
+ semver: 7.7.2
superstruct: 1.0.4
transitivePeerDependencies:
- supports-color
@@ -8044,7 +8995,7 @@ snapshots:
'@types/debug': 4.1.12
debug: 4.4.1
pony-cause: 2.1.11
- semver: 7.6.2
+ semver: 7.7.2
uuid: 9.0.1
transitivePeerDependencies:
- supports-color
@@ -8058,36 +9009,280 @@ snapshots:
'@types/debug': 4.1.12
debug: 4.4.1
pony-cause: 2.1.11
- semver: 7.6.2
+ semver: 7.7.2
uuid: 9.0.1
transitivePeerDependencies:
- supports-color
- '@mixmark-io/[email protected]': {}
+ '@mixmark-io/[email protected]': {}
+
+ '@msgpackr-extract/[email protected]':
+ optional: true
+
+ '@msgpackr-extract/[email protected]':
+ optional: true
+
+ '@msgpackr-extract/[email protected]':
+ optional: true
+
+ '@msgpackr-extract/[email protected]':
+ optional: true
+
+ '@msgpackr-extract/[email protected]':
+ optional: true
+
+ '@msgpackr-extract/[email protected]':
+ optional: true
+
+ '@nangohq/[email protected]':
+ dependencies:
+ axios: 1.11.0
+ transitivePeerDependencies:
+ - debug
+
+ dependencies:
+ '@inquirer/prompts': 7.8.4(@types/[email protected])
+ '@napi-rs/cross-toolchain': 1.0.3
+ '@napi-rs/wasm-tools': 1.0.1
+ '@octokit/rest': 22.0.0
+ clipanion: 4.0.0-rc.4([email protected])
+ colorette: 2.0.20
+ debug: 4.4.1
+ es-toolkit: 1.39.10
+ find-up: 7.0.0
+ js-yaml: 4.1.0
+ semver: 7.7.2
+ typanion: 3.14.0
+ optionalDependencies:
+ '@emnapi/runtime': 1.5.0
+ transitivePeerDependencies:
+ - '@napi-rs/cross-toolchain-arm64-target-aarch64'
+ - '@napi-rs/cross-toolchain-arm64-target-armv7'
+ - '@napi-rs/cross-toolchain-arm64-target-ppc64le'
+ - '@napi-rs/cross-toolchain-arm64-target-s390x'
+ - '@napi-rs/cross-toolchain-arm64-target-x86_64'
+ - '@napi-rs/cross-toolchain-x64-target-aarch64'
+ - '@napi-rs/cross-toolchain-x64-target-armv7'
+ - '@napi-rs/cross-toolchain-x64-target-ppc64le'
+ - '@napi-rs/cross-toolchain-x64-target-s390x'
+ - '@napi-rs/cross-toolchain-x64-target-x86_64'
+ - '@types/node'
+ - supports-color
+
+ '@napi-rs/[email protected]':
+ dependencies:
+ '@napi-rs/lzma': 1.4.5
+ '@napi-rs/tar': 1.1.0
+ debug: 4.4.1
+ transitivePeerDependencies:
+ - supports-color
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ dependencies:
+ '@napi-rs/wasm-runtime': 1.0.3
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optionalDependencies:
+ '@napi-rs/lzma-android-arm-eabi': 1.4.5
+ '@napi-rs/lzma-android-arm64': 1.4.5
+ '@napi-rs/lzma-darwin-arm64': 1.4.5
+ '@napi-rs/lzma-darwin-x64': 1.4.5
+ '@napi-rs/lzma-freebsd-x64': 1.4.5
+ '@napi-rs/lzma-linux-arm-gnueabihf': 1.4.5
+ '@napi-rs/lzma-linux-arm64-gnu': 1.4.5
+ '@napi-rs/lzma-linux-arm64-musl': 1.4.5
+ '@napi-rs/lzma-linux-ppc64-gnu': 1.4.5
+ '@napi-rs/lzma-linux-riscv64-gnu': 1.4.5
+ '@napi-rs/lzma-linux-s390x-gnu': 1.4.5
+ '@napi-rs/lzma-linux-x64-gnu': 1.4.5
+ '@napi-rs/lzma-linux-x64-musl': 1.4.5
+ '@napi-rs/lzma-wasm32-wasi': 1.4.5
+ '@napi-rs/lzma-win32-arm64-msvc': 1.4.5
+ '@napi-rs/lzma-win32-ia32-msvc': 1.4.5
+ '@napi-rs/lzma-win32-x64-msvc': 1.4.5
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ dependencies:
+ '@napi-rs/wasm-runtime': 1.0.3
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optionalDependencies:
+ '@napi-rs/tar-android-arm-eabi': 1.1.0
+ '@napi-rs/tar-android-arm64': 1.1.0
+ '@napi-rs/tar-darwin-arm64': 1.1.0
+ '@napi-rs/tar-darwin-x64': 1.1.0
+ '@napi-rs/tar-freebsd-x64': 1.1.0
+ '@napi-rs/tar-linux-arm-gnueabihf': 1.1.0
+ '@napi-rs/tar-linux-arm64-gnu': 1.1.0
+ '@napi-rs/tar-linux-arm64-musl': 1.1.0
+ '@napi-rs/tar-linux-ppc64-gnu': 1.1.0
+ '@napi-rs/tar-linux-s390x-gnu': 1.1.0
+ '@napi-rs/tar-linux-x64-gnu': 1.1.0
+ '@napi-rs/tar-linux-x64-musl': 1.1.0
+ '@napi-rs/tar-wasm32-wasi': 1.1.0
+ '@napi-rs/tar-win32-arm64-msvc': 1.1.0
+ '@napi-rs/tar-win32-ia32-msvc': 1.1.0
+ '@napi-rs/tar-win32-x64-msvc': 1.1.0
+
+ '@napi-rs/[email protected]':
+ dependencies:
+ '@emnapi/core': 1.5.0
+ '@emnapi/runtime': 1.5.0
+ '@tybys/wasm-util': 0.10.0
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
- '@msgpackr-extract/[email protected]':
+ '@napi-rs/[email protected]':
optional: true
- '@msgpackr-extract/[email protected]':
+ '@napi-rs/[email protected]':
optional: true
- '@msgpackr-extract/[email protected]':
+ '@napi-rs/[email protected]':
optional: true
- '@msgpackr-extract/[email protected]':
+ '@napi-rs/[email protected]':
optional: true
- '@msgpackr-extract/[email protected]':
+ '@napi-rs/[email protected]':
optional: true
- '@msgpackr-extract/[email protected]':
+ '@napi-rs/[email protected]':
optional: true
- '@nangohq/[email protected]':
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
dependencies:
- axios: 1.11.0
- transitivePeerDependencies:
- - debug
+ '@napi-rs/wasm-runtime': 1.0.3
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optional: true
+
+ '@napi-rs/[email protected]':
+ optionalDependencies:
+ '@napi-rs/wasm-tools-android-arm-eabi': 1.0.1
+ '@napi-rs/wasm-tools-android-arm64': 1.0.1
+ '@napi-rs/wasm-tools-darwin-arm64': 1.0.1
+ '@napi-rs/wasm-tools-darwin-x64': 1.0.1
+ '@napi-rs/wasm-tools-freebsd-x64': 1.0.1
+ '@napi-rs/wasm-tools-linux-arm64-gnu': 1.0.1
+ '@napi-rs/wasm-tools-linux-arm64-musl': 1.0.1
+ '@napi-rs/wasm-tools-linux-x64-gnu': 1.0.1
+ '@napi-rs/wasm-tools-linux-x64-musl': 1.0.1
+ '@napi-rs/wasm-tools-wasm32-wasi': 1.0.1
+ '@napi-rs/wasm-tools-win32-arm64-msvc': 1.0.1
+ '@napi-rs/wasm-tools-win32-ia32-msvc': 1.0.1
+ '@napi-rs/wasm-tools-win32-x64-msvc': 1.0.1
'@noble/[email protected]': {}
@@ -8121,6 +9316,68 @@ snapshots:
'@noble/[email protected]': {}
+ '@octokit/[email protected]': {}
+
+ '@octokit/[email protected]':
+ dependencies:
+ '@octokit/auth-token': 6.0.0
+ '@octokit/graphql': 9.0.1
+ '@octokit/request': 10.0.3
+ '@octokit/request-error': 7.0.0
+ '@octokit/types': 14.1.0
+ before-after-hook: 4.0.0
+ universal-user-agent: 7.0.3
+
+ '@octokit/[email protected]':
+ dependencies:
+ '@octokit/types': 14.1.0
+ universal-user-agent: 7.0.3
+
+ '@octokit/[email protected]':
+ dependencies:
+ '@octokit/request': 10.0.3
+ '@octokit/types': 14.1.0
+ universal-user-agent: 7.0.3
+
+ '@octokit/[email protected]': {}
+
+ '@octokit/[email protected](@octokit/[email protected])':
+ dependencies:
+ '@octokit/core': 7.0.3
+ '@octokit/types': 14.1.0
+
+ '@octokit/[email protected](@octokit/[email protected])':
+ dependencies:
+ '@octokit/core': 7.0.3
+
+ '@octokit/[email protected](@octokit/[email protected])':
+ dependencies:
+ '@octokit/core': 7.0.3
+ '@octokit/types': 14.1.0
+
+ '@octokit/[email protected]':
+ dependencies:
+ '@octokit/types': 14.1.0
+
+ '@octokit/[email protected]':
+ dependencies:
+ '@octokit/endpoint': 11.0.0
+ '@octokit/request-error': 7.0.0
+ '@octokit/types': 14.1.0
+ fast-content-type-parse: 3.0.0
+ universal-user-agent: 7.0.3
+
+ '@octokit/[email protected]':
+ dependencies:
+ '@octokit/core': 7.0.3
+ '@octokit/plugin-paginate-rest': 13.1.1(@octokit/[email protected])
+ '@octokit/plugin-request-log': 6.0.0(@octokit/[email protected])
+ '@octokit/plugin-rest-endpoint-methods': 16.0.0(@octokit/[email protected])
+
+ '@octokit/[email protected]':
+ dependencies:
+ '@octokit/openapi-types': 25.1.0
+
'@one-ini/[email protected]': {}
@@ -9041,6 +10298,105 @@ snapshots:
'@opentelemetry/api': 1.9.0
'@opentelemetry/core': 2.0.1(@opentelemetry/[email protected])
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ dependencies:
+ '@napi-rs/wasm-runtime': 1.0.3
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ optional: true
+
+ '@oxc-node/[email protected]':
+ dependencies:
+ pirates: 4.0.7
+ optionalDependencies:
+ '@oxc-node/core-android-arm-eabi': 0.0.32
+ '@oxc-node/core-android-arm64': 0.0.32
+ '@oxc-node/core-darwin-arm64': 0.0.32
+ '@oxc-node/core-darwin-x64': 0.0.32
+ '@oxc-node/core-freebsd-x64': 0.0.32
+ '@oxc-node/core-linux-arm-gnueabihf': 0.0.32
+ '@oxc-node/core-linux-arm64-gnu': 0.0.32
+ '@oxc-node/core-linux-arm64-musl': 0.0.32
+ '@oxc-node/core-linux-ppc64-gnu': 0.0.32
+ '@oxc-node/core-linux-s390x-gnu': 0.0.32
+ '@oxc-node/core-linux-x64-gnu': 0.0.32
+ '@oxc-node/core-linux-x64-musl': 0.0.32
+ '@oxc-node/core-openharmony-arm64': 0.0.32
+ '@oxc-node/core-wasm32-wasi': 0.0.32
+ '@oxc-node/core-win32-arm64-msvc': 0.0.32
+ '@oxc-node/core-win32-ia32-msvc': 0.0.32
+ '@oxc-node/core-win32-x64-msvc': 0.0.32
+
+ '@oxlint/[email protected]':
+ optional: true
+
+ '@oxlint/[email protected]':
+ optional: true
+
+ '@oxlint/[email protected]':
+ optional: true
+
+ '@oxlint/[email protected]':
+ optional: true
+
+ '@oxlint/[email protected]':
+ optional: true
+
+ '@oxlint/[email protected]':
+ optional: true
+
+ '@oxlint/[email protected]':
+ optional: true
+
+ '@oxlint/[email protected]':
+ optional: true
+
'@paralleldrive/[email protected]':
dependencies:
'@noble/hashes': 1.8.0
@@ -9545,7 +10901,7 @@ snapshots:
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9553,7 +10909,7 @@ snapshots:
'@smithy/types': 3.6.0
'@smithy/util-config-provider': 3.0.0
'@smithy/util-middleware': 3.0.8
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9564,7 +10920,7 @@ snapshots:
'@smithy/util-middleware': 3.0.8
'@smithy/util-stream': 3.2.1
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9572,14 +10928,14 @@ snapshots:
'@smithy/property-provider': 3.1.8
'@smithy/types': 3.6.0
'@smithy/url-parser': 3.0.8
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@aws-crypto/crc32': 3.0.0
'@smithy/types': 1.2.0
'@smithy/util-hex-encoding': 1.1.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9587,7 +10943,7 @@ snapshots:
'@smithy/querystring-builder': 3.0.8
'@smithy/types': 3.6.0
'@smithy/util-base64': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9595,37 +10951,37 @@ snapshots:
'@smithy/querystring-builder': 3.0.8
'@smithy/types': 3.6.0
'@smithy/util-base64': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
'@smithy/util-buffer-from': 3.0.0
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/protocol-http': 4.1.5
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9636,7 +10992,7 @@ snapshots:
'@smithy/types': 3.6.0
'@smithy/url-parser': 3.0.8
'@smithy/util-middleware': 3.0.8
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9647,25 +11003,25 @@ snapshots:
'@smithy/types': 3.6.0
'@smithy/util-middleware': 3.0.8
'@smithy/util-retry': 3.0.8
- tslib: 2.6.3
+ tslib: 2.8.1
uuid: 9.0.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/property-provider': 3.1.8
'@smithy/shared-ini-file-loader': 3.1.9
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9673,33 +11029,33 @@ snapshots:
'@smithy/protocol-http': 4.1.5
'@smithy/querystring-builder': 3.0.8
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 1.2.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
'@smithy/util-uri-escape': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9708,7 +11064,7 @@ snapshots:
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9719,7 +11075,7 @@ snapshots:
'@smithy/util-middleware': 1.1.0
'@smithy/util-uri-escape': 1.1.0
'@smithy/util-utf8': 1.1.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9730,7 +11086,7 @@ snapshots:
'@smithy/util-middleware': 3.0.8
'@smithy/util-uri-escape': 3.0.0
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9740,54 +11096,54 @@ snapshots:
'@smithy/protocol-http': 4.1.5
'@smithy/types': 3.6.0
'@smithy/util-stream': 3.2.1
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/querystring-parser': 3.0.8
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/util-buffer-from': 3.0.0
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/is-array-buffer': 1.1.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/is-array-buffer': 2.2.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/is-array-buffer': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9795,7 +11151,7 @@ snapshots:
'@smithy/smithy-client': 3.4.2
'@smithy/types': 3.6.0
bowser: 2.11.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9805,36 +11161,36 @@ snapshots:
'@smithy/property-provider': 3.1.8
'@smithy/smithy-client': 3.4.2
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/node-config-provider': 3.1.9
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/service-error-classification': 3.0.8
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
@@ -9845,36 +11201,36 @@ snapshots:
'@smithy/util-buffer-from': 3.0.0
'@smithy/util-hex-encoding': 3.0.0
'@smithy/util-utf8': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/util-buffer-from': 1.1.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/util-buffer-from': 2.2.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/util-buffer-from': 3.0.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@smithy/[email protected]':
dependencies:
'@smithy/abort-controller': 3.1.6
'@smithy/types': 3.6.0
- tslib: 2.6.3
+ tslib: 2.8.1
'@socket.io/[email protected]': {}
@@ -9944,13 +11300,13 @@ snapshots:
dependencies:
- chalk: 5.4.1
+ chalk: 5.6.0
commander: 12.1.0
typescript: 5.8.3
dependencies:
- chalk: 5.4.1
+ chalk: 5.6.0
commander: 14.0.0
typescript: 5.8.3
@@ -10073,6 +11429,8 @@ snapshots:
'@tanstack/query-core': 5.83.1
react: 18.3.1
+ '@taplo/[email protected]': {}
+
'@tootallnate/[email protected]': {}
'@tsconfig/[email protected]': {}
@@ -10085,6 +11443,10 @@ snapshots:
'@tsconfig/[email protected]': {}
+ '@tybys/[email protected]':
+ dependencies:
+ tslib: 2.8.1
+
'@types/[email protected]': {}
@@ -11111,6 +12473,8 @@ snapshots:
+
@@ -11282,12 +12646,12 @@ snapshots:
ansi-styles: 4.3.0
supports-color: 7.2.0
-
+
@@ -11328,6 +12692,12 @@ snapshots:
slice-ansi: 5.0.0
string-width: 7.2.0
+
+ dependencies:
+ typanion: 3.14.0
+
dependencies:
string-width: 4.2.3
@@ -11742,6 +13112,8 @@ snapshots:
+
@@ -11922,6 +13294,8 @@ snapshots:
+
@@ -11984,6 +13358,12 @@ snapshots:
locate-path: 5.0.0
path-exists: 4.0.0
+ dependencies:
+ locate-path: 7.2.0
+ path-exists: 5.0.0
+ unicorn-magic: 0.1.0
+
@@ -12419,6 +13799,8 @@ snapshots:
+
dependencies:
node-fetch: 2.7.0([email protected])
@@ -12895,6 +14277,8 @@ snapshots:
+
dependencies:
'@metamask/safe-event-emitter': 2.0.0
@@ -12915,7 +14299,7 @@ snapshots:
dependencies:
'@types/diff-match-patch': 1.0.36
- chalk: 5.4.1
+ chalk: 5.6.0
diff-match-patch: 1.0.5
@@ -13024,6 +14408,10 @@ snapshots:
dependencies:
p-locate: 4.1.0
+ dependencies:
+ p-locate: 6.0.0
+
@@ -13102,7 +14490,7 @@ snapshots:
dependencies:
- semver: 7.6.2
+ semver: 7.7.2
@@ -13137,6 +14525,8 @@ snapshots:
+
@@ -13226,6 +14616,8 @@ snapshots:
+
@@ -13283,6 +14675,17 @@ snapshots:
+ dependencies:
+ ansi-styles: 6.2.1
+ cross-spawn: 7.0.6
+ memorystream: 0.3.1
+ picomatch: 4.0.3
+ pidtree: 0.6.0
+ read-package-json-fast: 4.0.0
+ shell-quote: 1.8.3
+ which: 5.0.0
+
dependencies:
path-key: 3.1.1
@@ -13421,6 +14824,17 @@ snapshots:
transitivePeerDependencies:
- zod
+ optionalDependencies:
+ '@oxlint/darwin-arm64': 1.14.0
+ '@oxlint/darwin-x64': 1.14.0
+ '@oxlint/linux-arm64-gnu': 1.14.0
+ '@oxlint/linux-arm64-musl': 1.14.0
+ '@oxlint/linux-x64-gnu': 1.14.0
+ '@oxlint/linux-x64-musl': 1.14.0
+ '@oxlint/win32-arm64': 1.14.0
+ '@oxlint/win32-x64': 1.14.0
+
dependencies:
p-try: 2.2.0
@@ -13429,10 +14843,18 @@ snapshots:
dependencies:
yocto-queue: 0.1.0
+ dependencies:
+ yocto-queue: 1.2.1
+
dependencies:
p-limit: 2.3.0
+ dependencies:
+ p-limit: 4.0.0
+
@@ -13472,6 +14894,8 @@ snapshots:
+
@@ -13525,6 +14949,8 @@ snapshots:
+
@@ -13552,7 +14978,7 @@ snapshots:
sonic-boom: 2.8.0
thread-stream: 0.15.2
dependencies:
@@ -13720,6 +15146,11 @@ snapshots:
+ dependencies:
+ json-parse-even-better-errors: 4.0.0
+ npm-normalize-package-bin: 4.0.0
+
dependencies:
core-util-is: 1.0.3
@@ -13873,6 +15304,8 @@ snapshots:
+
dependencies:
debug: 2.6.9
@@ -13927,6 +15360,8 @@ snapshots:
+
@@ -14320,6 +15755,8 @@ snapshots:
dependencies:
'@mixmark-io/domino': 2.2.0
+
@@ -14337,6 +15774,8 @@ snapshots:
+
@@ -14351,6 +15790,10 @@ snapshots:
+
+
@@ -14602,6 +16045,10 @@ snapshots:
dependencies:
isexe: 2.0.0
+ dependencies:
+ isexe: 3.1.1
+
dependencies:
logform: 2.6.1
@@ -14832,6 +16279,10 @@ snapshots:
+
+
dependencies:
zod: 3.24.2
diff --git a/apps/api/pnpm-workspace.yaml b/apps/api/pnpm-workspace.yaml
new file mode 100644
index 000000000..ce4f0560b
--- /dev/null
+++ b/apps/api/pnpm-workspace.yaml
@@ -0,0 +1,3 @@
+packages:
+ - "native"
+injectWorkspacePackages: true
diff --git a/apps/api/sharedLibs/crawler/.gitignore b/apps/api/sharedLibs/crawler/.gitignore
deleted file mode 100644
index 1de565933..000000000
--- a/apps/api/sharedLibs/crawler/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-target
\ No newline at end of file
diff --git a/apps/api/sharedLibs/crawler/Cargo.lock b/apps/api/sharedLibs/crawler/Cargo.lock
deleted file mode 100644
index 8a753c488..000000000
--- a/apps/api/sharedLibs/crawler/Cargo.lock
+++ /dev/null
@@ -1,514 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 4
-
-[[package]]
-name = "aho-corasick"
-version = "1.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.98"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
-
-[[package]]
-name = "bstr"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
-dependencies = [
- "lazy_static",
- "memchr",
- "regex-automata 0.1.10",
-]
-
-[[package]]
-name = "crawler"
-version = "0.1.0"
-dependencies = [
- "libc",
- "regex",
- "roxmltree",
- "serde",
- "serde_json",
- "texting_robots",
- "url",
-]
-
-[[package]]
-name = "displaydoc"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "form_urlencoded"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
-dependencies = [
- "percent-encoding",
-]
-
-[[package]]
-name = "icu_collections"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47"
-dependencies = [
- "displaydoc",
- "potential_utf",
- "yoke",
- "zerofrom",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locale_core"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a"
-dependencies = [
- "displaydoc",
- "litemap",
- "tinystr",
- "writeable",
- "zerovec",
-]
-
-[[package]]
-name = "icu_normalizer"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_normalizer_data",
- "icu_properties",
- "icu_provider",
- "smallvec",
- "zerovec",
-]
-
-[[package]]
-name = "icu_normalizer_data"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3"
-
-[[package]]
-name = "icu_properties"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_locale_core",
- "icu_properties_data",
- "icu_provider",
- "potential_utf",
- "zerotrie",
- "zerovec",
-]
-
-[[package]]
-name = "icu_properties_data"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632"
-
-[[package]]
-name = "icu_provider"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af"
-dependencies = [
- "displaydoc",
- "icu_locale_core",
- "stable_deref_trait",
- "tinystr",
- "writeable",
- "yoke",
- "zerofrom",
- "zerotrie",
- "zerovec",
-]
-
-[[package]]
-name = "idna"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
-dependencies = [
- "idna_adapter",
- "smallvec",
- "utf8_iter",
-]
-
-[[package]]
-name = "idna_adapter"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
-dependencies = [
- "icu_normalizer",
- "icu_properties",
-]
-
-[[package]]
-name = "itoa"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
-
-[[package]]
-name = "lazy_static"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
-
-[[package]]
-name = "libc"
-version = "0.2.174"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
-
-[[package]]
-name = "litemap"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
-
-[[package]]
-name = "memchr"
-version = "2.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
-
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
-[[package]]
-name = "nom"
-version = "7.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
-[[package]]
-name = "percent-encoding"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
-
-[[package]]
-name = "potential_utf"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585"
-dependencies = [
- "zerovec",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.95"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "regex"
-version = "1.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata 0.4.9",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
-
-[[package]]
-name = "regex-automata"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
-
-[[package]]
-name = "roxmltree"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"
-
-[[package]]
-name = "ryu"
-version = "1.0.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
-
-[[package]]
-name = "serde"
-version = "1.0.219"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.219"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.140"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
-dependencies = [
- "itoa",
- "memchr",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.15.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
-
-[[package]]
-name = "stable_deref_trait"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
-
-[[package]]
-name = "syn"
-version = "2.0.104"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "synstructure"
-version = "0.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "texting_robots"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a"
-dependencies = [
- "anyhow",
- "bstr",
- "lazy_static",
- "nom",
- "percent-encoding",
- "regex",
- "thiserror",
- "url",
-]
-
-[[package]]
-name = "thiserror"
-version = "1.0.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "tinystr"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b"
-dependencies = [
- "displaydoc",
- "zerovec",
-]
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
-
-[[package]]
-name = "url"
-version = "2.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
-dependencies = [
- "form_urlencoded",
- "idna",
- "percent-encoding",
-]
-
-[[package]]
-name = "utf8_iter"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
-
-[[package]]
-name = "writeable"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
-
-[[package]]
-name = "yoke"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc"
-dependencies = [
- "serde",
- "stable_deref_trait",
- "yoke-derive",
- "zerofrom",
-]
-
-[[package]]
-name = "yoke-derive"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "synstructure",
-]
-
-[[package]]
-name = "zerofrom"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
-dependencies = [
- "zerofrom-derive",
-]
-
-[[package]]
-name = "zerofrom-derive"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "synstructure",
-]
-
-[[package]]
-name = "zerotrie"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595"
-dependencies = [
- "displaydoc",
- "yoke",
- "zerofrom",
-]
-
-[[package]]
-name = "zerovec"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"
-dependencies = [
- "yoke",
- "zerofrom",
- "zerovec-derive",
-]
-
-[[package]]
-name = "zerovec-derive"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
diff --git a/apps/api/sharedLibs/crawler/Cargo.toml b/apps/api/sharedLibs/crawler/Cargo.toml
deleted file mode 100644
index 1e43b72e9..000000000
--- a/apps/api/sharedLibs/crawler/Cargo.toml
+++ /dev/null
@@ -1,16 +0,0 @@
-[package]
-name = "crawler"
-version = "0.1.0"
-edition = "2024"
-
-[dependencies]
-libc = "0.2.0"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-url = "2.5.4"
-regex = "1.11.1"
-texting_robots = "0.2.2"
-roxmltree = "0.20"
-
-[lib]
-crate-type = ["cdylib"]
diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rs
deleted file mode 100644
index 0ee510f98..000000000
--- a/apps/api/sharedLibs/crawler/src/lib.rs
+++ /dev/null
@@ -1,635 +0,0 @@
-use std::{collections::HashMap, usize};
-use serde::{Deserialize, Serialize};
-use std::ffi::{CStr, CString};
-use url::Url;
-use regex::Regex;
-use texting_robots::Robot;
-
-#[derive(Deserialize)]
-struct FilterLinksCall {
- links: Vec<String>,
- limit: Option<i64>,
- max_depth: u32,
- base_url: String,
- initial_url: String,
- regex_on_full_url: bool,
- excludes: Vec<String>,
- includes: Vec<String>,
- allow_backward_crawling: bool,
- ignore_robots_txt: bool,
- robots_txt: String,
-}
-
-#[derive(Serialize)]
-struct FilterLinksResult {
- links: Vec<String>,
- denial_reasons: HashMap<String, String>,
-}
-
-#[derive(Serialize, Debug)]
-struct SitemapUrl {
- loc: Vec<String>,
-}
-
-#[derive(Serialize, Debug)]
-struct SitemapEntry {
- loc: Vec<String>,
-}
-
-#[derive(Serialize, Debug)]
-#[serde(untagged)]
-enum ParsedSitemap {
- Urlset { urlset: SitemapUrlset },
- SitemapIndex { sitemapindex: SitemapIndex },
-}
-
-#[derive(Serialize, Debug)]
-struct SitemapUrlset {
- url: Vec<SitemapUrl>,
-}
-
-#[derive(Serialize, Debug)]
-struct SitemapIndex {
- sitemap: Vec<SitemapEntry>,
-}
-
-#[derive(Serialize, Debug)]
-struct SitemapInstruction {
- action: String,
- urls: Vec<String>,
- count: usize,
-}
-
-#[derive(Serialize, Debug)]
-struct SitemapProcessingResult {
- instructions: Vec<SitemapInstruction>,
- total_count: usize,
-}
-
-fn _is_file(url: &Url) -> bool {
- let file_extensions = vec![
- ".png",
- ".jpg",
- ".jpeg",
- ".gif",
- ".css",
- ".js",
- ".ico",
- ".svg",
- ".tiff",
- ".zip",
- ".exe",
- ".dmg",
- ".mp4",
- ".mp3",
- ".wav",
- ".pptx",
- ".xlsx",
- ".avi",
- ".flv",
- ".woff",
- ".ttf",
- ".woff2",
- ".webp",
- ".inc",
- ];
- let url_without_query = url.path().to_lowercase();
- file_extensions.iter().any(|ext| url_without_query.ends_with(ext))
-}
-
-fn _get_url_depth(path: &str) -> u32 {
- path.split('/').filter(|x| *x != "" && *x != "index.php" && *x != "index.html").count() as u32
-}
-
-fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std::error::Error>> {
- let mut denial_reasons = HashMap::new();
-
- let limit = data.limit
- .and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) })
- .unwrap_or(usize::MAX);
-
- if limit == 0 {
- return Ok(FilterLinksResult {
- links: Vec::with_capacity(0),
- denial_reasons,
- });
- }
-
- let base_url = Url::parse(&data.base_url)?;
- let initial_url = Url::parse(&data.initial_url)?;
-
- let excludes_regex = data.excludes.iter().map(|exclude| Regex::new(exclude)).collect::<Vec<Result<Regex, regex::Error>>>();
- let excludes_regex = excludes_regex.into_iter().filter_map(|x| x.ok()).collect::<Vec<Regex>>();
-
- let includes_regex = data.includes.iter().map(|include| Regex::new(include)).collect::<Vec<Result<Regex, regex::Error>>>();
- let includes_regex = includes_regex.into_iter().filter_map(|x| x.ok()).collect::<Vec<Regex>>();
-
- let links = data.links.into_iter()
- .filter(|link| {
- let url = match base_url.join(link) {
- Ok(x) => x,
- Err(_) => {
- denial_reasons.insert(link.clone(), "URL_PARSE_ERROR".to_string());
- return false;
- }
- };
-
- let path = url.path();
- let depth = _get_url_depth(path);
- if depth > data.max_depth {
- denial_reasons.insert(link.clone(), "DEPTH_LIMIT".to_string());
- return false;
- }
-
- let exinc_path = if data.regex_on_full_url {
- url.as_str()
- } else {
- url.path()
- };
-
- if !excludes_regex.is_empty() && excludes_regex.iter().any(|regex| regex.is_match(exinc_path)) {
- denial_reasons.insert(link.clone(), "EXCLUDE_PATTERN".to_string());
- return false;
- }
-
- if !includes_regex.is_empty() && !includes_regex.iter().any(|regex| regex.is_match(exinc_path)) {
- denial_reasons.insert(link.clone(), "INCLUDE_PATTERN".to_string());
- return false;
- }
-
- if !data.allow_backward_crawling {
- if !url.path().starts_with(initial_url.path()) {
- denial_reasons.insert(link.clone(), "BACKWARD_CRAWLING".to_string());
- return false;
- }
- }
-
- if !data.ignore_robots_txt {
- match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {
- Ok(robot) => {
- let allowed = robot.allowed(url.as_str());
- if !allowed {
- match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {
- Ok(robot_alt) => {
- let allowed_alt = robot_alt.allowed(url.as_str());
- if !allowed_alt {
- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
- return false;
- }
- }
- Err(_) => {
- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());
- return false;
- }
- }
- }
- }
- Err(_) => {
- }
- }
- }
-
- if _is_file(&url) {
- denial_reasons.insert(link.clone(), "FILE_TYPE".to_string());
- return false;
- }
-
- true
- })
- .take(limit)
- .collect::<Vec<_>>();
-
- Ok(FilterLinksResult {
- links,
- denial_reasons,
- })
-}
-
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn filter_links(data: *const libc::c_char) -> *mut libc::c_char {
- let data = match serde_json::from_slice(unsafe { CStr::from_ptr(data).to_bytes() }) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as C string: {}", e)).unwrap().into_raw();
- }
- };
-
- let result = match _filter_links(data) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- let result = match serde_json::to_string(&result) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(result).unwrap().into_raw()
-}
-
-fn _parse_sitemap_xml(xml_content: &str) -> Result<ParsedSitemap, Box<dyn std::error::Error>> {
- let doc = roxmltree::Document::parse_with_options(
- xml_content,
- roxmltree::ParsingOptions { allow_dtd: true, ..Default::default() },
- )?;
- let root = doc.root_element();
-
- match root.tag_name().name() {
- "sitemapindex" => {
- let mut sitemaps = Vec::new();
-
- for sitemap_node in root.children().filter(|n| n.is_element() && n.tag_name().name() == "sitemap") {
- if let Some(loc_node) = sitemap_node.children().find(|n| n.is_element() && n.tag_name().name() == "loc") {
- if let Some(loc_text) = loc_node.text() {
- sitemaps.push(SitemapEntry {
- loc: vec![loc_text.to_string()],
- });
- }
- }
- }
-
- Ok(ParsedSitemap::SitemapIndex {
- sitemapindex: SitemapIndex { sitemap: sitemaps }
- })
- },
- "urlset" => {
- let mut urls = Vec::new();
-
- for url_node in root.children().filter(|n| n.is_element() && n.tag_name().name() == "url") {
- if let Some(loc_node) = url_node.children().find(|n| n.is_element() && n.tag_name().name() == "loc") {
- if let Some(loc_text) = loc_node.text() {
- urls.push(SitemapUrl {
- loc: vec![loc_text.to_string()],
- });
- }
- }
- }
-
- Ok(ParsedSitemap::Urlset {
- urlset: SitemapUrlset { url: urls }
- })
- },
- _ => {
- Err("Invalid sitemap format: root element must be 'sitemapindex' or 'urlset'".into())
- }
- }
-}
-
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn parse_sitemap_xml(data: *const libc::c_char) -> *mut libc::c_char {
- let xml_content = match unsafe { CStr::from_ptr(data).to_str() } {
- Ok(s) => s,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as UTF-8 string: {}", e)).unwrap().into_raw();
- }
- };
-
- let result = match _parse_sitemap_xml(xml_content) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- let result = match serde_json::to_string(&result) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(result).unwrap().into_raw()
-}
-
-fn _process_sitemap(xml_content: &str) -> Result<SitemapProcessingResult, Box<dyn std::error::Error>> {
- let parsed = _parse_sitemap_xml(xml_content)?;
- let mut instructions = Vec::new();
- let mut total_count = 0;
-
- match parsed {
- ParsedSitemap::SitemapIndex { sitemapindex } => {
- let sitemap_urls: Vec<String> = sitemapindex.sitemap
- .iter()
- .filter_map(|sitemap| {
- if !sitemap.loc.is_empty() {
- Some(sitemap.loc[0].trim().to_string())
- } else {
- None
- }
- })
- .collect();
-
- if !sitemap_urls.is_empty() {
- instructions.push(SitemapInstruction {
- action: "recurse".to_string(),
- urls: sitemap_urls.clone(),
- count: sitemap_urls.len(),
- });
- total_count += sitemap_urls.len();
- }
- },
- ParsedSitemap::Urlset { urlset } => {
- let mut xml_sitemaps = Vec::new();
- let mut valid_urls = Vec::new();
-
- for url_entry in urlset.url {
- if !url_entry.loc.is_empty() {
- let url = url_entry.loc[0].trim();
- if url.to_lowercase().ends_with(".xml") {
- xml_sitemaps.push(url.to_string());
- } else if let Ok(parsed_url) = Url::parse(url) {
- if !_is_file(&parsed_url) {
- valid_urls.push(url.to_string());
- }
- }
- }
- }
-
- if !xml_sitemaps.is_empty() {
- instructions.push(SitemapInstruction {
- action: "recurse".to_string(),
- urls: xml_sitemaps.clone(),
- count: xml_sitemaps.len(),
- });
- total_count += xml_sitemaps.len();
- }
-
- if !valid_urls.is_empty() {
- instructions.push(SitemapInstruction {
- action: "process".to_string(),
- urls: valid_urls.clone(),
- count: valid_urls.len(),
- });
- total_count += valid_urls.len();
- }
- }
- }
-
- Ok(SitemapProcessingResult {
- instructions,
- total_count,
- })
-}
-
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn process_sitemap(data: *const libc::c_char) -> *mut libc::c_char {
- let xml_content = match unsafe { CStr::from_ptr(data).to_str() } {
- Ok(s) => s,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as UTF-8 string: {}", e)).unwrap().into_raw();
- }
- };
-
- let result = match _process_sitemap(xml_content) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- let result = match serde_json::to_string(&result) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(result).unwrap().into_raw()
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_parse_sitemap_xml_urlset() {
- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- <url>
- <loc>https://example.com/page1</loc>
- </url>
- <url>
- <loc>https://example.com/page2</loc>
- </url>
-</urlset>"#;
-
- let result = _parse_sitemap_xml(xml_content).unwrap();
- match result {
- ParsedSitemap::Urlset { urlset } => {
- assert_eq!(urlset.url.len(), 2);
- assert_eq!(urlset.url[0].loc[0], "https://example.com/page1");
- assert_eq!(urlset.url[1].loc[0], "https://example.com/page2");
- }
- _ => panic!("Expected Urlset variant"),
- }
- }
-
- #[test]
- fn test_parse_sitemap_xml_sitemapindex() {
- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
-<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- <sitemap>
- <loc>https://example.com/sitemap1.xml</loc>
- </sitemap>
- <sitemap>
- <loc>https://example.com/sitemap2.xml</loc>
- </sitemap>
-</sitemapindex>"#;
-
- let result = _parse_sitemap_xml(xml_content).unwrap();
- match result {
- ParsedSitemap::SitemapIndex { sitemapindex } => {
- assert_eq!(sitemapindex.sitemap.len(), 2);
- assert_eq!(sitemapindex.sitemap[0].loc[0], "https://example.com/sitemap1.xml");
- assert_eq!(sitemapindex.sitemap[1].loc[0], "https://example.com/sitemap2.xml");
- }
- _ => panic!("Expected SitemapIndex variant"),
- }
- }
-
- #[test]
- fn test_parse_sitemap_xml_invalid_root() {
- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
-<invalid xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- <url>
- <loc>https://example.com/page1</loc>
- </url>
-</invalid>"#;
-
- let result = _parse_sitemap_xml(xml_content);
- assert!(result.is_err());
- assert!(result.unwrap_err().to_string().contains("Invalid sitemap format"));
- }
-
- #[test]
- fn test_parse_sitemap_xml_malformed() {
- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- <url>
- <loc>https://example.com/page1</loc>
- </url>
-</urlset"#; // Missing closing >
-
- let result = _parse_sitemap_xml(xml_content);
- assert!(result.is_err());
- }
-
- #[test]
- fn test_process_sitemap_urlset() {
- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- <url>
- <loc>https://example.com/page1</loc>
- </url>
- <url>
- <loc>https://example.com/sitemap2.xml</loc>
- </url>
- <url>
- <loc>https://example.com/image.png</loc>
- </url>
-</urlset>"#;
-
- let result = _process_sitemap(xml_content).unwrap();
- assert_eq!(result.instructions.len(), 2);
-
- let recurse_instruction = result.instructions.iter().find(|i| i.action == "recurse").unwrap();
- assert_eq!(recurse_instruction.urls.len(), 1);
- assert_eq!(recurse_instruction.urls[0], "https://example.com/sitemap2.xml");
-
- let process_instruction = result.instructions.iter().find(|i| i.action == "process").unwrap();
- assert_eq!(process_instruction.urls.len(), 1);
- assert_eq!(process_instruction.urls[0], "https://example.com/page1");
- }
-
- #[test]
- fn test_process_sitemap_sitemapindex() {
- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>
-<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
- <sitemap>
- <loc>https://example.com/sitemap1.xml</loc>
- </sitemap>
- <sitemap>
- <loc>https://example.com/sitemap2.xml</loc>
- </sitemap>
-</sitemapindex>"#;
-
- let result = _process_sitemap(xml_content).unwrap();
- assert_eq!(result.instructions.len(), 1);
- assert_eq!(result.instructions[0].action, "recurse");
- assert_eq!(result.instructions[0].urls.len(), 2);
- assert_eq!(result.instructions[0].urls[0], "https://example.com/sitemap1.xml");
- assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");
- }
-
- #[test]
- fn test_filter_links_normal_robots_txt() {
- let data = FilterLinksCall {
- links: vec!["https://example.com/allowed".to_string(), "https://example.com/disallowed".to_string()],
- limit: Some(10),
- includes: vec![],
- excludes: vec![],
- ignore_robots_txt: false,
- robots_txt: "User-agent: *\nDisallow: /disallowed".to_string(),
- max_depth: 10,
- base_url: "https://example.com".to_string(),
- initial_url: "https://example.com".to_string(),
- regex_on_full_url: false,
- allow_backward_crawling: true,
- };
-
- let result = _filter_links(data).unwrap();
- assert_eq!(result.links.len(), 1);
- assert_eq!(result.links[0], "https://example.com/allowed");
- assert!(result.denial_reasons.contains_key("https://example.com/disallowed"));
- assert_eq!(result.denial_reasons.get("https://example.com/disallowed").unwrap(), "ROBOTS_TXT");
- }
-
- #[test]
- fn test_filter_links_malformed_robots_txt() {
- let data = FilterLinksCall {
- links: vec!["https://example.com/test".to_string()],
- limit: Some(10),
- includes: vec![],
- excludes: vec![],
- ignore_robots_txt: false,
- robots_txt: "Invalid robots.txt content with \x00 null bytes and malformed syntax".to_string(),
- max_depth: 10,
- base_url: "https://example.com".to_string(),
- initial_url: "https://example.com".to_string(),
- regex_on_full_url: false,
- allow_backward_crawling: true,
- };
-
- let result = _filter_links(data);
- assert!(result.is_ok());
- let result = result.unwrap();
- assert_eq!(result.links.len(), 1);
- assert_eq!(result.links[0], "https://example.com/test");
- }
-
- #[test]
- fn test_filter_links_non_utf8_robots_txt() {
- let mut non_utf8_bytes = vec![0xFF, 0xFE];
- non_utf8_bytes.extend_from_slice(b"User-agent: *\nDisallow: /blocked");
- let non_utf8_string = String::from_utf8_lossy(&non_utf8_bytes).to_string();
-
- let data = FilterLinksCall {
- links: vec!["https://example.com/allowed".to_string()],
- limit: Some(10),
- includes: vec![],
- excludes: vec![],
- ignore_robots_txt: false,
- robots_txt: non_utf8_string,
- max_depth: 10,
- base_url: "https://example.com".to_string(),
- initial_url: "https://example.com".to_string(),
- regex_on_full_url: false,
- allow_backward_crawling: true,
- };
-
- let result = _filter_links(data);
- assert!(result.is_ok());
- let result = result.unwrap();
- assert_eq!(result.links.len(), 1);
- assert_eq!(result.links[0], "https://example.com/allowed");
- }
-
- #[test]
- fn test_filter_links_char_boundary_issue() {
- let problematic_content = "User-agent: *\nDisallow: /\u{a0}test";
-
- let data = FilterLinksCall {
- links: vec!["https://example.com/test".to_string()],
- limit: Some(10),
- includes: vec![],
- excludes: vec![],
- ignore_robots_txt: false,
- robots_txt: problematic_content.to_string(),
- max_depth: 10,
- base_url: "https://example.com".to_string(),
- initial_url: "https://example.com".to_string(),
- regex_on_full_url: false,
- allow_backward_crawling: true,
- };
-
- let result = _filter_links(data);
- assert!(result.is_ok());
- let result = result.unwrap();
- assert_eq!(result.links.len(), 1);
- assert_eq!(result.links[0], "https://example.com/test");
- }
-
-}
-
-/// Frees a string allocated in Rust-land.
-///
-/// # Safety
-/// ptr must be a non-freed string pointer returned by Rust code.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn free_string(ptr: *mut libc::c_char) {
- drop(unsafe { CString::from_raw(ptr) })
-}
diff --git a/apps/api/sharedLibs/html-transformer/.gitignore b/apps/api/sharedLibs/html-transformer/.gitignore
deleted file mode 100644
index 1de565933..000000000
--- a/apps/api/sharedLibs/html-transformer/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-target
\ No newline at end of file
diff --git a/apps/api/sharedLibs/html-transformer/Cargo.lock b/apps/api/sharedLibs/html-transformer/Cargo.lock
deleted file mode 100644
index 4cfb944ce..000000000
--- a/apps/api/sharedLibs/html-transformer/Cargo.lock
+++ /dev/null
@@ -1,1396 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 4
-
-[[package]]
-name = "allocator-api2"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
-
-[[package]]
-name = "autocfg"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
-
-[[package]]
-name = "base64"
-version = "0.22.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "bitflags"
-version = "2.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
-
-[[package]]
-name = "block-buffer"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "convert_case"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
-
-[[package]]
-name = "cpufeatures"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "crypto-common"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
-[[package]]
-name = "cssparser"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
-dependencies = [
- "cssparser-macros",
- "dtoa-short",
- "itoa 0.4.8",
- "matches",
- "phf 0.8.0",
- "proc-macro2",
- "quote",
- "smallvec",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "cssparser"
-version = "0.29.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f93d03419cb5950ccfd3daf3ff1c7a36ace64609a1a8746d493df1ca0afde0fa"
-dependencies = [
- "cssparser-macros",
- "dtoa-short",
- "itoa 1.0.14",
- "matches",
- "phf 0.10.1",
- "proc-macro2",
- "quote",
- "smallvec",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "cssparser-macros"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
-dependencies = [
- "quote",
- "syn 2.0.96",
-]
-
-[[package]]
-name = "derive_more"
-version = "0.99.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
-dependencies = [
- "convert_case",
- "proc-macro2",
- "quote",
- "rustc_version",
- "syn 2.0.96",
-]
-
-[[package]]
-name = "digest"
-version = "0.10.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
-dependencies = [
- "block-buffer",
- "crypto-common",
-]
-
-[[package]]
-name = "displaydoc"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
-]
-
-[[package]]
-name = "dtoa"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
-
-[[package]]
-name = "dtoa-short"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
-dependencies = [
- "dtoa",
-]
-
-[[package]]
-name = "encoding_rs"
-version = "0.8.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "equivalent"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
-
-[[package]]
-name = "foldhash"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"
-
-[[package]]
-name = "form_urlencoded"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
-dependencies = [
- "percent-encoding",
-]
-
-[[package]]
-name = "futf"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
-dependencies = [
- "mac",
- "new_debug_unreachable",
-]
-
-[[package]]
-name = "fxhash"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
-dependencies = [
- "byteorder",
-]
-
-[[package]]
-name = "generic-array"
-version = "0.14.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
-dependencies = [
- "typenum",
- "version_check",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.1.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.9.0+wasi-snapshot-preview1",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
-dependencies = [
- "cfg-if",
- "libc",
- "r-efi",
- "wasi 0.14.2+wasi-0.2.4",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-
-[[package]]
-name = "hashbrown"
-version = "0.15.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
-dependencies = [
- "allocator-api2",
- "equivalent",
- "foldhash",
-]
-
-[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
-[[package]]
-name = "html-transformer"
-version = "0.1.0"
-dependencies = [
- "kuchikiki",
- "libc",
- "lol_html",
- "nodesig",
- "serde",
- "serde_json",
- "url",
-]
-
-[[package]]
-name = "html5ever"
-version = "0.26.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7"
-dependencies = [
- "log",
- "mac",
- "markup5ever",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "icu_collections"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
-dependencies = [
- "displaydoc",
- "yoke",
- "zerofrom",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
-dependencies = [
- "displaydoc",
- "litemap",
- "tinystr",
- "writeable",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid_transform"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
-dependencies = [
- "displaydoc",
- "icu_locid",
- "icu_locid_transform_data",
- "icu_provider",
- "tinystr",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid_transform_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
-
-[[package]]
-name = "icu_normalizer"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_normalizer_data",
- "icu_properties",
- "icu_provider",
- "smallvec",
- "utf16_iter",
- "utf8_iter",
- "write16",
- "zerovec",
-]
-
-[[package]]
-name = "icu_normalizer_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
-
-[[package]]
-name = "icu_properties"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_locid_transform",
- "icu_properties_data",
- "icu_provider",
- "tinystr",
- "zerovec",
-]
-
-[[package]]
-name = "icu_properties_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
-
-[[package]]
-name = "icu_provider"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
-dependencies = [
- "displaydoc",
- "icu_locid",
- "icu_provider_macros",
- "stable_deref_trait",
- "tinystr",
- "writeable",
- "yoke",
- "zerofrom",
- "zerovec",
-]
-
-[[package]]
-name = "icu_provider_macros"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
-]
-
-[[package]]
-name = "idna"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
-dependencies = [
- "idna_adapter",
- "smallvec",
- "utf8_iter",
-]
-
-[[package]]
-name = "idna_adapter"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
-dependencies = [
- "icu_normalizer",
- "icu_properties",
-]
-
-[[package]]
-name = "indexmap"
-version = "1.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
-dependencies = [
- "autocfg",
- "hashbrown 0.12.3",
-]
-
-[[package]]
-name = "itoa"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
-
-[[package]]
-name = "itoa"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
-
-[[package]]
-name = "kuchikiki"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f29e4755b7b995046f510a7520c42b2fed58b77bd94d5a87a8eb43d2fd126da8"
-dependencies = [
- "cssparser 0.27.2",
- "html5ever",
- "indexmap",
- "matches",
- "selectors 0.22.0",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.169"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
-
-[[package]]
-name = "litemap"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
-
-[[package]]
-name = "lock_api"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
-dependencies = [
- "autocfg",
- "scopeguard",
-]
-
-[[package]]
-name = "log"
-version = "0.4.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
-
-[[package]]
-name = "lol_html"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b1058123f6262982b891dccc395cff0144d9439de366460b47fab719258b96e"
-dependencies = [
- "bitflags 2.8.0",
- "cfg-if",
- "cssparser 0.29.6",
- "encoding_rs",
- "hashbrown 0.15.2",
- "memchr",
- "mime",
- "selectors 0.24.0",
- "thiserror",
-]
-
-[[package]]
-name = "mac"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
-
-[[package]]
-name = "markup5ever"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016"
-dependencies = [
- "log",
- "phf 0.10.1",
- "phf_codegen 0.10.0",
- "string_cache",
- "string_cache_codegen",
- "tendril",
-]
-
-[[package]]
-name = "matches"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
-
-[[package]]
-name = "memchr"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
-
-[[package]]
-name = "mime"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
-
-[[package]]
-name = "new_debug_unreachable"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
-
-[[package]]
-name = "nodesig"
-version = "1.0.0"
-source = "git+https://github.com/firecrawl/nodesig#b8ebc4a710810dbc36e189d9a0df87b877702b3c"
-dependencies = [
- "base64",
- "hex",
- "kuchikiki",
- "rand 0.9.1",
- "sha2",
-]
-
-[[package]]
-name = "nodrop"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
-
-[[package]]
-name = "once_cell"
-version = "1.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
-
-[[package]]
-name = "parking_lot"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-targets",
-]
-
-[[package]]
-name = "percent-encoding"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
-
-[[package]]
-name = "phf"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
-dependencies = [
- "phf_macros 0.8.0",
- "phf_shared 0.8.0",
- "proc-macro-hack",
-]
-
-[[package]]
-name = "phf"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
-dependencies = [
- "phf_macros 0.10.0",
- "phf_shared 0.10.0",
- "proc-macro-hack",
-]
-
-[[package]]
-name = "phf_codegen"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
-dependencies = [
- "phf_generator 0.8.0",
- "phf_shared 0.8.0",
-]
-
-[[package]]
-name = "phf_codegen"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"
-dependencies = [
- "phf_generator 0.10.0",
- "phf_shared 0.10.0",
-]
-
-[[package]]
-name = "phf_generator"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
-dependencies = [
- "phf_shared 0.8.0",
- "rand 0.7.3",
-]
-
-[[package]]
-name = "phf_generator"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
-dependencies = [
- "phf_shared 0.10.0",
- "rand 0.8.5",
-]
-
-[[package]]
-name = "phf_macros"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"
-dependencies = [
- "phf_generator 0.8.0",
- "phf_shared 0.8.0",
- "proc-macro-hack",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "phf_macros"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0"
-dependencies = [
- "phf_generator 0.10.0",
- "phf_shared 0.10.0",
- "proc-macro-hack",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "phf_shared"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
-dependencies = [
- "siphasher",
-]
-
-[[package]]
-name = "phf_shared"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
-dependencies = [
- "siphasher",
-]
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
-dependencies = [
- "zerocopy",
-]
-
-[[package]]
-name = "precomputed-hash"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
-
-[[package]]
-name = "proc-macro-hack"
-version = "0.5.20+deprecated"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.93"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "r-efi"
-version = "5.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
-
-[[package]]
-name = "rand"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
-dependencies = [
- "getrandom 0.1.16",
- "libc",
- "rand_chacha 0.2.2",
- "rand_core 0.5.1",
- "rand_hc",
- "rand_pcg",
-]
-
-[[package]]
-name = "rand"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
-dependencies = [
- "libc",
- "rand_chacha 0.3.1",
- "rand_core 0.6.4",
-]
-
-[[package]]
-name = "rand"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
-dependencies = [
- "rand_chacha 0.9.0",
- "rand_core 0.9.3",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.5.1",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.6.4",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.9.3",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
-dependencies = [
- "getrandom 0.1.16",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom 0.2.15",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
-dependencies = [
- "getrandom 0.3.3",
-]
-
-[[package]]
-name = "rand_hc"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
-dependencies = [
- "rand_core 0.5.1",
-]
-
-[[package]]
-name = "rand_pcg"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
-dependencies = [
- "rand_core 0.5.1",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.5.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
-dependencies = [
- "bitflags 2.8.0",
-]
-
-[[package]]
-name = "rustc_version"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
-dependencies = [
- "semver",
-]
-
-[[package]]
-name = "ryu"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
-
-[[package]]
-name = "scopeguard"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-
-[[package]]
-name = "selectors"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
-dependencies = [
- "bitflags 1.3.2",
- "cssparser 0.27.2",
- "derive_more",
- "fxhash",
- "log",
- "matches",
- "phf 0.8.0",
- "phf_codegen 0.8.0",
- "precomputed-hash",
- "servo_arc 0.1.1",
- "smallvec",
- "thin-slice",
-]
-
-[[package]]
-name = "selectors"
-version = "0.24.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c37578180969d00692904465fb7f6b3d50b9a2b952b87c23d0e2e5cb5013416"
-dependencies = [
- "bitflags 1.3.2",
- "cssparser 0.29.6",
- "derive_more",
- "fxhash",
- "log",
- "phf 0.8.0",
- "phf_codegen 0.8.0",
- "precomputed-hash",
- "servo_arc 0.2.0",
- "smallvec",
-]
-
-[[package]]
-name = "semver"
-version = "1.0.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03"
-
-[[package]]
-name = "serde"
-version = "1.0.217"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.217"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.137"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b"
-dependencies = [
- "itoa 1.0.14",
- "memchr",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "servo_arc"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"
-dependencies = [
- "nodrop",
- "stable_deref_trait",
-]
-
-[[package]]
-name = "servo_arc"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52aa42f8fdf0fed91e5ce7f23d8138441002fa31dca008acf47e6fd4721f741"
-dependencies = [
- "nodrop",
- "stable_deref_trait",
-]
-
-[[package]]
-name = "sha2"
-version = "0.10.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
-
-[[package]]
-name = "siphasher"
-version = "0.3.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
-
-[[package]]
-name = "smallvec"
-version = "1.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
-
-[[package]]
-name = "stable_deref_trait"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
-
-[[package]]
-name = "string_cache"
-version = "0.8.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
-dependencies = [
- "new_debug_unreachable",
- "once_cell",
- "parking_lot",
- "phf_shared 0.10.0",
- "precomputed-hash",
- "serde",
-]
-
-[[package]]
-name = "string_cache_codegen"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
-dependencies = [
- "phf_generator 0.10.0",
- "phf_shared 0.10.0",
- "proc-macro2",
- "quote",
-]
-
-[[package]]
-name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.96"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "synstructure"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
-]
-
-[[package]]
-name = "tendril"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
-dependencies = [
- "futf",
- "mac",
- "utf-8",
-]
-
-[[package]]
-name = "thin-slice"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
-
-[[package]]
-name = "thiserror"
-version = "2.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "2.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
-]
-
-[[package]]
-name = "tinystr"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
-dependencies = [
- "displaydoc",
- "zerovec",
-]
-
-[[package]]
-name = "typenum"
-version = "1.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243"
-
-[[package]]
-name = "url"
-version = "2.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
-dependencies = [
- "form_urlencoded",
- "idna",
- "percent-encoding",
-]
-
-[[package]]
-name = "utf-8"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
-
-[[package]]
-name = "utf16_iter"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
-
-[[package]]
-name = "utf8_iter"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
-
-[[package]]
-name = "version_check"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-
-[[package]]
-name = "wasi"
-version = "0.9.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
-[[package]]
-name = "wasi"
-version = "0.14.2+wasi-0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
-dependencies = [
- "wit-bindgen-rt",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
-dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_gnullvm",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
-
-[[package]]
-name = "windows_i686_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
-
-[[package]]
-name = "wit-bindgen-rt"
-version = "0.39.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
-dependencies = [
- "bitflags 2.8.0",
-]
-
-[[package]]
-name = "write16"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
-
-[[package]]
-name = "writeable"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
-
-[[package]]
-name = "yoke"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
-dependencies = [
- "serde",
- "stable_deref_trait",
- "yoke-derive",
- "zerofrom",
-]
-
-[[package]]
-name = "yoke-derive"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
- "synstructure",
-]
-
-[[package]]
-name = "zerocopy"
-version = "0.7.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
-dependencies = [
- "byteorder",
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.7.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
-]
-
-[[package]]
-name = "zerofrom"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e"
-dependencies = [
- "zerofrom-derive",
-]
-
-[[package]]
-name = "zerofrom-derive"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
- "synstructure",
-]
-
-[[package]]
-name = "zerovec"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
-dependencies = [
- "yoke",
- "zerofrom",
- "zerovec-derive",
-]
-
-[[package]]
-name = "zerovec-derive"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.96",
-]
diff --git a/apps/api/sharedLibs/html-transformer/Cargo.toml b/apps/api/sharedLibs/html-transformer/Cargo.toml
deleted file mode 100644
index 34d06a614..000000000
--- a/apps/api/sharedLibs/html-transformer/Cargo.toml
+++ /dev/null
@@ -1,16 +0,0 @@
-[package]
-name = "html-transformer"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-libc = "0.2.0"
-lol_html = "2.2.0"
-kuchikiki = "0.8.2"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-url = "2.5.4"
-nodesig = { git = "https://github.com/firecrawl/nodesig" }
-
-[lib]
-crate-type = ["cdylib"]
diff --git a/apps/api/sharedLibs/html-transformer/src/lib.rs b/apps/api/sharedLibs/html-transformer/src/lib.rs
deleted file mode 100644
index 5a93e8a86..000000000
--- a/apps/api/sharedLibs/html-transformer/src/lib.rs
+++ /dev/null
@@ -1,823 +0,0 @@
-use std::{collections::{HashMap, HashSet}, ffi::{CStr, CString}};
-
-use nodesig::{get_node_signature, SignatureMode};
-use kuchikiki::{iter::NodeEdge, parse_html, traits::TendrilSink, NodeRef};
-use serde::Deserialize;
-use serde_json::Value;
-use url::Url;
-
-fn _extract_base_href_from_document(document: &NodeRef, url: &Url) -> Result<String, Box<dyn std::error::Error>> {
- if let Some(base) = document.select("base[href]").map_err(|_| "Failed to select base href".to_string())?.next()
- .and_then(|base| base.attributes.borrow().get("href").map(|x| x.to_string())) {
-
- if let Ok(base) = url.join(&base) {
- return Ok(base.to_string());
- }
- }
-
- Ok(url.to_string())
-}
-
-fn _extract_base_href(html: &str, url: &str) -> Result<String, Box<dyn std::error::Error>> {
- let document = parse_html().one(html);
- let url = Url::parse(url)?;
-
- _extract_base_href_from_document(&document, &url)
-}
-
-/// Extracts base href from HTML
-///
-/// # Safety
-/// Input must be a C HTML string and a C URL string. Output will be a string. Output string must be freed with free_string.
-#[no_mangle]
-pub unsafe extern "C" fn extract_base_href(html: *const libc::c_char, url: *const libc::c_char) -> *mut libc::c_char {
- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();
- }
- };
- let url = match unsafe { CStr::from_ptr(url) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input URL as C string").unwrap().into_raw();
- }
- };
-
- let base_href = match _extract_base_href(html, url) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(base_href).unwrap().into_raw()
-}
-
-/// Extracts links from HTML
-///
-/// # Safety
-/// Input options must be a C HTML string. Output will be a JSON string array. Output string must be freed with free_string.
-#[no_mangle]
-pub unsafe extern "C" fn extract_links(html: *const libc::c_char) -> *mut libc::c_char {
- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();
- }
- };
-
- let document = parse_html().one(html);
-
- let mut out: Vec<String> = Vec::new();
-
- let anchors: Vec<_> = match document.select("a[href]").map_err(|_| "Failed to select links") {
- Ok(x) => x.collect(),
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- for anchor in anchors {
- let mut href = match anchor.attributes.borrow().get("href") {
- Some(x) => x.to_string(),
- None => continue,
- };
-
- if href.starts_with("http:/") && !href.starts_with("http://") {
- href = format!("http://{}", &href[6..]);
- } else if href.starts_with("https:/") && !href.starts_with("https://") {
- href = format!("https://{}", &href[7..]);
- }
-
- out.push(href);
- }
-
- let links_out = match serde_json::ser::to_string(&out) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(links_out).unwrap().into_raw()
-}
-
-macro_rules! insert_meta_name {
- ($out:ident, $document:ident, $metaName:expr, $outName:expr) => {
- if let Some(x) = $document.select(&format!("meta[name=\"{}\"]", $metaName)).map_err(|_| "Failed to select meta name")?.next().and_then(|description| description.attributes.borrow().get("content").map(|x| x.to_string())) {
- $out.insert(($outName).to_string(), Value::String(x));
- }
- };
-}
-
-macro_rules! insert_meta_property {
- ($out:ident, $document:ident, $metaName:expr, $outName:expr) => {
- if let Some(x) = $document.select(&format!("meta[property=\"{}\"]", $metaName)).map_err(|_| "Failed to select meta property")?.next().and_then(|description| description.attributes.borrow().get("content").map(|x| x.to_string())) {
- $out.insert(($outName).to_string(), Value::String(x));
- }
- };
-}
-
-fn _extract_metadata(html: &str) -> Result<String, Box<dyn std::error::Error>> {
- let document = parse_html().one(html);
- let mut out = HashMap::<String, Value>::new();
-
- if let Some(title) = document.select("title").map_err(|_| "Failed to select title")?.next() {
- out.insert("title".to_string(), Value::String(title.text_contents()));
- }
- // insert_meta_name!(out, document, "description", "description");
-
- if let Some(favicon_link) = document.select("link[rel=\"icon\"]").map_err(|_| "Failed to select favicon")?.next()
- .and_then(|x| x.attributes.borrow().get("href").map(|x| x.to_string()))
- .or_else(|| document.select("link[rel*=\"icon\"]").unwrap().next()
- .and_then(|x| x.attributes.borrow().get("href").map(|x| x.to_string()))) {
- out.insert("favicon".to_string(), Value::String(favicon_link));
- }
-
- if let Some(lang) = document.select("html[lang]").map_err(|_| "Failed to select lang")?.next().and_then(|x| x.attributes.borrow().get("lang").map(|x| x.to_string())) {
- out.insert("language".to_string(), Value::String(lang));
- }
-
- // insert_meta_name!(out, document, "keywords", "keywords");
- // insert_meta_name!(out, document, "robots", "robots");
- insert_meta_property!(out, document, "og:title", "ogTitle");
- insert_meta_property!(out, document, "og:description", "ogDescription");
- insert_meta_property!(out, document, "og:url", "ogUrl");
- insert_meta_property!(out, document, "og:image", "ogImage");
- insert_meta_property!(out, document, "og:audio", "ogAudio");
- insert_meta_property!(out, document, "og:determiner", "ogDeterminer");
- insert_meta_property!(out, document, "og:locale", "ogLocale");
-
- for meta in document.select("meta[property=\"og:locale:alternate\"]").map_err(|_| "Failed to select og locale alternate")? {
- let attrs = meta.attributes.borrow();
-
- if let Some(content) = attrs.get("content") {
- if let Some(v) = out.get_mut("ogLocaleAlternate") {
- match v {
- Value::Array(x) => {
- x.push(Value::String(content.to_string()));
- },
- _ => unreachable!(),
- }
- } else {
- out.insert("ogLocaleAlternate".to_string(), Value::Array(vec! [Value::String(content.to_string())]));
- }
- }
- }
-
- insert_meta_property!(out, document, "og:site_name", "ogSiteName");
- insert_meta_property!(out, document, "og:video", "ogVideo");
- insert_meta_name!(out, document, "article:section", "articleSection");
- insert_meta_name!(out, document, "article:tag", "articleTag");
- insert_meta_property!(out, document, "article:published_time", "publishedTime");
- insert_meta_property!(out, document, "article:modified_time", "modifiedTime");
- insert_meta_name!(out, document, "dcterms.keywords", "dcTermsKeywords");
- insert_meta_name!(out, document, "dc.description", "dcDescription");
- insert_meta_name!(out, document, "dc.subject", "dcSubject");
- insert_meta_name!(out, document, "dcterms.subject", "dcTermsSubject");
- insert_meta_name!(out, document, "dcterms.audience", "dcTermsAudience");
- insert_meta_name!(out, document, "dc.type", "dcType");
- insert_meta_name!(out, document, "dcterms.type", "dcTermsType");
- insert_meta_name!(out, document, "dc.date", "dcDate");
- insert_meta_name!(out, document, "dc.date.created", "dcDateCreated");
- insert_meta_name!(out, document, "dcterms.created", "dcTermsCreated");
-
- for meta in document.select("meta").map_err(|_| "Failed to select meta")? {
- let meta = meta.as_node().as_element().unwrap();
- let attrs = meta.attributes.borrow();
-
- if let Some(name) = attrs.get("name").or_else(|| attrs.get("property")).or_else(|| attrs.get("itemprop")) {
- if let Some(content) = attrs.get("content") {
- if let Some(v) = out.get(name) {
- match v {
- Value::String(existing) => {
- if name == "description" {
- out.insert(name.to_string(), Value::String(format!("{}, {}", existing, content)));
- } else if name != "title" { // preserve title tag in metadata
- out.insert(name.to_string(), Value::Array(vec! [Value::String(existing.clone()), Value::String(content.to_string())]));
- }
- },
- Value::Array(existing_array) => {
- if name == "description" {
- let mut values: Vec<String> = existing_array.iter()
- .filter_map(|v| match v {
- Value::String(s) => Some(s.clone()),
- _ => None,
- })
- .collect();
- values.push(content.to_string());
- out.insert(name.to_string(), Value::String(values.join(", ")));
- } else {
- match out.get_mut(name) {
- Some(Value::Array(x)) => {
- x.push(Value::String(content.to_string()));
- },
- _ => unreachable!(),
- }
- }
- },
- _ => unreachable!(),
- }
- } else {
- out.insert(name.to_string(), Value::String(content.to_string()));
- }
- }
- }
- }
-
- Ok(serde_json::ser::to_string(&out)?)
-}
-
-/// Extracts metadata from HTML
-///
-/// # Safety
-/// Input options must be a C HTML string. Output will be a JSON object. Output string must be freed with free_string.
-#[no_mangle]
-pub unsafe extern "C" fn extract_metadata(html: *const libc::c_char) -> *mut libc::c_char {
- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();
- }
- };
-
- let metadata_out = match _extract_metadata(html) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(metadata_out).unwrap().into_raw()
-}
-
-const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [
- "header",
- "footer",
- "nav",
- "aside",
- ".header",
- ".top",
- ".navbar",
- "#header",
- ".footer",
- ".bottom",
- "#footer",
- ".sidebar",
- ".side",
- ".aside",
- "#sidebar",
- ".modal",
- ".popup",
- "#modal",
- ".overlay",
- ".ad",
- ".ads",
- ".advert",
- "#ad",
- ".lang-selector",
- ".language",
- "#language-selector",
- ".social",
- ".social-media",
- ".social-links",
- "#social",
- ".menu",
- ".navigation",
- "#nav",
- ".breadcrumbs",
- "#breadcrumbs",
- ".share",
- "#share",
- ".widget",
- "#widget",
- ".cookie",
- "#cookie",
- ".fc-decoration",
-];
-
-const FORCE_INCLUDE_MAIN_TAGS: [&str; 13] = [
- "#main",
-
- // swoogo event software as .widget in all of their content
- ".swoogo-cols",
- ".swoogo-text",
- ".swoogo-table-div",
- ".swoogo-space",
- ".swoogo-alert",
- ".swoogo-sponsors",
- ".swoogo-title",
- ".swoogo-tabs",
- ".swoogo-logo",
- ".swoogo-image",
- ".swoogo-button",
- ".swoogo-agenda",
-];
-
-#[derive(Deserialize)]
-struct TranformHTMLOptions {
- html: String,
- url: String,
- include_tags: Vec<String>,
- exclude_tags: Vec<String>,
- only_main_content: bool,
- omce_signatures: Option<Vec<String>>,
-}
-
-struct ImageSource {
- url: String,
- size: f64,
- is_x: bool,
-}
-
-fn _transform_html_inner(opts: TranformHTMLOptions) -> Result<String, Box<dyn std::error::Error>> {
- let mut document = parse_html().one(opts.html.as_ref());
- let url = Url::parse(&_extract_base_href_from_document(&document, &Url::parse(&opts.url)?)?)?;
-
- if !opts.include_tags.is_empty() {
- let new_document = parse_html().one("<div></div>");
- let root = new_document.select_first("div").map_err(|_| "Failed to select root element")?;
-
- for x in opts.include_tags.iter() {
- let matching_nodes: Vec<_> = document.select(x).map_err(|_| "Failed to include_tags tags")?.collect();
- for tag in matching_nodes {
- root.as_node().append(tag.as_node().clone());
- }
- }
-
- document = new_document;
- }
-
- while let Ok(x) = document.select_first("head") {
- x.as_node().detach();
- }
-
- while let Ok(x) = document.select_first("meta") {
- x.as_node().detach();
- }
-
- while let Ok(x) = document.select_first("noscript") {
- x.as_node().detach();
- }
-
- while let Ok(x) = document.select_first("style") {
- x.as_node().detach();
- }
-
- while let Ok(x) = document.select_first("script") {
- x.as_node().detach();
- }
-
- // The first operation applied must be OMCE.
- if opts.only_main_content {
- if let Some(signatures) = opts.omce_signatures.as_ref() {
- let mut nodes_to_drop: Vec<NodeRef> = Vec::new();
-
- let modes = signatures.iter().map(|x| Into::<SignatureMode>::into(x.split(':').nth(1).unwrap().to_string())).collect::<HashSet<_>>();
- for mode in modes {
- let matcher = format!(":{}:", Into::<String>::into(mode));
- let signatures = signatures.iter().filter(|x| x.contains(&matcher)).cloned().collect::<HashSet<_>>();
-
- for edge in document.traverse() {
- match edge {
- NodeEdge::Start(_) => {},
- NodeEdge::End(node) => {
- if node.as_element().is_none() {
- continue;
- }
-
- if node.text_contents().trim().is_empty() {
- continue;
- }
-
- let signature = get_node_signature(&node, mode);
- if signatures.contains(&signature) {
- nodes_to_drop.push(node);
- }
- }
- }
- }
- }
-
- for node in nodes_to_drop {
- node.detach();
- }
- }
- }
-
- for x in opts.exclude_tags.iter() {
- // TODO: implement weird version
- while let Ok(x) = document.select_first(x) {
- x.as_node().detach();
- }
- }
-
- if opts.only_main_content {
- for x in EXCLUDE_NON_MAIN_TAGS.iter() {
- let x: Vec<_> = document.select(x).map_err(|_| "Failed to select tags")?.collect();
- for tag in x {
- if !FORCE_INCLUDE_MAIN_TAGS.iter().any(|x| tag.as_node().select(x).is_ok_and(|mut x| x.next().is_some())) {
- tag.as_node().detach();
- }
- }
- }
- }
-
- let srcset_images: Vec<_> = document.select("img[srcset]").map_err(|_| "Failed to select srcset images")?.collect();
- for img in srcset_images {
- let mut sizes: Vec<ImageSource> = img.attributes.borrow().get("srcset").ok_or("Failed to get srcset")?.split(",").filter_map(|x| {
- let tok: Vec<&str> = x.trim().split(" ").collect();
- let last_token = tok[tok.len() - 1]; // SAFETY: split is guaranteed to return at least one token
- let (last_token, last_token_used) = if tok.len() > 1 && !last_token.is_empty() && (last_token.ends_with("x") || last_token.ends_with("w")) {
- (last_token, true)
- } else {
- ("1x", false)
- };
-
- // split off the last character of the last token and parse as size
- if let Some((last_index, _)) = last_token.char_indices().last() {
- if let Ok(parsed_size) = last_token[..last_index].parse() {
- Some(ImageSource {
- url: if last_token_used { tok[0..tok.len()-1].join(" ") } else { tok.join(" ") },
- size: parsed_size,
- is_x: last_token.ends_with("x")
- })
- } else {
- None
- }
- } else {
- None
- }
- }).collect();
-
- if sizes.iter().all(|x| x.is_x) {
- if let Some(src) = img.attributes.borrow().get("src").map(|x| x.to_string()) {
- sizes.push(ImageSource {
- url: src,
- size: 1.0,
- is_x: true,
- });
- }
- }
-
- sizes.sort_by(|a, b| b.size.partial_cmp(&a.size).unwrap_or(std::cmp::Ordering::Equal));
-
- if let Some(biggest) = sizes.first() {
- img.attributes.borrow_mut().insert("src", biggest.url.clone());
- }
- }
-
- let src_images: Vec<_> = document.select("img[src]").map_err(|_| "Failed to select src images")?.collect();
- for img in src_images {
- let old = img.attributes.borrow().get("src").map(|x| x.to_string()).ok_or("Failed to get src")?;
- if let Ok(new) = url.join(&old) {
- img.attributes.borrow_mut().insert("src", new.to_string());
- }
- }
-
- let href_anchors: Vec<_> = document.select("a[href]").map_err(|_| "Failed to select href anchors")?.collect();
- for anchor in href_anchors {
- let old = anchor.attributes.borrow().get("href").map(|x| x.to_string()).ok_or("Failed to get href")?;
- if let Ok(new) = url.join(&old) {
- anchor.attributes.borrow_mut().insert("href", new.to_string());
- }
- }
-
- Ok(document.to_string())
-}
-
-/// Transforms rawHtml to html (formerly removeUnwantedElements)
-///
-/// # Safety
-/// Input options must be a C JSON string. Output will be an HTML string. Output string must be freed with free_string.
-#[no_mangle]
-pub unsafe extern "C" fn transform_html(opts: *const libc::c_char) -> *mut libc::c_char {
- let opts: TranformHTMLOptions = match unsafe { CStr::from_ptr(opts) }.to_str().map_err(|_| ()).and_then(|x| serde_json::de::from_str(x).map_err(|_| ())) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR").unwrap().into_raw();
- }
- };
-
- let out = match _transform_html_inner(opts) {
- Ok(x) => x,
- Err(e) => format!("RUSTFC:ERROR:{}", e),
- };
-
- CString::new(out).unwrap().into_raw()
-}
-
-fn _get_inner_json(html: &str) -> Result<String, ()> {
- Ok(parse_html().one(html).select_first("body")?.text_contents())
-}
-
-/// For JSON pages retrieved by browser engines, this function can be used to transform it back into valid JSON.
-///
-/// # Safety
-/// Input must be a C HTML string. Output will be an HTML string. Output string must be freed with free_string.
-#[no_mangle]
-pub unsafe extern "C" fn get_inner_json(html: *const libc::c_char) -> *mut libc::c_char {
- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();
- }
- };
-
- let out = match _get_inner_json(html) {
- Ok(x) => x,
- Err(_) => "RUSTFC:ERROR".to_string(),
- };
-
- CString::new(out).unwrap().into_raw()
-}
-
-#[derive(Deserialize)]
-struct AttributeSelector {
- selector: String,
- attribute: String,
-}
-
-#[derive(Deserialize)]
-struct ExtractAttributesOptions {
- selectors: Vec<AttributeSelector>,
-}
-
-fn _extract_attributes(html: &str, options: &ExtractAttributesOptions) -> Result<String, Box<dyn std::error::Error>> {
- let document = parse_html().one(html);
- let mut results = Vec::new();
-
- for selector_config in &options.selectors {
- let mut values = Vec::new();
-
- let elements: Vec<_> = match document.select(&selector_config.selector).map_err(|_| format!("Failed to select with selector: {}", selector_config.selector)) {
- Ok(x) => x.collect(),
- Err(_) => {
- // Return empty values for invalid selectors rather than failing completely
- Vec::new()
- }
- };
-
- for element in elements {
- // Try the attribute as-is first
- if let Some(attr_value) = element.attributes.borrow().get(selector_config.attribute.as_str()) {
- values.push(attr_value.to_string());
- continue;
- }
-
- // If not found and doesn't start with 'data-', try with 'data-' prefix
- if !selector_config.attribute.starts_with("data-") {
- let data_attr = format!("data-{}", selector_config.attribute);
- if let Some(attr_value) = element.attributes.borrow().get(data_attr.as_str()) {
- values.push(attr_value.to_string());
- }
- }
- }
-
- results.push(serde_json::json!({
- "selector": selector_config.selector,
- "attribute": selector_config.attribute,
- "values": values
- }));
- }
-
- Ok(serde_json::to_string(&results)?)
-}
-
-/// Extracts attributes from HTML elements using CSS selectors
-///
-/// # Safety
-/// html must be a C HTML string. options must be a C JSON string with format:
-/// {"selectors": [{"selector": "...", "attribute": "..."}]}
-/// Output will be a JSON string. Output string must be freed with free_string.
-#[no_mangle]
-pub unsafe extern "C" fn extract_attributes(html: *const libc::c_char, options: *const libc::c_char) -> *mut libc::c_char {
- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();
- }
- };
-
- let options_str = match unsafe { CStr::from_ptr(options) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input options as C string").unwrap().into_raw();
- }
- };
-
- let parsed_options: ExtractAttributesOptions = match serde_json::from_str(options_str) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Failed to parse options JSON: {}", e)).unwrap().into_raw();
- }
- };
-
- let result = match _extract_attributes(html, &parsed_options) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(result).unwrap().into_raw()
-}
-
-fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn std::error::Error>> {
- let document = parse_html().one(html);
- let base_url = Url::parse(base_url)?;
- let base_href = _extract_base_href_from_document(&document, &base_url)?;
- let base_href_url = Url::parse(&base_href)?;
- let mut images = HashSet::<String>::new();
-
- // Helper function to resolve image URLs
- let resolve_image_url = |src: &str| -> Result<String, Box<dyn std::error::Error>> {
- // Skip data URIs and blob URLs
- if src.starts_with("data:") || src.starts_with("blob:") {
- return Ok(src.to_string());
- }
-
- // Handle absolute URLs
- if src.starts_with("http://") || src.starts_with("https://") {
- return Ok(src.to_string());
- }
-
- // Handle protocol-relative URLs
- if src.starts_with("//") {
- let resolved = base_url.join(src)?;
- return Ok(resolved.to_string());
- }
-
- // Handle relative URLs
- let resolved = base_href_url.join(src)?;
- Ok(resolved.to_string())
- };
-
- // Extract from img tags
- let img_elements: Vec<_> = match document.select("img").map_err(|_| "Failed to select img tags") {
- Ok(x) => x.collect(),
- Err(e) => return Err(e.into()),
- };
-
- for img in img_elements {
- let attrs = img.attributes.borrow();
-
- // Extract from src attribute
- if let Some(src) = attrs.get("src") {
- if let Ok(resolved) = resolve_image_url(src) {
- images.insert(resolved);
- }
- }
-
- // Extract from data-src (lazy loading)
- if let Some(data_src) = attrs.get("data-src") {
- if let Ok(resolved) = resolve_image_url(data_src) {
- images.insert(resolved);
- }
- }
-
- // Extract from srcset (responsive images)
- if let Some(srcset) = attrs.get("srcset") {
- for part in srcset.split(',') {
- if let Some(url) = part.trim().split_whitespace().next() {
- if !url.is_empty() {
- if let Ok(resolved) = resolve_image_url(url) {
- images.insert(resolved);
- }
- }
- }
- }
- }
- }
-
- // Extract from picture source elements
- let source_elements: Vec<_> = match document.select("picture source").map_err(|_| "Failed to select picture source") {
- Ok(x) => x.collect(),
- Err(_) => Vec::new(),
- };
-
- for source in source_elements {
- if let Some(srcset) = source.attributes.borrow().get("srcset") {
- for part in srcset.split(',') {
- if let Some(url) = part.trim().split_whitespace().next() {
- if !url.is_empty() {
- if let Ok(resolved) = resolve_image_url(url) {
- images.insert(resolved);
- }
- }
- }
- }
- }
- }
-
- // Extract from meta tags (Open Graph, Twitter Cards)
- let meta_selectors = [
- "meta[property=\"og:image\"]",
- "meta[property=\"og:image:url\"]",
- "meta[property=\"og:image:secure_url\"]",
- "meta[name=\"twitter:image\"]",
- "meta[name=\"twitter:image:src\"]",
- "meta[itemprop=\"image\"]"
- ];
-
- for selector in &meta_selectors {
- if let Ok(elements) = document.select(selector) {
- for element in elements {
- if let Some(content) = element.attributes.borrow().get("content") {
- if let Ok(resolved) = resolve_image_url(content) {
- images.insert(resolved);
- }
- }
- }
- }
- }
-
- // Extract from link tags (favicons, apple-touch-icons)
- let link_selectors = [
- "link[rel*=\"icon\"]",
- "link[rel*=\"apple-touch-icon\"]",
- "link[rel*=\"image_src\"]"
- ];
-
- for selector in &link_selectors {
- if let Ok(elements) = document.select(selector) {
- for element in elements {
- if let Some(href) = element.attributes.borrow().get("href") {
- if let Ok(resolved) = resolve_image_url(href) {
- images.insert(resolved);
- }
- }
- }
- }
- }
-
- // Extract from video poster attributes
- if let Ok(video_elements) = document.select("video[poster]") {
- for video in video_elements {
- if let Some(poster) = video.attributes.borrow().get("poster") {
- if let Ok(resolved) = resolve_image_url(poster) {
- images.insert(resolved);
- }
- }
- }
- }
-
- // Filter out javascript: URLs for security and validate URLs
- let filtered_images: Vec<String> = images.into_iter()
- .filter(|url| !url.to_lowercase().starts_with("javascript:"))
- .filter(|url| !url.is_empty())
- .filter(|url| {
- // Validate URLs (allow data URIs and valid URLs)
- url.starts_with("data:") || url.starts_with("blob:") || Url::parse(url).is_ok()
- })
- .collect();
-
- Ok(filtered_images)
-}
-
-/// Extracts images from HTML
-///
-/// # Safety
-/// Input must be a C HTML string and base URL string. Output will be a JSON string array. Output string must be freed with free_string.
-#[no_mangle]
-pub unsafe extern "C" fn extract_images(html: *const libc::c_char, base_url: *const libc::c_char) -> *mut libc::c_char {
- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();
- }
- };
-
- let base_url = match unsafe { CStr::from_ptr(base_url) }.to_str().map_err(|_| ()) {
- Ok(x) => x,
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input base URL as C string").unwrap().into_raw();
- }
- };
-
- let images = match _extract_images(html, base_url) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- let images_out = match serde_json::ser::to_string(&images) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(images_out).unwrap().into_raw()
-}
-
-/// Frees a string allocated in Rust-land.
-///
-/// # Safety
-/// ptr must be a non-freed string pointer returned by Rust code.
-#[no_mangle]
-pub unsafe extern "C" fn free_string(ptr: *mut libc::c_char) {
- drop(unsafe { CString::from_raw(ptr) })
-}
diff --git a/apps/api/sharedLibs/pdf-parser/.gitignore b/apps/api/sharedLibs/pdf-parser/.gitignore
deleted file mode 100644
index 1de565933..000000000
--- a/apps/api/sharedLibs/pdf-parser/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-target
\ No newline at end of file
diff --git a/apps/api/sharedLibs/pdf-parser/Cargo.lock b/apps/api/sharedLibs/pdf-parser/Cargo.lock
deleted file mode 100644
index 5635c68aa..000000000
--- a/apps/api/sharedLibs/pdf-parser/Cargo.lock
+++ /dev/null
@@ -1,1019 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 4
-
-[[package]]
-name = "adler2"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
-
-[[package]]
-name = "aes"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
-dependencies = [
- "cfg-if",
- "cipher",
- "cpufeatures",
-]
-
-[[package]]
-name = "android-tzdata"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
-
-[[package]]
-name = "android_system_properties"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
-
-[[package]]
-name = "bitflags"
-version = "2.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
-
-[[package]]
-name = "block-buffer"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "block-padding"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "bumpalo"
-version = "3.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
-
-[[package]]
-name = "bytecount"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
-
-[[package]]
-name = "cbc"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
-dependencies = [
- "cipher",
-]
-
-[[package]]
-name = "cc"
-version = "1.2.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766"
-dependencies = [
- "shlex",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "chrono"
-version = "0.4.41"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"
-dependencies = [
- "android-tzdata",
- "iana-time-zone",
- "num-traits",
- "windows-link",
-]
-
-[[package]]
-name = "cipher"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
-dependencies = [
- "crypto-common",
- "inout",
-]
-
-[[package]]
-name = "core-foundation-sys"
-version = "0.8.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
-
-[[package]]
-name = "cpufeatures"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "crc32fast"
-version = "1.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
-dependencies = [
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
-
-[[package]]
-name = "crypto-common"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
-[[package]]
-name = "deranged"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e"
-dependencies = [
- "powerfmt",
-]
-
-[[package]]
-name = "digest"
-version = "0.10.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
-dependencies = [
- "block-buffer",
- "crypto-common",
-]
-
-[[package]]
-name = "ecb"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7"
-dependencies = [
- "cipher",
-]
-
-[[package]]
-name = "either"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
-
-[[package]]
-name = "encoding_rs"
-version = "0.8.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
-dependencies = [
- "cfg-if",
-]
-
-[[package]]
-name = "equivalent"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
-
-[[package]]
-name = "flate2"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"
-dependencies = [
- "crc32fast",
- "miniz_oxide",
-]
-
-[[package]]
-name = "generic-array"
-version = "0.14.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
-dependencies = [
- "typenum",
- "version_check",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
-dependencies = [
- "cfg-if",
- "libc",
- "r-efi",
- "wasi",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.15.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"
-
-[[package]]
-name = "iana-time-zone"
-version = "0.1.63"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8"
-dependencies = [
- "android_system_properties",
- "core-foundation-sys",
- "iana-time-zone-haiku",
- "js-sys",
- "log",
- "wasm-bindgen",
- "windows-core",
-]
-
-[[package]]
-name = "iana-time-zone-haiku"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "indexmap"
-version = "2.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
-dependencies = [
- "equivalent",
- "hashbrown",
-]
-
-[[package]]
-name = "inout"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
-dependencies = [
- "block-padding",
- "generic-array",
-]
-
-[[package]]
-name = "itoa"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
-
-[[package]]
-name = "jiff"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93"
-dependencies = [
- "jiff-static",
- "jiff-tzdb-platform",
- "log",
- "portable-atomic",
- "portable-atomic-util",
- "serde",
- "windows-sys",
-]
-
-[[package]]
-name = "jiff-static"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "jiff-tzdb"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1283705eb0a21404d2bfd6eef2a7593d240bc42a0bdb39db0ad6fa2ec026524"
-
-[[package]]
-name = "jiff-tzdb-platform"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8"
-dependencies = [
- "jiff-tzdb",
-]
-
-[[package]]
-name = "js-sys"
-version = "0.3.77"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
-dependencies = [
- "once_cell",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.172"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
-
-[[package]]
-name = "log"
-version = "0.4.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
-
-[[package]]
-name = "lopdf"
-version = "0.36.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fa2559e99ba0f26a12458aabc754432c805bbb8cba516c427825a997af1fb7"
-dependencies = [
- "aes",
- "bitflags",
- "cbc",
- "chrono",
- "ecb",
- "encoding_rs",
- "flate2",
- "indexmap",
- "itoa",
- "jiff",
- "log",
- "md-5",
- "nom",
- "nom_locate",
- "rand",
- "rangemap",
- "rayon",
- "sha2",
- "stringprep",
- "thiserror",
- "time",
- "weezl",
-]
-
-[[package]]
-name = "md-5"
-version = "0.10.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
-dependencies = [
- "cfg-if",
- "digest",
-]
-
-[[package]]
-name = "memchr"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
-
-[[package]]
-name = "miniz_oxide"
-version = "0.8.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
-dependencies = [
- "adler2",
-]
-
-[[package]]
-name = "nom"
-version = "8.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "nom_locate"
-version = "5.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d"
-dependencies = [
- "bytecount",
- "memchr",
- "nom",
-]
-
-[[package]]
-name = "num-conv"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
-
-[[package]]
-name = "num-traits"
-version = "0.2.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.21.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
-
-[[package]]
-name = "pdf-parser"
-version = "0.1.0"
-dependencies = [
- "libc",
- "lopdf",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "portable-atomic"
-version = "1.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
-
-[[package]]
-name = "portable-atomic-util"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
-dependencies = [
- "portable-atomic",
-]
-
-[[package]]
-name = "powerfmt"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
-dependencies = [
- "zerocopy",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.95"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "r-efi"
-version = "5.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
-
-[[package]]
-name = "rand"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
-dependencies = [
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "rangemap"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f60fcc7d6849342eff22c4350c8b9a989ee8ceabc4b481253e8946b9fe83d684"
-
-[[package]]
-name = "rayon"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
-dependencies = [
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
-dependencies = [
- "crossbeam-deque",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "rustversion"
-version = "1.0.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
-
-[[package]]
-name = "ryu"
-version = "1.0.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
-
-[[package]]
-name = "serde"
-version = "1.0.219"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.219"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.143"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a"
-dependencies = [
- "itoa",
- "memchr",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "sha2"
-version = "0.10.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
-dependencies = [
- "cfg-if",
- "cpufeatures",
- "digest",
-]
-
-[[package]]
-name = "shlex"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
-
-[[package]]
-name = "stringprep"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
-dependencies = [
- "unicode-bidi",
- "unicode-normalization",
- "unicode-properties",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.101"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "thiserror"
-version = "2.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "2.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "time"
-version = "0.3.41"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40"
-dependencies = [
- "deranged",
- "itoa",
- "num-conv",
- "powerfmt",
- "serde",
- "time-core",
- "time-macros",
-]
-
-[[package]]
-name = "time-core"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"
-
-[[package]]
-name = "time-macros"
-version = "0.2.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49"
-dependencies = [
- "num-conv",
- "time-core",
-]
-
-[[package]]
-name = "tinyvec"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"
-dependencies = [
- "tinyvec_macros",
-]
-
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
-
-[[package]]
-name = "typenum"
-version = "1.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
-
-[[package]]
-name = "unicode-bidi"
-version = "0.3.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
-
-[[package]]
-name = "unicode-normalization"
-version = "0.1.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
-dependencies = [
- "tinyvec",
-]
-
-[[package]]
-name = "unicode-properties"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
-
-[[package]]
-name = "version_check"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-
-[[package]]
-name = "wasi"
-version = "0.14.2+wasi-0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
-dependencies = [
- "wit-bindgen-rt",
-]
-
-[[package]]
-name = "wasm-bindgen"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
-dependencies = [
- "cfg-if",
- "once_cell",
- "rustversion",
- "wasm-bindgen-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
-dependencies = [
- "bumpalo",
- "log",
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "wasm-bindgen-backend",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "weezl"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3"
-
-[[package]]
-name = "windows-core"
-version = "0.61.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
-dependencies = [
- "windows-implement",
- "windows-interface",
- "windows-link",
- "windows-result",
- "windows-strings",
-]
-
-[[package]]
-name = "windows-implement"
-version = "0.60.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "windows-interface"
-version = "0.59.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "windows-link"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
-
-[[package]]
-name = "windows-result"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
-dependencies = [
- "windows-link",
-]
-
-[[package]]
-name = "windows-strings"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
-dependencies = [
- "windows-link",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
-dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_gnullvm",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
-
-[[package]]
-name = "windows_i686_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
-
-[[package]]
-name = "wit-bindgen-rt"
-version = "0.39.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "zerocopy"
-version = "0.8.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb"
-dependencies = [
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.8.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
diff --git a/apps/api/sharedLibs/pdf-parser/Cargo.toml b/apps/api/sharedLibs/pdf-parser/Cargo.toml
deleted file mode 100644
index 7f3945f64..000000000
--- a/apps/api/sharedLibs/pdf-parser/Cargo.toml
+++ /dev/null
@@ -1,13 +0,0 @@
-[package]
-name = "pdf-parser"
-version = "0.1.0"
-edition = "2021"
-
-[dependencies]
-libc = "0.2.0"
-lopdf = "0.36.0"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-
-[lib]
-crate-type = ["cdylib"]
diff --git a/apps/api/sharedLibs/pdf-parser/src/lib.rs b/apps/api/sharedLibs/pdf-parser/src/lib.rs
deleted file mode 100644
index b83262f4d..000000000
--- a/apps/api/sharedLibs/pdf-parser/src/lib.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-use std::ffi::{CStr, CString};
-use serde::Serialize;
-
-#[derive(Debug, Clone, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub struct PDFMetadata {
- num_pages: i32,
-
- #[serde(skip_serializing_if = "Option::is_none")]
- title: Option<String>,
-}
-
-fn _get_pdf_metadata(path: &str) -> Result<PDFMetadata, String> {
- let doc = match lopdf::Document::load(path) {
- Ok(x) => x,
- Err(_) => {
- return Err("Failed to load PDF".to_string());
- }
- };
-
- let num_pages = doc.get_pages().len() as i32;
- let title = doc.trailer.get(b"Info")
- .and_then(|info| info.as_dict()
- .and_then(|info| info.get(b"Title"))
- .and_then(|title| title.as_str().map(|s| String::from_utf8_lossy(s).to_string()))
- ).ok()
- .or_else(|| doc.objects.iter()
- .find_map(|(_i, obj)| obj.as_dict()
- .and_then(|obj| obj.get(b"Title"))
- .and_then(|title| title.as_str())
- .map(|s| String::from_utf8_lossy(s).to_string())
- .ok()
- )
- )
- .map(|x| x.trim_matches(|x| char::is_whitespace(x) || x == '\0').to_string());
-
- Ok(PDFMetadata { num_pages, title })
-}
-
-/// Returns the metadata of a PDF file
-///
-/// # Safety
-/// Input path must be a C string of a path pointing to a PDF file. Output will be a JSON string of the PDF metadata.
-#[no_mangle]
-pub unsafe extern "C" fn get_pdf_metadata(path: *const libc::c_char) -> *const libc::c_char {
- let path: String = match unsafe { CStr::from_ptr(path) }.to_str().map_err(|_| ()) {
- Ok(x) => x.to_string(),
- Err(_) => {
- return CString::new("RUSTFC:ERROR:Failed to parse input path as C string").unwrap().into_raw();
- }
- };
-
- let metadata = match _get_pdf_metadata(&path) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Failed to get PDF metadata: {}", e)).unwrap().into_raw();
- }
- };
-
- let json = match serde_json::to_string(&metadata) {
- Ok(x) => x,
- Err(e) => {
- return CString::new(format!("RUSTFC:ERROR:Serde failed to serialize metadata: {}", e)).unwrap().into_raw();
- }
- };
-
- CString::new(json).unwrap().into_raw()
-}
diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.ts
index 46e398feb..bb033d5cf 100644
--- a/apps/api/src/__tests__/snips/v1/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts
@@ -1,3 +1,4 @@
+import { filterLinks } from "@mendable/firecrawl-rs";
import {
asyncCrawl,
asyncCrawlWaitForFinish,
@@ -9,7 +10,6 @@ import {
scrapeTimeout,
} from "./lib";
import { describe, it, expect } from "@jest/globals";
-import { filterLinks } from "../../../lib/crawler";
let identity: Identity;
@@ -398,23 +398,23 @@ describe("Robots.txt FFI Integration tests", () => {
"https://example.com/disallowed",
],
limit: 10,
- max_depth: 10,
- base_url: "https://example.com",
- initial_url: "https://example.com",
- regex_on_full_url: false,
+ maxDepth: 10,
+ baseUrl: "https://example.com",
+ initialUrl: "https://example.com",
+ regexOnFullUrl: false,
excludes: [],
includes: [],
- allow_backward_crawling: true,
- ignore_robots_txt: false,
- robots_txt: "User-agent: *\nDisallow: /disallowed",
+ allowBackwardCrawling: true,
+ ignoreRobotsTxt: false,
+ robotsTxt: "User-agent: *\nDisallow: /disallowed",
});
expect(result.links).toHaveLength(1);
expect(result.links[0]).toBe("https://example.com/allowed");
- expect(result.denial_reasons.has("https://example.com/disallowed")).toBe(
+ expect("https://example.com/disallowed" in result.denialReasons).toBe(
true,
);
- expect(result.denial_reasons.get("https://example.com/disallowed")).toBe(
+ expect(result.denialReasons["https://example.com/disallowed"]).toBe(
"ROBOTS_TXT",
);
},
@@ -427,15 +427,15 @@ describe("Robots.txt FFI Integration tests", () => {
const result = await filterLinks({
links: ["https://example.com/test"],
limit: 10,
- max_depth: 10,
- base_url: "https://example.com",
- initial_url: "https://example.com",
- regex_on_full_url: false,
+ maxDepth: 10,
+ baseUrl: "https://example.com",
+ initialUrl: "https://example.com",
+ regexOnFullUrl: false,
excludes: [],
includes: [],
- allow_backward_crawling: true,
- ignore_robots_txt: false,
- robots_txt:
+ allowBackwardCrawling: true,
+ ignoreRobotsTxt: false,
+ robotsTxt:
"Invalid robots.txt content with \x00 null bytes and malformed syntax",
});
@@ -453,15 +453,15 @@ describe("Robots.txt FFI Integration tests", () => {
const result = await filterLinks({
links: ["https://example.com/allowed"],
limit: 10,
- max_depth: 10,
- base_url: "https://example.com",
- initial_url: "https://example.com",
- regex_on_full_url: false,
+ maxDepth: 10,
+ baseUrl: "https://example.com",
+ initialUrl: "https://example.com",
+ regexOnFullUrl: false,
excludes: [],
includes: [],
- allow_backward_crawling: true,
- ignore_robots_txt: false,
- robots_txt: nonUtf8Content,
+ allowBackwardCrawling: true,
+ ignoreRobotsTxt: false,
+ robotsTxt: nonUtf8Content,
});
expect(result.links).toHaveLength(1);
@@ -477,15 +477,15 @@ describe("Robots.txt FFI Integration tests", () => {
const result = await filterLinks({
links: ["https://example.com/safe"],
limit: 10,
- max_depth: 10,
- base_url: "https://example.com",
- initial_url: "https://example.com",
- regex_on_full_url: false,
+ maxDepth: 10,
+ baseUrl: "https://example.com",
+ initialUrl: "https://example.com",
+ regexOnFullUrl: false,
excludes: [],
includes: [],
- allow_backward_crawling: true,
- ignore_robots_txt: false,
- robots_txt: problematicContent,
+ allowBackwardCrawling: true,
+ ignoreRobotsTxt: false,
+ robotsTxt: problematicContent,
});
expect(result.links).toHaveLength(1);
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 8eef9302b..70c077abc 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -9,7 +9,7 @@ import {
scrapeTimeout,
} from "./lib";
import { describe, it, expect } from "@jest/globals";
-import { filterLinks } from "../../../lib/crawler";
+import { filterLinks } from "@mendable/firecrawl-rs";
let identity: Identity;
@@ -400,23 +400,23 @@ describe("Robots.txt FFI Integration tests", () => {
"https://example.com/disallowed",
],
limit: 10,
- max_depth: 10,
- base_url: "https://example.com",
- initial_url: "https://example.com",
- regex_on_full_url: false,
+ maxDepth: 10,
+ baseUrl: "https://example.com",
+ initialUrl: "https://example.com",
+ regexOnFullUrl: false,
excludes: [],
includes: [],
- allow_backward_crawling: true,
- ignore_robots_txt: false,
- robots_txt: "User-agent: *\nDisallow: /disallowed",
+ allowBackwardCrawling: true,
+ ignoreRobotsTxt: false,
+ robotsTxt: "User-agent: *\nDisallow: /disallowed",
});
expect(result.links).toHaveLength(1);
expect(result.links[0]).toBe("https://example.com/allowed");
- expect(result.denial_reasons.has("https://example.com/disallowed")).toBe(
+ expect("https://example.com/disallowed" in result.denialReasons).toBe(
true,
);
- expect(result.denial_reasons.get("https://example.com/disallowed")).toBe(
+ expect(result.denialReasons["https://example.com/disallowed"]).toBe(
"ROBOTS_TXT",
);
},
@@ -429,15 +429,15 @@ describe("Robots.txt FFI Integration tests", () => {
const result = await filterLinks({
links: ["https://example.com/test"],
limit: 10,
- max_depth: 10,
- base_url: "https://example.com",
- initial_url: "https://example.com",
- regex_on_full_url: false,
+ maxDepth: 10,
+ baseUrl: "https://example.com",
+ initialUrl: "https://example.com",
+ regexOnFullUrl: false,
excludes: [],
includes: [],
- allow_backward_crawling: true,
- ignore_robots_txt: false,
- robots_txt:
+ allowBackwardCrawling: true,
+ ignoreRobotsTxt: false,
+ robotsTxt:
"Invalid robots.txt content with \x00 null bytes and malformed syntax",
});
@@ -455,15 +455,15 @@ describe("Robots.txt FFI Integration tests", () => {
const result = await filterLinks({
links: ["https://example.com/allowed"],
limit: 10,
- max_depth: 10,
- base_url: "https://example.com",
- initial_url: "https://example.com",
- regex_on_full_url: false,
+ maxDepth: 10,
+ baseUrl: "https://example.com",
+ initialUrl: "https://example.com",
+ regexOnFullUrl: false,
excludes: [],
includes: [],
- allow_backward_crawling: true,
- ignore_robots_txt: false,
- robots_txt: nonUtf8Content,
+ allowBackwardCrawling: true,
+ ignoreRobotsTxt: false,
+ robotsTxt: nonUtf8Content,
});
expect(result.links).toHaveLength(1);
@@ -479,15 +479,15 @@ describe("Robots.txt FFI Integration tests", () => {
const result = await filterLinks({
links: ["https://example.com/safe"],
limit: 10,
- max_depth: 10,
- base_url: "https://example.com",
- initial_url: "https://example.com",
- regex_on_full_url: false,
+ maxDepth: 10,
+ baseUrl: "https://example.com",
+ initialUrl: "https://example.com",
+ regexOnFullUrl: false,
excludes: [],
includes: [],
- allow_backward_crawling: true,
- ignore_robots_txt: false,
- robots_txt: problematicContent,
+ allowBackwardCrawling: true,
+ ignoreRobotsTxt: false,
+ robotsTxt: problematicContent,
});
expect(result.links).toHaveLength(1);
diff --git a/apps/api/src/lib/__tests__/html-transformer.test.ts b/apps/api/src/lib/__tests__/html-transformer.test.ts
index e3ddd76d7..8e93e3f82 100644
--- a/apps/api/src/lib/__tests__/html-transformer.test.ts
+++ b/apps/api/src/lib/__tests__/html-transformer.test.ts
@@ -2,7 +2,8 @@ import {
extractLinks,
extractMetadata,
transformHtml,
-} from "../html-transformer";
+ TransformHtmlOptions,
+} from "@mendable/firecrawl-rs";
describe("HTML Transformer", () => {
describe("extractLinks", () => {
@@ -190,9 +191,9 @@ describe("HTML Transformer", () => {
});
describe("extractMetadata", () => {
- it("should return empty array for null or undefined input", async () => {
- expect(await extractMetadata(null)).toEqual([]);
- expect(await extractMetadata(undefined)).toEqual([]);
+ it("should return empty object for null or undefined input", async () => {
+ expect(await extractMetadata(null)).toEqual({});
+ expect(await extractMetadata(undefined)).toEqual({});
});
it("should extract comprehensive metadata from HTML content", async () => {
@@ -264,12 +265,12 @@ describe("HTML Transformer", () => {
describe("transformHtml", () => {
it("should transform HTML content according to options", async () => {
- const options = {
+ const options: TransformHtmlOptions = {
html: "<div><p>Test</p><span>Remove me</span></div>",
url: "https://example.com",
- include_tags: ["p"],
- exclude_tags: ["span"],
- only_main_content: true,
+ includeTags: ["p"],
+ excludeTags: ["span"],
+ onlyMainContent: true,
};
const result = await transformHtml(options);
@@ -278,7 +279,7 @@ describe("HTML Transformer", () => {
});
it("should handle complex content filtering", async () => {
- const options = {
+ const options: TransformHtmlOptions = {
html: `
<div class="wrapper">
<header>
@@ -297,9 +298,9 @@ describe("HTML Transformer", () => {
</div>
`,
url: "https://example.com",
- include_tags: ["article", "h1", "p"],
- exclude_tags: ["nav", "aside", "footer", ".ads", ".social-share"],
- only_main_content: true,
+ includeTags: ["article", "h1", "p"],
+ excludeTags: ["nav", "aside", "footer", ".ads", ".social-share"],
+ onlyMainContent: true,
};
const result = await transformHtml(options);
@@ -312,7 +313,7 @@ describe("HTML Transformer", () => {
});
it("should handle nested content preservation and absolute links", async () => {
- const options = {
+ const options: TransformHtmlOptions = {
html: `
<article>
<div class="content">
@@ -326,9 +327,9 @@ describe("HTML Transformer", () => {
</article>
`,
url: "https://example.com",
- include_tags: ["article", "p", "ul", "li"],
- exclude_tags: [],
- only_main_content: true,
+ includeTags: ["article", "p", "ul", "li"],
+ excludeTags: [],
+ onlyMainContent: true,
};
const result = await transformHtml(options);
@@ -338,12 +339,12 @@ describe("HTML Transformer", () => {
});
it("should handle empty HTML content", async () => {
- const options = {
+ const options: TransformHtmlOptions = {
html: "",
url: "https://example.com",
- include_tags: [],
- exclude_tags: [],
- only_main_content: false,
+ includeTags: [],
+ excludeTags: [],
+ onlyMainContent: false,
};
const result = await transformHtml(options);
@@ -351,12 +352,12 @@ describe("HTML Transformer", () => {
});
it("should handle malformed HTML", async () => {
- const options = {
+ const options: TransformHtmlOptions = {
html: "<div>Unclosed div",
url: "https://example.com",
- include_tags: [],
- exclude_tags: [],
- only_main_content: false,
+ includeTags: [],
+ excludeTags: [],
+ onlyMainContent: false,
};
const result = await transformHtml(options);
@@ -364,7 +365,7 @@ describe("HTML Transformer", () => {
});
it("should handle HTML with comments and scripts", async () => {
- const options = {
+ const options: TransformHtmlOptions = {
html: `
<div>
<!-- Comment -->
@@ -375,9 +376,9 @@ describe("HTML Transformer", () => {
</div>
`,
url: "https://example.com",
- include_tags: ["p"],
- exclude_tags: ["script", "style", "noscript"],
- only_main_content: true,
+ includeTags: ["p"],
+ excludeTags: ["script", "style", "noscript"],
+ onlyMainContent: true,
};
const result = await transformHtml(options);
@@ -388,7 +389,7 @@ describe("HTML Transformer", () => {
});
it("should handle special characters and encoding", async () => {
- const options = {
+ const options: TransformHtmlOptions = {
html: `
<div>
<p>&copy; 2024</p>
@@ -398,9 +399,9 @@ describe("HTML Transformer", () => {
</div>
`,
url: "https://example.com",
- include_tags: ["p"],
- exclude_tags: [],
- only_main_content: true,
+ includeTags: ["p"],
+ excludeTags: [],
+ onlyMainContent: true,
};
const result = await transformHtml(options);
@@ -410,7 +411,7 @@ describe("HTML Transformer", () => {
});
it("should make all URLs absolute", async () => {
- const options = {
+ const options: TransformHtmlOptions = {
html: `
<div>
<a href="https://example.com/fullurl">hi</a>
@@ -424,9 +425,9 @@ describe("HTML Transformer", () => {
</div>
`,
url: "https://example.com",
- include_tags: [],
- exclude_tags: [],
- only_main_content: true,
+ includeTags: [],
+ excludeTags: [],
+ onlyMainContent: true,
};
const result = await transformHtml(options);
diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.ts
deleted file mode 100644
index cf4c311d5..000000000
--- a/apps/api/src/lib/crawler.ts
+++ /dev/null
@@ -1,164 +0,0 @@
-import koffi, { KoffiFunction } from "koffi";
-import { stat } from "fs/promises";
-import type { DenialReason } from "../scraper/WebScraper/crawler";
-import { CRAWLER_PATH } from "../natives";
-
-// TODO: add a timeout to the Rust transformer
-
-export type FilterLinksCall = {
- links: string[];
- limit: number | undefined;
- max_depth: number;
- base_url: string;
- initial_url: string;
- regex_on_full_url: boolean;
- excludes: string[];
- includes: string[];
- allow_backward_crawling: boolean;
- ignore_robots_txt: boolean;
- robots_txt: string;
-};
-
-export type FilterLinksResult = {
- links: string[];
- denial_reasons: Map<string, keyof typeof DenialReason>;
-};
-
-class RustCrawler {
- private static instance: RustCrawler;
- private _freeString: KoffiFunction;
- private _filterLinks: KoffiFunction;
- private _parseSitemapXml: KoffiFunction;
- private _processSitemap: KoffiFunction;
-
- private constructor() {
- const lib = koffi.load(CRAWLER_PATH);
- this._freeString = lib.func("free_string", "void", ["string"]);
- const cstn = "CString:" + crypto.randomUUID();
- const freedResultString = koffi.disposable(
- cstn,
- "string",
- this._freeString,
- );
- this._filterLinks = lib.func("filter_links", freedResultString, ["string"]);
- this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, [
- "string",
- ]);
- this._processSitemap = lib.func("process_sitemap", freedResultString, [
- "string",
- ]);
- }
-
- public static async getInstance(): Promise<RustCrawler> {
- if (!RustCrawler.instance) {
- try {
- await stat(CRAWLER_PATH);
- } catch (_) {
- throw Error("Rust crawler shared library not found");
- }
- RustCrawler.instance = new RustCrawler();
- }
- return RustCrawler.instance;
- }
-
- public async filterLinks(call: FilterLinksCall): Promise<FilterLinksResult> {
- return new Promise<FilterLinksResult>((resolve, reject) => {
- this._filterLinks.async(
- JSON.stringify(call),
- (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- return reject(new Error(res.split("RUSTFC:ERROR:")[1]));
- }
-
- if (res === "RUSTFC:ERROR") {
- return reject(
- new Error("Something went wrong on the Rust side."),
- );
- }
-
- try {
- const raw = JSON.parse(res);
- const result: FilterLinksResult = {
- links: raw.links,
- denial_reasons: new Map(Object.entries(raw.denial_reasons)),
- };
- resolve(result);
- } catch (e) {
- reject(e);
- }
- }
- },
- );
- });
- }
-
- public async parseSitemapXml(xmlContent: string): Promise<any> {
- return new Promise<any>((resolve, reject) => {
- this._parseSitemapXml.async(xmlContent, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- return reject(new Error(res.split("RUSTFC:ERROR:")[1]));
- }
-
- if (res === "RUSTFC:ERROR") {
- return reject(new Error("Something went wrong on the Rust side."));
- }
-
- try {
- const result = JSON.parse(res);
- resolve(result);
- } catch (e) {
- reject(e);
- }
- }
- });
- });
- }
-
- public async processSitemap(xmlContent: string): Promise<any> {
- return new Promise<any>((resolve, reject) => {
- this._processSitemap.async(xmlContent, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- return reject(new Error(res.split("RUSTFC:ERROR:")[1]));
- }
-
- if (res === "RUSTFC:ERROR") {
- return reject(new Error("Something went wrong on the Rust side."));
- }
-
- try {
- const result = JSON.parse(res);
- resolve(result);
- } catch (e) {
- reject(e);
- }
- }
- });
- });
- }
-}
-
-export async function filterLinks(
- call: FilterLinksCall,
-): Promise<FilterLinksResult> {
- const converter = await RustCrawler.getInstance();
- return await converter.filterLinks(call);
-}
-
-export async function parseSitemapXml(xmlContent: string): Promise<any> {
- const converter = await RustCrawler.getInstance();
- return await converter.parseSitemapXml(xmlContent);
-}
-
-export async function processSitemap(xmlContent: string): Promise<any> {
- const converter = await RustCrawler.getInstance();
- return await converter.processSitemap(xmlContent);
-}
diff --git a/apps/api/src/lib/html-transformer.ts b/apps/api/src/lib/html-transformer.ts
deleted file mode 100644
index 247837e25..000000000
--- a/apps/api/src/lib/html-transformer.ts
+++ /dev/null
@@ -1,284 +0,0 @@
-import koffi, { KoffiFunction } from "koffi";
-import { stat } from "fs/promises";
-import { HTML_TRANSFORMER_PATH } from "../natives";
-
-type TransformHtmlOptions = {
- html: string;
- url: string;
- include_tags: string[];
- exclude_tags: string[];
- only_main_content: boolean;
- omce_signatures?: string[];
-};
-
-class RustHTMLTransformer {
- private static instance: RustHTMLTransformer;
- private _extractLinks: KoffiFunction;
- private _extractImages: KoffiFunction;
- private _extractBaseHref: KoffiFunction;
- private _extractMetadata: KoffiFunction;
- private _transformHtml: KoffiFunction;
- private _freeString: KoffiFunction;
- private _getInnerJSON: KoffiFunction;
- private _extractAttributes: KoffiFunction;
-
- private constructor() {
- const lib = koffi.load(HTML_TRANSFORMER_PATH);
- this._freeString = lib.func("free_string", "void", ["string"]);
- const cstn = "CString:" + crypto.randomUUID();
- const freedResultString = koffi.disposable(
- cstn,
- "string",
- this._freeString,
- );
- this._extractLinks = lib.func("extract_links", freedResultString, [
- "string",
- ]);
- this._extractImages = lib.func("extract_images", freedResultString, [
- "string",
- "string",
- ]);
- this._extractBaseHref = lib.func("extract_base_href", freedResultString, [
- "string",
- "string",
- ]);
- this._extractMetadata = lib.func("extract_metadata", freedResultString, [
- "string",
- ]);
- this._transformHtml = lib.func("transform_html", freedResultString, [
- "string",
- ]);
- this._getInnerJSON = lib.func("get_inner_json", freedResultString, [
- "string",
- ]);
- this._extractAttributes = lib.func(
- "extract_attributes",
- freedResultString,
- ["string", "string"],
- );
- }
-
- public static async getInstance(): Promise<RustHTMLTransformer> {
- if (!RustHTMLTransformer.instance) {
- try {
- await stat(HTML_TRANSFORMER_PATH);
- } catch (_) {
- throw Error("Rust html-transformer shared library not found");
- }
- RustHTMLTransformer.instance = new RustHTMLTransformer();
- }
- return RustHTMLTransformer.instance;
- }
-
- public async extractLinks(html: string): Promise<string[]> {
- return new Promise<string[]>((resolve, reject) => {
- this._extractLinks.async(html, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- resolve(JSON.parse(res));
- }
- });
- });
- }
-
- public async extractImages(html: string, baseUrl: string): Promise<string[]> {
- return new Promise<string[]>((resolve, reject) => {
- this._extractImages.async(html, baseUrl, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- reject(new Error(res.replace("RUSTFC:ERROR:", "")));
- } else {
- resolve(JSON.parse(res));
- }
- }
- });
- });
- }
-
- public async extractBaseHref(html: string, url: string): Promise<string> {
- return new Promise<string>((resolve, reject) => {
- this._extractBaseHref.async(html, url, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- resolve(res);
- }
- });
- });
- }
-
- public async extractMetadata(html: string): Promise<any> {
- return new Promise<string[]>((resolve, reject) => {
- this._extractMetadata.async(html, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- resolve(JSON.parse(res));
- }
- });
- });
- }
-
- public async transformHtml(opts: TransformHtmlOptions): Promise<string> {
- return new Promise<string>((resolve, reject) => {
- this._transformHtml.async(
- JSON.stringify(opts),
- (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res === "RUSTFC:ERROR" || res.startsWith("RUSTFC:ERROR:")) {
- reject(
- new Error(
- res.startsWith("RUSTFC:ERROR:")
- ? "Something went wrong on the Rust side. " +
- res.split("RUSTFC:ERROR:")[1]
- : "Something went wrong on the Rust side.",
- ),
- );
- } else {
- resolve(res);
- }
- }
- },
- );
- });
- }
-
- public async getInnerJSON(html: string): Promise<string> {
- return new Promise<string>((resolve, reject) => {
- this._getInnerJSON.async(html, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res === "RUSTFC:ERROR") {
- reject(new Error("Something went wrong on the Rust side."));
- } else {
- resolve(res);
- }
- }
- });
- });
- }
-
- public async extractAttributes(
- html: string,
- options: string,
- ): Promise<string> {
- return new Promise<string>((resolve, reject) => {
- this._extractAttributes.async(
- html,
- options,
- (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- reject(
- new Error(
- "Rust attribute extraction failed: " +
- res.split("RUSTFC:ERROR:")[1],
- ),
- );
- } else {
- resolve(res);
- }
- }
- },
- );
- });
- }
-}
-
-export async function extractLinks(
- html: string | null | undefined,
-): Promise<string[]> {
- if (!html) {
- return [];
- }
-
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.extractLinks(html);
-}
-
-export async function extractImages(
- html: string | null | undefined,
- baseUrl: string = "",
-): Promise<string[]> {
- if (!html) {
- return [];
- }
-
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.extractImages(html, baseUrl);
-}
-
-export async function extractBaseHref(
- html: string | null | undefined,
- url: string,
-): Promise<string> {
- if (!html) {
- return url;
- }
-
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.extractBaseHref(html, url);
-}
-
-export async function extractMetadata(
- html: string | null | undefined,
-): Promise<any> {
- if (!html) {
- return [];
- }
-
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.extractMetadata(html);
-}
-
-export async function transformHtml(
- opts: TransformHtmlOptions,
-): Promise<string> {
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.transformHtml(opts);
-}
-
-export async function getInnerJSON(html: string): Promise<string> {
- const converter = await RustHTMLTransformer.getInstance();
- return await converter.getInnerJSON(html);
-}
-
-export type AttributeSelector = {
- selector: string;
- attribute: string;
-};
-
-export type AttributeResult = {
- selector: string;
- attribute: string;
- values: string[];
-};
-
-export async function extractAttributesRust(
- html: string,
- selectors: AttributeSelector[],
-): Promise<AttributeResult[]> {
- if (!html || selectors.length === 0) {
- return [];
- }
-
- const converter = await RustHTMLTransformer.getInstance();
- const options = JSON.stringify({ selectors });
- const resultJson = await converter.extractAttributes(html, options);
-
- try {
- return JSON.parse(resultJson);
- } catch (error) {
- throw new Error(
- `Failed to parse Rust attribute extraction result: ${error}`,
- );
- }
-}
diff --git a/apps/api/src/lib/pdf-parser.ts b/apps/api/src/lib/pdf-parser.ts
deleted file mode 100644
index b930b78a7..000000000
--- a/apps/api/src/lib/pdf-parser.ts
+++ /dev/null
@@ -1,74 +0,0 @@
-import koffi, { KoffiFunction } from "koffi";
-import { join } from "path";
-import { stat } from "fs/promises";
-import { platform } from "os";
-import { PDF_PARSER_PATH } from "../natives";
-
-// TODO: add a timeout to the Rust parser
-
-export type PDFMetadata = {
- numPages: number;
- title?: string;
-};
-
-class RustPDFParser {
- private static instance: RustPDFParser;
- private _getPDFMetadata: KoffiFunction;
-
- private constructor() {
- const lib = koffi.load(PDF_PARSER_PATH);
- this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]);
- }
-
- public static async isParserAvailable(): Promise<boolean> {
- if (RustPDFParser.instance) {
- return true;
- }
-
- try {
- await stat(PDF_PARSER_PATH);
- RustPDFParser.instance = new RustPDFParser();
- return true;
- } catch (_) {
- return false;
- }
- }
-
- public static async getInstance(): Promise<RustPDFParser> {
- if (!RustPDFParser.instance) {
- try {
- await stat(PDF_PARSER_PATH);
- } catch (_) {
- throw Error("Rust pdf-parser shared library not found");
- }
- RustPDFParser.instance = new RustPDFParser();
- }
- return RustPDFParser.instance;
- }
-
- public async getPDFMetadata(path: string): Promise<PDFMetadata> {
- return new Promise<PDFMetadata>((resolve, reject) => {
- this._getPDFMetadata.async(path, (err: Error, res: string) => {
- if (err) {
- reject(err);
- } else {
- if (res.startsWith("RUSTFC:ERROR:")) {
- reject(new Error(res.replace("RUSTFC:ERROR:", "")));
- } else {
- try {
- const metadata = JSON.parse(res) as PDFMetadata;
- resolve(metadata);
- } catch (e) {
- reject(new Error("Failed to parse PDF metadata."));
- }
- }
- }
- });
- });
- }
-}
-
-export async function getPDFMetadata(path: string): Promise<PDFMetadata> {
- const converter = await RustPDFParser.getInstance();
- return await converter.getPDFMetadata(path);
-}
diff --git a/apps/api/src/natives.ts b/apps/api/src/natives.ts
index 560a14cb7..c12e3b8af 100644
--- a/apps/api/src/natives.ts
+++ b/apps/api/src/natives.ts
@@ -18,21 +18,6 @@ function createNativePath(subPath: string, filename: string): string {
return join(process.cwd(), "sharedLibs", subPath, fullFilename);
}
-export const HTML_TRANSFORMER_PATH = createNativePath(
- "html-transformer/target/release",
- "html_transformer",
-);
-
-export const PDF_PARSER_PATH = createNativePath(
- "pdf-parser/target/release",
- "pdf_parser",
-);
-
-export const CRAWLER_PATH = createNativePath(
- "crawler/target/release",
- "crawler",
-);
-
export const HTML_TO_MARKDOWN_PATH = createNativePath(
"go-html-to-md",
"html-to-markdown",
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 0f060e5ca..be91be585 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -7,8 +7,7 @@ import psl from "psl";
import { getURLDepth } from "./utils/maxDepthUtils";
import { logger as _logger } from "../../lib/logger";
import { redisEvictConnection } from "../../services/redis";
-import { extractLinks } from "../../lib/html-transformer";
-import { filterLinks } from "../../lib/crawler";
+import { extractLinks } from "@mendable/firecrawl-rs";
import {
fetchRobotsTxt,
createRobotsChecker,
@@ -17,6 +16,7 @@ import {
import { ScrapeJobTimeoutError } from "../../lib/error";
import { Logger } from "winston";
import { ScrapeOptions } from "../../controllers/v2/types";
+import { filterLinks } from "@mendable/firecrawl-rs";
export const SITEMAP_LIMIT = 100;
@@ -167,19 +167,19 @@ export class WebCrawler {
const res = await filterLinks({
links: sitemapLinks,
limit: isFinite(limit) ? limit : undefined,
- max_depth: maxDepth,
- base_url: this.baseUrl,
- initial_url: this.initialUrl,
- regex_on_full_url: this.regexOnFullURL,
+ maxDepth: maxDepth,
+ baseUrl: this.baseUrl,
+ initialUrl: this.initialUrl,
+ regexOnFullUrl: this.regexOnFullURL,
excludes: this.excludes,
includes: this.includes,
- allow_backward_crawling: this.allowBackwardCrawling,
- ignore_robots_txt: this.ignoreRobotsTxt,
- robots_txt: this.robotsTxt,
+ allowBackwardCrawling: this.allowBackwardCrawling,
+ ignoreRobotsTxt: this.ignoreRobotsTxt,
+ robotsTxt: this.robotsTxt,
});
const fancyDenialReasons = new Map<string, string>();
- res.denial_reasons.forEach((value, key) => {
+ Object.entries(res.denialReasons).forEach(([key, value]) => {
fancyDenialReasons.set(key, DenialReason[value]);
});
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 4616df85b..27a02a609 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -4,10 +4,10 @@ import { scrapeURL } from "../scrapeURL";
import { scrapeOptions } from "../../controllers/v2/types";
import type { Logger } from "winston";
import { CostTracking } from "../../lib/cost-tracking";
-import { parseSitemapXml, processSitemap } from "../../lib/crawler";
import { ScrapeJobTimeoutError } from "../../lib/error";
import type { ScrapeOptions } from "../../controllers/v2/types";
import { Engine } from "../scrapeURL/engines";
+import { parseSitemapXml, processSitemap } from "@mendable/firecrawl-rs";
const useFireEngine =
process.env.FIRE_ENGINE_BETA_URL !== "" &&
process.env.FIRE_ENGINE_BETA_URL !== undefined;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 899ee2c6a..b764a5b50 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -27,7 +27,7 @@ import * as Sentry from "@sentry/node";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import { fireEngineDelete } from "./delete";
import { MockState } from "../../lib/mock";
-import { getInnerJSON } from "../../../../lib/html-transformer";
+import { getInnerJson } from "@mendable/firecrawl-rs";
import { hasFormatOfType } from "../../../../lib/format-utils";
import { Action } from "../../../../controllers/v1/types";
import { AbortManagerThrownError } from "../../lib/abortManager";
@@ -161,7 +161,7 @@ async function performFireEngineScrape<
) ?? [])[1] ?? "";
if (contentType.includes("application/json")) {
- status.content = await getInnerJSON(status.content);
+ status.content = await getInnerJson(status.content);
}
if (status.file) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 6c7d4c9d3..24028270d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -16,8 +16,8 @@ import {
import { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";
import { useIndex } from "../../../services";
import { hasFormatOfType } from "../../../lib/format-utils";
-import { PDFMetadata } from "../../../lib/pdf-parser";
import { getPDFMaxPages } from "../../../controllers/v2/types";
+import { PdfMetadata } from "@mendable/firecrawl-rs";
export type Engine =
| "fire-engine;chrome-cdp"
@@ -119,7 +119,7 @@ export type EngineScrapeResult = {
pdfs: string[];
};
- pdfMetadata?: PDFMetadata;
+ pdfMetadata?: PdfMetadata;
cacheInfo?: {
created_at: Date;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 4399ba3c2..c34b5d6fa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -16,7 +16,6 @@ import {
import { readFile, unlink } from "node:fs/promises";
import path from "node:path";
import type { Response } from "undici";
-import { getPDFMetadata } from "../../../../lib/pdf-parser";
import {
getPdfResultFromCache,
savePdfResultToCache,
@@ -26,6 +25,7 @@ import {
shouldParsePDF,
getPDFMaxPages,
} from "../../../../controllers/v2/types";
+import { getPdfMetadata } from "@mendable/firecrawl-rs";
type PDFProcessorResult = { html: string; markdown?: string };
@@ -240,7 +240,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
}
}
- const pdfMetadata = await getPDFMetadata(tempFilePath);
+ const pdfMetadata = await getPdfMetadata(tempFilePath);
const effectivePageCount = maxPages
? Math.min(pdfMetadata.numPages, maxPages)
: pdfMetadata.numPages;
diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
index 92187307b..ae2bfec97 100644
--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
@@ -2,7 +2,7 @@ import { z } from "zod";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
import { robustFetch } from "../../lib/fetch";
-import { getInnerJSON } from "../../../../lib/html-transformer";
+import { getInnerJson } from "@mendable/firecrawl-rs";
export async function scrapeURLWithPlaywright(
meta: Meta,
@@ -32,7 +32,7 @@ export async function scrapeURLWithPlaywright(
});
if (response.contentType?.includes("application/json")) {
- response.content = await getInnerJSON(response.content);
+ response.content = await getInnerJson(response.content);
}
return {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 688708a06..2be357363 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -64,7 +64,6 @@ import {
AbortManagerThrownError,
} from "./lib/abortManager";
import { ScrapeJobTimeoutError } from "../../lib/error";
-import { transformHtml } from "../../lib/html-transformer";
import { htmlTransform } from "./lib/removeUnwantedElements";
export type ScrapeUrlResponse =
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts b/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts
index 51fbe00ec..3b6d75e89 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts
@@ -1,6 +1,6 @@
import { load } from "cheerio";
import { logger } from "../../../lib/logger";
-import { extractAttributesRust } from "../../../lib/html-transformer";
+import { extractAttributes as _extractAttributes } from "@mendable/firecrawl-rs";
export type AttributeResult = {
selector: string;
@@ -29,7 +29,7 @@ export async function extractAttributes(
// Try Rust implementation first (faster, non-blocking)
try {
- const results = await extractAttributesRust(html, selectors);
+ const results = await _extractAttributes(html, { selectors });
logger.debug("Attribute extraction via Rust", {
selectorsCount: selectors.length,
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractImages.ts b/apps/api/src/scraper/scrapeURL/lib/extractImages.ts
index df87bfd28..494a989f7 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractImages.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractImages.ts
@@ -1,6 +1,6 @@
import { load } from "cheerio";
import { logger } from "../../../lib/logger";
-import { extractImages as _extractImages } from "../../../lib/html-transformer";
+import { extractImages as _extractImages } from "@mendable/firecrawl-rs";
function resolveImageUrl(
src: string,
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts b/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts
index e55731840..7aecd12de 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts
@@ -4,7 +4,7 @@ import { logger } from "../../../lib/logger";
import {
extractLinks as _extractLinks,
extractBaseHref as _extractBaseHref,
-} from "../../../lib/html-transformer";
+} from "@mendable/firecrawl-rs";
function resolveUrlWithBaseHref(
href: string,
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
index 515af3c76..9a891122e 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts
@@ -1,7 +1,7 @@
import { load } from "cheerio"; // rustified
import { Document } from "../../../controllers/v1/types";
import { Meta } from "..";
-import { extractMetadata as _extractMetadata } from "../../../lib/html-transformer";
+import { extractMetadata as _extractMetadata } from "@mendable/firecrawl-rs";
export async function extractMetadataRust(
meta: Meta,
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
index d966228ea..68dcfc6cd 100644
--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -2,7 +2,7 @@
import { AnyNode, Cheerio, load } from "cheerio"; // rustified
import { ScrapeOptions } from "../../../controllers/v2/types";
-import { transformHtml } from "../../../lib/html-transformer";
+import { transformHtml } from "@mendable/firecrawl-rs";
import { logger } from "../../../lib/logger";
import { queryOMCESignatures } from "../../../services/index";
@@ -91,14 +91,14 @@ export const htmlTransform = async (
return await transformHtml({
html,
url,
- include_tags: (scrapeOptions.includeTags ?? [])
+ includeTags: (scrapeOptions.includeTags ?? [])
.map(x => x.trim())
.filter(x => x.length !== 0),
- exclude_tags: (scrapeOptions.excludeTags ?? [])
+ excludeTags: (scrapeOptions.excludeTags ?? [])
.map(x => x.trim())
.filter(x => x.length !== 0),
- only_main_content: scrapeOptions.onlyMainContent,
- omce_signatures: omce_signatures,
+ onlyMainContent: scrapeOptions.onlyMainContent,
+ omceSignatures: omce_signatures,
});
} catch (error) {
logger.warn("Failed to call html-transformer! Falling back to cheerio...", {
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 9e49d6558..ddaf50392 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -7,7 +7,7 @@ import { redisEvictConnection } from "./redis";
import type { Logger } from "winston";
import psl from "psl";
import { MapDocument } from "../controllers/v2/types";
-import { PDFMetadata } from "../lib/pdf-parser";
+import { PdfMetadata } from "@mendable/firecrawl-rs";
import { storage } from "../lib/gcs-jobs";
configDotenv();
@@ -148,7 +148,7 @@ export async function saveIndexToGCS(
statusCode: number;
error?: string;
screenshot?: string;
- pdfMetadata?: PDFMetadata;
+ pdfMetadata?: PdfMetadata;
contentType?: string;
},
): Promise<void> {