Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Convert all Rust natives to a single library using NAPI (ENG-3397) (#2105)
tom7569cd3a67a1
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.ymlindex 18730fa4b..9c97c8f00 100644--- a/.github/workflows/test-server-self-host.yml+++ b/.github/workflows/test-server-self-host.yml@@ -19,11 +19,11 @@ jobs:name: Run testsstrategy:matrix:- ai: ["openai", "no-ai"]- search: ["searxng", "google"]- engine: ["playwright", "fetch"]+ ai: ['openai', 'no-ai']+ search: ['searxng', 'google']+ engine: ['playwright', 'fetch']# proxy: ["proxy", "no-proxy"]- proxy: ["proxy"]+ proxy: ['proxy']fail-fast: falseruns-on: ubuntu-latestservices:@@ -47,8 +47,8 @@ jobs:- name: Set up Node.jsuses: actions/setup-node@v4with:- node-version: "20"- cache: "pnpm"+ node-version: '20'+ cache: 'pnpm'cache-dependency-path: './apps/api/pnpm-lock.yaml'- name: Install dependenciesrun: pnpm install@@ -65,29 +65,12 @@ jobs:with:go-version: '1.24'cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum- - name: Set up Rust- uses: actions-rust-lang/setup-rust-toolchain@v1- name: Build go-html-to-mdrun: |go mod tidygo build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.gochmod +x libhtml-to-markdown.soworking-directory: ./apps/api/sharedLibs/go-html-to-md- - name: Build html-transformer- run: |- cargo build --release- chmod +x target/release/libhtml_transformer.so- working-directory: ./apps/api/sharedLibs/html-transformer- - name: Build pdf-parser- run: |- cargo build --release- chmod +x target/release/libpdf_parser.so- working-directory: ./apps/api/sharedLibs/pdf-parser- - name: Build crawler- run: |- cargo build --release- chmod +x target/release/libcrawler.so- working-directory: ./apps/api/sharedLibs/crawler- name: Set up SearXNGif: matrix.search == 'searxng'run: |@@ -98,28 +81,28 @@ jobs:formats: [html, json, csv]server:secret_key: 'fcsecret'" > searxng/settings.yml-+docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxngpnpx wait-on tcp:3434 -t 30sworking-directory: ./- name: Build the application- run: npm run build+ run: pnpm run buildworking-directory: ./apps/api- name: Start server- run: npm run start:production:nobuild > api.log 2>&1 &+ run: pnpm run start:production:nobuild > api.log 2>&1 &env:PORT: 3002HOST: 0.0.0.0working-directory: ./apps/api- name: Start worker- run: npm run worker:production > worker.log 2>&1 &+ run: pnpm run worker:production > worker.log 2>&1 &env:PORT: 3005HOST: 0.0.0.0working-directory: ./apps/api- name: Start playwrightif: matrix.engine == 'playwright'- run: npm run dev > playwright.log 2>&1 &+ run: pnpm run dev > playwright.log 2>&1 &working-directory: ./apps/playwright-service-tsenv:PORT: 3003diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 8c76cd08b..6278db813 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -71,8 +71,8 @@ jobs:- name: Set up Node.jsuses: actions/setup-node@v4with:- node-version: "20"- cache: "pnpm"+ node-version: '20'+ cache: 'pnpm'cache-dependency-path: './apps/api/pnpm-lock.yaml'- name: Install dependenciesrun: pnpm install@@ -82,48 +82,31 @@ jobs:with:go-version: '1.24'cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum- - name: Set up Rust- uses: actions-rust-lang/setup-rust-toolchain@v1- name: Build go-html-to-mdrun: |go mod tidygo build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.gochmod +x libhtml-to-markdown.soworking-directory: ./apps/api/sharedLibs/go-html-to-md- - name: Build html-transformer- run: |- cargo build --release- chmod +x target/release/libhtml_transformer.so- working-directory: ./apps/api/sharedLibs/html-transformer- - name: Build pdf-parser- run: |- cargo build --release- chmod +x target/release/libpdf_parser.so- working-directory: ./apps/api/sharedLibs/pdf-parser- - name: Build crawler- run: |- cargo build --release- chmod +x target/release/libcrawler.so- working-directory: ./apps/api/sharedLibs/crawler- name: Build the application- run: npm run build+ run: pnpm run buildworking-directory: ./apps/api- name: Start the application- run: npm run start:production:nobuild > api.log 2>&1 &+ run: pnpm run start:production:nobuild > api.log 2>&1 &env:PORT: 3002HOST: 0.0.0.0working-directory: ./apps/apiid: start_app- name: Start worker- run: npm run worker:production > worker.log 2>&1 &+ run: pnpm run worker:production > worker.log 2>&1 &env:PORT: 3005HOST: 0.0.0.0working-directory: ./apps/apiid: start_workers- name: Start index worker- run: npm run index-worker:production > index-worker.log 2>&1 &+ run: pnpm run index-worker:production > index-worker.log 2>&1 &working-directory: ./apps/apiid: start_index_worker- name: Wait for API@@ -156,4 +139,4 @@ jobs:with:name: Encrypted Logspath: logs/logs.zip.gpg- retention-days: 5\ No newline at end of file+ retention-days: 5diff --git a/apps/api/.dockerignore b/apps/api/.dockerignoreindex 679a66475..0f38d9cc0 100644--- a/apps/api/.dockerignore+++ b/apps/api/.dockerignore@@ -1,4 +1,5 @@-/node_modules/-/dist/+**/node_modules/+**/dist/+**/target/.env*.csvdiff --git a/apps/api/Dockerfile b/apps/api/Dockerfileindex 544e06385..bf96a1615 100644--- a/apps/api/Dockerfile+++ b/apps/api/Dockerfile@@ -1,64 +1,77 @@+# syntax=docker/dockerfile:1FROM node:22-slim AS baseENV PNPM_HOME="/pnpm"ENV PATH="$PNPM_HOME:$PATH"+ENV CI=trueRUN corepack enable-COPY . /app++# Build Go shared library+FROM golang:1.24 AS go-buildWORKDIR /app+COPY sharedLibs/go-html-to-md ./sharedLibs/go-html-to-md-FROM base AS prod-deps-RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile --ignore-scripts+RUN cd sharedLibs/go-html-to-md && \+ go mod download && \+ go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.goFROM base AS build-RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install && pnpm run build--# Install Go-FROM golang:1.24 AS go-base-COPY sharedLibs/go-html-to-md /app/sharedLibs/go-html-to-md+WORKDIR /app-# Install Go dependencies and build parser lib-RUN cd /app/sharedLibs/go-html-to-md && \- go mod download && \- go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go && \- chmod +x libhtml-to-markdown.so+# Install system dependencies+RUN apt-get update && apt-get install -y \+ curl \+ build-essential \+ pkg-config \+ && rm -rf /var/lib/apt/lists/*# Install Rust-FROM rust:1-slim AS rust-base-COPY sharedLibs/html-transformer /app/sharedLibs/html-transformer-COPY sharedLibs/pdf-parser /app/sharedLibs/pdf-parser-COPY sharedLibs/crawler /app/sharedLibs/crawler--# Install Rust dependencies and build transformer lib-RUN cd /app/sharedLibs/html-transformer && \- cargo build --release --locked && \- chmod +x target/release/libhtml_transformer.so--# Install Rust dependencies and build PDF parser lib-RUN cd /app/sharedLibs/pdf-parser && \- cargo build --release --locked && \- chmod +x target/release/libpdf_parser.so--# Install Rust dependencies and build crawler lib-RUN cd /app/sharedLibs/crawler && \- cargo build --release --locked && \- chmod +x target/release/libcrawler.so--FROM base-COPY --from=build /app/dist /app/dist-COPY --from=prod-deps /app/node_modules /app/node_modules-COPY --from=go-base /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so-COPY --from=rust-base /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so-COPY --from=rust-base /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so-COPY --from=rust-base /app/sharedLibs/crawler/target/release/libcrawler.so /app/sharedLibs/crawler/target/release/libcrawler.so--# Install git-RUN apt-get update && apt-get install -y git procps && rm -rf /var/lib/apt/lists/*--# Start the server by default, this can be overwritten at runtime+ENV RUSTUP_HOME=/usr/local/rustup \+ CARGO_HOME=/usr/local/cargo \+ PATH=/usr/local/cargo/bin:$PATH++RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path \+ && chmod -R a+w $RUSTUP_HOME $CARGO_HOME++# Copy source files+COPY pnpm-lock.yaml pnpm-workspace.yaml package.json ./+COPY . .++# Install dependencies+RUN --mount=type=cache,id=pnpm,target=/pnpm/store \+ --mount=type=cache,target=/usr/local/cargo/registry \+ --mount=type=cache,target=/app/native/target \+ pnpm install --frozen-lockfile++# Build the application+RUN pnpm run build++# Remove dev dependencies+RUN pnpm prune --prod --ignore-scripts++# Runtime stage+FROM base AS runtime++# Install runtime dependencies+RUN apt-get update && apt-get install -y \+ git \+ procps \+ && rm -rf /var/lib/apt/lists/*+EXPOSE 8080+WORKDIR /app++# Copy built application+COPY --from=build /app/node_modules ./node_modules+COPY --from=build /app/dist ./dist+COPY --from=build /app/native ./native++# Copy Go shared library+COPY --from=go-build /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so ./sharedLibs/go-html-to-md/-# Make sure the entrypoint script has the correct line endings-RUN sed -i 's/\r$//' /app/docker-entrypoint.sh+# Copy and prepare entrypoint+COPY docker-entrypoint.sh ./+RUN sed -i 's/\r$//' ./docker-entrypoint.sh && chmod +x ./docker-entrypoint.sh-ENTRYPOINT "/app/docker-entrypoint.sh"+ENTRYPOINT ["./docker-entrypoint.sh"]\ No newline at end of filediff --git a/apps/api/native/.cargo/config.toml b/apps/api/native/.cargo/config.tomlnew file mode 100644index 000000000..ac2b23f81--- /dev/null+++ b/apps/api/native/.cargo/config.toml@@ -0,0 +1,2 @@+[target.x86_64-pc-windows-msvc]+rustflags = ["-C", "target-feature=+crt-static"]diff --git a/apps/api/native/.editorconfig b/apps/api/native/.editorconfignew file mode 100644index 000000000..182fda23a--- /dev/null+++ b/apps/api/native/.editorconfig@@ -0,0 +1,15 @@+# EditorConfig helps developers define and maintain consistent+# coding styles between different editors or IDEs+# http://editorconfig.org+root = true++[*]+indent_style = space+indent_size = 2+end_of_line = lf+charset = utf-8+trim_trailing_whitespace = true+insert_final_newline = true++[*.md]+trim_trailing_whitespace = falsediff --git a/apps/api/native/.gitattributes b/apps/api/native/.gitattributesnew file mode 100644index 000000000..e310f6263--- /dev/null+++ b/apps/api/native/.gitattributes@@ -0,0 +1,18 @@+# Auto detect text files and perform LF normalization+* text=auto+++*.ts text eol=lf merge=union+*.tsx text eol=lf merge=union+*.rs text eol=lf merge=union+*.js text eol=lf merge=union+*.json text eol=lf merge=union+*.debug text eol=lf merge=union++# Generated codes+index.js linguist-detectable=false+index.d.ts linguist-detectable=false+firecrawl-rs.wasi-browser.js linguist-detectable=false+firecrawl-rs.wasi.cjs linguist-detectable=false+wasi-worker-browser.mjs linguist-detectable=false+wasi-worker.mjs linguist-detectable=falsediff --git a/apps/api/native/.gitignore b/apps/api/native/.gitignorenew file mode 100644index 000000000..cf7f86697--- /dev/null+++ b/apps/api/native/.gitignore@@ -0,0 +1,120 @@+### Node ###+# Logs+logs+*.log+npm-debug.log*+yarn-debug.log*+yarn-error.log*+lerna-debug.log*++# Diagnostic reports (https://nodejs.org/api/report.html)+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json++# Runtime data+pids+*.pid+*.seed+*.pid.lock++# Directory for instrumented libs generated by jscoverage/JSCover+lib-cov++# Coverage directory used by tools like istanbul+coverage+*.lcov++# nyc test coverage+.nyc_output++# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)+.grunt++# Bower dependency directory (https://bower.io/)+bower_components++# node-waf configuration+.lock-wscript++# Compiled binary addons (https://nodejs.org/api/addons.html)+build/Release++# Dependency directories+node_modules/+jspm_packages/++# TypeScript v1 declaration files+typings/++# TypeScript cache+*.tsbuildinfo++# Optional npm cache directory+.npm++# Optional eslint cache+.eslintcache++# Microbundle cache+.rpt2_cache/+.rts2_cache_cjs/+.rts2_cache_es/+.rts2_cache_umd/++# Optional REPL history+.node_repl_history++# Output of 'npm pack'+*.tgz++# Yarn Integrity file+.yarn-integrity++# dotenv environment variables file+.env+.env.test++# parcel-bundler cache (https://parceljs.org/)+.cache++# Next.js build output+.next++# Nuxt.js build / generate output+.nuxt+dist++# Gatsby files+.cache/+# Comment in the public line in if your project uses Gatsby and not Next.js+# https://nextjs.org/blog/next-9-1#public-directory-support+# public++# vuepress build output+.vuepress/dist++# Serverless directories+.serverless/++# FuseBox cache+.fusebox/++# DynamoDB Local files+.dynamodb/++# TernJS port file+.tern-port++# Stores VSCode versions used for testing VSCode extensions+.vscode-test++# Rust+/target+Cargo.lock++*.node+.pnp.*+/npm++index.js+index.d.ts+browser.jsdiff --git a/apps/api/native/.prettierignore b/apps/api/native/.prettierignorenew file mode 100644index 000000000..901e6b62c--- /dev/null+++ b/apps/api/native/.prettierignore@@ -0,0 +1,8 @@+target+.yarn+index.js+package-template.wasi-browser.js+package-template.wasi.cjs+wasi-worker-browser.mjs+wasi-worker.mjs+.yarnrc.yml\ No newline at end of filediff --git a/apps/api/native/.taplo.toml b/apps/api/native/.taplo.tomlnew file mode 100644index 000000000..b2d27a70b--- /dev/null+++ b/apps/api/native/.taplo.toml@@ -0,0 +1,7 @@+exclude = ["node_modules/**/*.toml"]++# https://taplo.tamasfe.dev/configuration/formatter-options.html+[formatting]+align_entries = true+indent_tables = true+reorder_keys = truediff --git a/apps/api/native/.yarnrc.yml b/apps/api/native/.yarnrc.ymlnew file mode 100644index 000000000..52d58b687--- /dev/null+++ b/apps/api/native/.yarnrc.yml@@ -0,0 +1,5 @@+nodeLinker: node-modules++npmAuditRegistry: "https://registry.npmjs.org"++yarnPath: .yarn/releases/yarn-4.9.4.cjsdiff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlnew file mode 100644index 000000000..e08132429--- /dev/null+++ b/apps/api/native/Cargo.toml@@ -0,0 +1,30 @@++[package]+authors = ["Firecrawl"]+edition = "2021"+name = "firecrawl_rs"+version = "0.1.0"++[lib]+crate-type = ["cdylib"]++[dependencies]+kuchikiki = "0.8.2"+lol_html = "2.6.0"+lopdf = "0.38.0"+napi = { version = "3.0.0", features = ["serde-json"] }+napi-derive = "3.0.0"+nodesig = { git = "https://github.com/firecrawl/nodesig" }+regex = "1.11.2"+roxmltree = "0.20.0"+serde = { version = "1.0.219", features = ["derive"] }+serde_json = "1.0.143"+texting_robots = "0.2.2"+url = "2.5.7"++[build-dependencies]+napi-build = "2"++[profile.release]+lto = true+strip = "symbols"diff --git a/apps/api/native/README.md b/apps/api/native/README.mdnew file mode 100644index 000000000..92523e06a--- /dev/null+++ b/apps/api/native/README.md@@ -0,0 +1,87 @@+# `@napi-rs/package-template`++++> Template project for writing node packages with napi-rs.++# Usage++1. Click **Use this template**.+2. **Clone** your project.+3. Run `yarn install` to install dependencies.+4. Run `yarn napi rename -n [@your-scope/package-name] -b [binary-name]` command under the project folder to rename your package.++## Install this test package++```bash+yarn add @napi-rs/package-template+```++## Ability++### Build++After `yarn build/npm run build` command, you can see `package-template.[darwin|win32|linux].node` file in project root. This is the native addon built from [lib.rs](./src/lib.rs).++### Test++With [ava](https://github.com/avajs/ava), run `yarn test/npm run test` to testing native addon. You can also switch to another testing framework if you want.++### CI++With GitHub Actions, each commit and pull request will be built and tested automatically in [`node@20`, `@node22`] x [`macOS`, `Linux`, `Windows`] matrix. You will never be afraid of the native addon broken in these platforms.++### Release++Release native package is very difficult in old days. Native packages may ask developers who use it to install `build toolchain` like `gcc/llvm`, `node-gyp` or something more.++With `GitHub actions`, we can easily prebuild a `binary` for major platforms. And with `N-API`, we should never be afraid of **ABI Compatible**.++The other problem is how to deliver prebuild `binary` to users. Downloading it in `postinstall` script is a common way that most packages do it right now. The problem with this solution is it introduced many other packages to download binary that has not been used by `runtime codes`. The other problem is some users may not easily download the binary from `GitHub/CDN` if they are behind a private network (But in most cases, they have a private NPM mirror).++In this package, we choose a better way to solve this problem. We release different `npm packages` for different platforms. And add it to `optionalDependencies` before releasing the `Major` package to npm.++`NPM` will choose which native package should download from `registry` automatically. You can see [npm](./npm) dir for details. And you can also run `yarn add @napi-rs/package-template` to see how it works.++## Develop requirements++- Install the latest `Rust`+- Install `Node.js@10+` which fully supported `Node-API`+- Install `[email protected]`++## Test in local++- yarn+- yarn build+- yarn test++And you will see:++```bash+$ ava --verbose++ ✔ sync function from native code+ ✔ sleep function from native code (201ms)+ ─++ 2 tests passed+✨ Done in 1.12s.+```++## Release package++Ensure you have set your **NPM_TOKEN** in the `GitHub` project setting.++In `Settings -> Secrets`, add **NPM_TOKEN** into it.++When you want to release the package:++```bash+npm version [<newversion> | major | minor | patch | premajor | preminor | prepatch | prerelease [--preid=<prerelease-id>] | from-git]++git push+```++GitHub actions will do the rest job for you.++> WARN: Don't run `npm publish` manually.diff --git a/apps/api/native/build.rs b/apps/api/native/build.rsnew file mode 100644index 000000000..bbfc9e4b9--- /dev/null+++ b/apps/api/native/build.rs@@ -0,0 +1,3 @@+fn main() {+ napi_build::setup();+}diff --git a/apps/api/native/package.json b/apps/api/native/package.jsonnew file mode 100644index 000000000..99b35f0bc--- /dev/null+++ b/apps/api/native/package.json@@ -0,0 +1,74 @@+{+ "name": "@mendable/firecrawl-rs",+ "version": "1.0.0",+ "description": "Template project for writing node package with napi-rs",+ "main": "index.js",+ "repository": {+ "type": "git",+ "url": "git+ssh://[email protected]/napi-rs/package-template.git"+ },+ "license": "AGPL",+ "browser": "browser.js",+ "keywords": [+ "napi-rs",+ "NAPI",+ "N-API",+ "Rust",+ "node-addon",+ "node-addon-api"+ ],+ "files": [+ "index.d.ts",+ "index.js",+ "browser.js"+ ],+ "napi": {+ "binaryName": "firecrawl-rs",+ "targets": [+ "x86_64-pc-windows-msvc",+ "x86_64-apple-darwin",+ "x86_64-unknown-linux-gnu",+ "aarch64-apple-darwin"+ ]+ },+ "engines": {+ "node": ">= 6.14.2 < 7 || >= 8.11.2 < 9 || >= 9.11.0 < 10 || >= 10.0.0"+ },+ "publishConfig": {+ "registry": "https://registry.npmjs.org/",+ "access": "public"+ },+ "scripts": {+ "artifacts": "napi artifacts",+ "build": "napi build --platform --release",+ "build:debug": "napi build --platform",+ "format": "run-p format:prettier format:rs format:toml",+ "format:prettier": "prettier . -w",+ "format:toml": "taplo format",+ "format:rs": "cargo fmt",+ "lint": "oxlint .",+ "prepublishOnly": "napi prepublish -t npm",+ "version": "napi version",+ "install": "pnpm build"+ },+ "devDependencies": {+ "@emnapi/core": "^1.5.0",+ "@emnapi/runtime": "^1.5.0",+ "@napi-rs/cli": "^3.1.5",+ "@oxc-node/core": "^0.0.32",+ "@taplo/cli": "^0.7.0",+ "@tybys/wasm-util": "^0.10.0",+ "chalk": "^5.6.0",+ "npm-run-all2": "^8.0.4",+ "oxlint": "^1.14.0",+ "prettier": "^3.6.2",+ "typescript": "^5.9.2"+ },+ "prettier": {+ "printWidth": 120,+ "semi": false,+ "trailingComma": "all",+ "singleQuote": true,+ "arrowParens": "always"+ }+}diff --git a/apps/api/native/rustfmt.toml b/apps/api/native/rustfmt.tomlnew file mode 100644index 000000000..b196eaa2d--- /dev/null+++ b/apps/api/native/rustfmt.toml@@ -0,0 +1 @@+tab_spaces = 2diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsnew file mode 100644index 000000000..8a4c7aa66--- /dev/null+++ b/apps/api/native/src/crawler.rs@@ -0,0 +1,640 @@+use napi::bindgen_prelude::*;+use napi_derive::napi;+use regex::Regex;+use serde::{Deserialize, Serialize};+use std::{collections::HashMap, usize};+use texting_robots::Robot;+use url::Url;++#[derive(Deserialize)]+#[napi(object)]+pub struct FilterLinksCall {+ pub links: Vec<String>,+ pub limit: Option<i64>,+ pub max_depth: u32,+ pub base_url: String,+ pub initial_url: String,+ pub regex_on_full_url: bool,+ pub excludes: Vec<String>,+ pub includes: Vec<String>,+ pub allow_backward_crawling: bool,+ pub ignore_robots_txt: bool,+ pub robots_txt: String,+}++#[derive(Serialize)]+#[napi(object)]+pub struct FilterLinksResult {+ pub links: Vec<String>,+ pub denial_reasons: HashMap<String, String>,+}++#[derive(Serialize, Debug)]+#[napi(object)]+pub struct SitemapUrl {+ pub loc: Vec<String>,+}++#[derive(Serialize, Debug)]+#[napi(object)]+pub struct SitemapEntry {+ pub loc: Vec<String>,+}++#[derive(Serialize, Debug)]+#[napi(object)]+pub struct SitemapUrlset {+ pub url: Vec<SitemapUrl>,+}++#[derive(Serialize, Debug)]+#[napi(object)]+pub struct SitemapIndex {+ pub sitemap: Vec<SitemapEntry>,+}++#[derive(Serialize, Debug)]+#[napi(object)]+pub struct ParsedSitemap {+ pub urlset: Option<SitemapUrlset>,+ pub sitemapindex: Option<SitemapIndex>,+}++#[derive(Serialize, Debug)]+#[napi(object)]+pub struct SitemapInstruction {+ pub action: String,+ pub urls: Vec<String>,+ pub count: u32,+}++#[derive(Serialize, Debug)]+#[napi(object)]+pub struct SitemapProcessingResult {+ pub instructions: Vec<SitemapInstruction>,+ pub total_count: u32,+}++fn _is_file(url: &Url) -> bool {+ let file_extensions = vec![+ ".png", ".jpg", ".jpeg", ".gif", ".css", ".js", ".ico", ".svg", ".tiff", ".zip", ".exe",+ ".dmg", ".mp4", ".mp3", ".wav", ".pptx", ".xlsx", ".avi", ".flv", ".woff", ".ttf", ".woff2",+ ".webp", ".inc",+ ];+ let url_without_query = url.path().to_lowercase();+ file_extensions+ .iter()+ .any(|ext| url_without_query.ends_with(ext))+}++fn _get_url_depth(path: &str) -> u32 {+ path+ .split('/')+ .filter(|x| *x != "" && *x != "index.php" && *x != "index.html")+ .count() as u32+}++fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult, String> {+ let mut denial_reasons = HashMap::new();++ let limit = data+ .limit+ .and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) })+ .unwrap_or(usize::MAX);++ if limit == 0 {+ return Ok(FilterLinksResult {+ links: Vec::with_capacity(0),+ denial_reasons,+ });+ }++ let base_url = Url::parse(&data.base_url).map_err(|e| format!("Base URL parse error: {}", e))?;+ let initial_url = Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {}", e))?;++ let excludes_regex = data+ .excludes+ .iter()+ .map(|exclude| Regex::new(exclude))+ .collect::<Vec<std::result::Result<Regex, regex::Error>>>();+ let excludes_regex = excludes_regex+ .into_iter()+ .filter_map(|x| x.ok())+ .collect::<Vec<Regex>>();++ let includes_regex = data+ .includes+ .iter()+ .map(|include| Regex::new(include))+ .collect::<Vec<std::result::Result<Regex, regex::Error>>>();+ let includes_regex = includes_regex+ .into_iter()+ .filter_map(|x| x.ok())+ .collect::<Vec<Regex>>();++ let links = data+ .links+ .into_iter()+ .filter(|link| {+ let url = match base_url.join(link) {+ Ok(x) => x,+ Err(_) => {+ denial_reasons.insert(link.clone(), "URL_PARSE_ERROR".to_string());+ return false;+ }+ };++ let path = url.path();+ let depth = _get_url_depth(path);+ if depth > data.max_depth {+ denial_reasons.insert(link.clone(), "DEPTH_LIMIT".to_string());+ return false;+ }++ let exinc_path = if data.regex_on_full_url {+ url.as_str()+ } else {+ url.path()+ };++ if !excludes_regex.is_empty()+ && excludes_regex+ .iter()+ .any(|regex| regex.is_match(exinc_path))+ {+ denial_reasons.insert(link.clone(), "EXCLUDE_PATTERN".to_string());+ return false;+ }++ if !includes_regex.is_empty()+ && !includes_regex+ .iter()+ .any(|regex| regex.is_match(exinc_path))+ {+ denial_reasons.insert(link.clone(), "INCLUDE_PATTERN".to_string());+ return false;+ }++ if !data.allow_backward_crawling {+ if !url.path().starts_with(initial_url.path()) {+ denial_reasons.insert(link.clone(), "BACKWARD_CRAWLING".to_string());+ return false;+ }+ }++ if !data.ignore_robots_txt {+ match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {+ Ok(robot) => {+ let allowed = robot.allowed(url.as_str());+ if !allowed {+ match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {+ Ok(robot_alt) => {+ let allowed_alt = robot_alt.allowed(url.as_str());+ if !allowed_alt {+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());+ return false;+ }+ }+ Err(_) => {+ denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());+ return false;+ }+ }+ }+ }+ Err(_) => {}+ }+ }++ if _is_file(&url) {+ denial_reasons.insert(link.clone(), "FILE_TYPE".to_string());+ return false;+ }++ true+ })+ .take(limit)+ .collect::<Vec<_>>();++ Ok(FilterLinksResult {+ links,+ denial_reasons,+ })+}++/// Filter links based on crawling rules and constraints.+#[napi]+pub fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {+ _filter_links(data)+ .map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {}", e)))+}++fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, String> {+ let doc = roxmltree::Document::parse_with_options(+ xml_content,+ roxmltree::ParsingOptions {+ allow_dtd: true,+ ..Default::default()+ },+ ).map_err(|e| format!("XML parsing error: {}", e))?;+ let root = doc.root_element();++ match root.tag_name().name() {+ "sitemapindex" => {+ let mut sitemaps = Vec::new();++ for sitemap_node in root+ .children()+ .filter(|n| n.is_element() && n.tag_name().name() == "sitemap")+ {+ if let Some(loc_node) = sitemap_node+ .children()+ .find(|n| n.is_element() && n.tag_name().name() == "loc")+ {+ if let Some(loc_text) = loc_node.text() {+ sitemaps.push(SitemapEntry {+ loc: vec![loc_text.to_string()],+ });+ }+ }+ }++ Ok(ParsedSitemap {+ urlset: None,+ sitemapindex: Some(SitemapIndex { sitemap: sitemaps }),+ })+ }+ "urlset" => {+ let mut urls = Vec::new();++ for url_node in root+ .children()+ .filter(|n| n.is_element() && n.tag_name().name() == "url")+ {+ if let Some(loc_node) = url_node+ .children()+ .find(|n| n.is_element() && n.tag_name().name() == "loc")+ {+ if let Some(loc_text) = loc_node.text() {+ urls.push(SitemapUrl {+ loc: vec![loc_text.to_string()],+ });+ }+ }+ }++ Ok(ParsedSitemap {+ urlset: Some(SitemapUrlset { url: urls }),+ sitemapindex: None,+ })+ }+ _ => Err("Invalid sitemap format: root element must be 'sitemapindex' or 'urlset'".to_string()),+ }+}++/// Parse XML sitemap content into structured data.+#[napi]+pub fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {+ _parse_sitemap_xml(&xml_content).map_err(|e| {+ Error::new(+ Status::GenericFailure,+ format!("Parse sitemap XML error: {}", e),+ )+ })+}++fn _process_sitemap(+ xml_content: &str,+) -> std::result::Result<SitemapProcessingResult, String> {+ let parsed = _parse_sitemap_xml(xml_content)?;+ let mut instructions = Vec::new();+ let mut total_count: u32 = 0;++ if let Some(sitemapindex) = parsed.sitemapindex {+ let sitemap_urls: Vec<String> = sitemapindex+ .sitemap+ .iter()+ .filter_map(|sitemap| {+ if !sitemap.loc.is_empty() {+ Some(sitemap.loc[0].trim().to_string())+ } else {+ None+ }+ })+ .collect();++ if !sitemap_urls.is_empty() {+ instructions.push(SitemapInstruction {+ action: "recurse".to_string(),+ urls: sitemap_urls.clone(),+ count: sitemap_urls.len() as u32,+ });+ total_count += sitemap_urls.len() as u32;+ }+ } else if let Some(urlset) = parsed.urlset {+ let mut xml_sitemaps = Vec::new();+ let mut valid_urls = Vec::new();++ for url_entry in urlset.url {+ if !url_entry.loc.is_empty() {+ let url = url_entry.loc[0].trim();+ if url.to_lowercase().ends_with(".xml") {+ xml_sitemaps.push(url.to_string());+ } else if let Ok(parsed_url) = Url::parse(url) {+ if !_is_file(&parsed_url) {+ valid_urls.push(url.to_string());+ }+ }+ }+ }++ if !xml_sitemaps.is_empty() {+ instructions.push(SitemapInstruction {+ action: "recurse".to_string(),+ urls: xml_sitemaps.clone(),+ count: xml_sitemaps.len() as u32,+ });+ total_count += xml_sitemaps.len() as u32;+ }++ if !valid_urls.is_empty() {+ instructions.push(SitemapInstruction {+ action: "process".to_string(),+ urls: valid_urls.clone(),+ count: valid_urls.len() as u32,+ });+ total_count += valid_urls.len() as u32;+ }+ }++ Ok(SitemapProcessingResult {+ instructions,+ total_count,+ })+}++/// Process sitemap XML and extract crawling instructions.+#[napi]+pub fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResult> {+ _process_sitemap(&xml_content).map_err(|e| {+ Error::new(+ Status::GenericFailure,+ format!("Process sitemap error: {}", e),+ )+ })+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_parse_sitemap_xml_urlset() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>https://example.com/page1</loc>+ </url>+ <url>+ <loc>https://example.com/page2</loc>+ </url>+</urlset>"#;++ let result = _parse_sitemap_xml(xml_content).unwrap();+ assert!(result.urlset.is_some());+ let urlset = result.urlset.unwrap();+ assert_eq!(urlset.url.len(), 2);+ assert_eq!(urlset.url[0].loc[0], "https://example.com/page1");+ assert_eq!(urlset.url[1].loc[0], "https://example.com/page2");+ }++ #[test]+ fn test_parse_sitemap_xml_sitemapindex() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <sitemap>+ <loc>https://example.com/sitemap1.xml</loc>+ </sitemap>+ <sitemap>+ <loc>https://example.com/sitemap2.xml</loc>+ </sitemap>+</sitemapindex>"#;++ let result = _parse_sitemap_xml(xml_content).unwrap();+ assert!(result.sitemapindex.is_some());+ let sitemapindex = result.sitemapindex.unwrap();+ assert_eq!(sitemapindex.sitemap.len(), 2);+ assert_eq!(+ sitemapindex.sitemap[0].loc[0],+ "https://example.com/sitemap1.xml"+ );+ assert_eq!(+ sitemapindex.sitemap[1].loc[0],+ "https://example.com/sitemap2.xml"+ );+ }++ #[test]+ fn test_parse_sitemap_xml_invalid_root() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<invalid xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>https://example.com/page1</loc>+ </url>+</invalid>"#;++ let result = _parse_sitemap_xml(xml_content);+ assert!(result.is_err());+ assert!(result+ .unwrap_err()+ .to_string()+ .contains("Invalid sitemap format"));+ }++ #[test]+ fn test_parse_sitemap_xml_malformed() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>https://example.com/page1</loc>+ </url>+</urlset"#; // Missing closing >++ let result = _parse_sitemap_xml(xml_content);+ assert!(result.is_err());+ }++ #[test]+ fn test_process_sitemap_urlset() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>https://example.com/page1</loc>+ </url>+ <url>+ <loc>https://example.com/sitemap2.xml</loc>+ </url>+ <url>+ <loc>https://example.com/image.png</loc>+ </url>+</urlset>"#;++ let result = _process_sitemap(xml_content).unwrap();+ assert_eq!(result.instructions.len(), 2);++ let recurse_instruction = result+ .instructions+ .iter()+ .find(|i| i.action == "recurse")+ .unwrap();+ assert_eq!(recurse_instruction.urls.len(), 1);+ assert_eq!(+ recurse_instruction.urls[0],+ "https://example.com/sitemap2.xml"+ );++ let process_instruction = result+ .instructions+ .iter()+ .find(|i| i.action == "process")+ .unwrap();+ assert_eq!(process_instruction.urls.len(), 1);+ assert_eq!(process_instruction.urls[0], "https://example.com/page1");+ }++ #[test]+ fn test_process_sitemap_sitemapindex() {+ let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <sitemap>+ <loc>https://example.com/sitemap1.xml</loc>+ </sitemap>+ <sitemap>+ <loc>https://example.com/sitemap2.xml</loc>+ </sitemap>+</sitemapindex>"#;++ let result = _process_sitemap(xml_content).unwrap();+ assert_eq!(result.instructions.len(), 1);+ assert_eq!(result.instructions[0].action, "recurse");+ assert_eq!(result.instructions[0].urls.len(), 2);+ assert_eq!(+ result.instructions[0].urls[0],+ "https://example.com/sitemap1.xml"+ );+ assert_eq!(+ result.instructions[0].urls[1],+ "https://example.com/sitemap2.xml"+ );+ }++ #[test]+ fn test_filter_links_normal_robots_txt() {+ let data = FilterLinksCall {+ links: vec![+ "https://example.com/allowed".to_string(),+ "https://example.com/disallowed".to_string(),+ ],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: "User-agent: *\nDisallow: /disallowed".to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data).unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/allowed");+ assert!(result+ .denial_reasons+ .contains_key("https://example.com/disallowed"));+ assert_eq!(+ result+ .denial_reasons+ .get("https://example.com/disallowed")+ .unwrap(),+ "ROBOTS_TXT"+ );+ }++ #[test]+ fn test_filter_links_malformed_robots_txt() {+ let data = FilterLinksCall {+ links: vec!["https://example.com/test".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: "Invalid robots.txt content with \x00 null bytes and malformed syntax"+ .to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/test");+ }++ #[test]+ fn test_filter_links_non_utf8_robots_txt() {+ let mut non_utf8_bytes = vec![0xFF, 0xFE];+ non_utf8_bytes.extend_from_slice(b"User-agent: *\nDisallow: /blocked");+ let non_utf8_string = String::from_utf8_lossy(&non_utf8_bytes).to_string();++ let data = FilterLinksCall {+ links: vec!["https://example.com/allowed".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: non_utf8_string,+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/allowed");+ }++ #[test]+ fn test_filter_links_char_boundary_issue() {+ let problematic_content = "User-agent: *\nDisallow: /\u{a0}test";++ let data = FilterLinksCall {+ links: vec!["https://example.com/test".to_string()],+ limit: Some(10),+ includes: vec![],+ excludes: vec![],+ ignore_robots_txt: false,+ robots_txt: problematic_content.to_string(),+ max_depth: 10,+ base_url: "https://example.com".to_string(),+ initial_url: "https://example.com".to_string(),+ regex_on_full_url: false,+ allow_backward_crawling: true,+ };++ let result = _filter_links(data);+ assert!(result.is_ok());+ let result = result.unwrap();+ assert_eq!(result.links.len(), 1);+ assert_eq!(result.links[0], "https://example.com/test");+ }+}diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsnew file mode 100644index 000000000..3d2372c36--- /dev/null+++ b/apps/api/native/src/html.rs@@ -0,0 +1,810 @@+use std::collections::{HashMap, HashSet};++use kuchikiki::{iter::NodeEdge, parse_html, traits::TendrilSink, NodeRef};+use napi_derive::napi;+use nodesig::{get_node_signature, SignatureMode};+use serde::{Deserialize, Serialize};+use serde_json::Value;+use url::Url;++use crate::utils::to_napi_err;++fn _extract_base_href_from_document(+ document: &NodeRef,+ url: &Url,+) -> Result<String, Box<dyn std::error::Error>> {+ if let Some(base) = document+ .select("base[href]")+ .map_err(|_| "Failed to select base href".to_string())?+ .next()+ .and_then(|base| base.attributes.borrow().get("href").map(|x| x.to_string()))+ {+ if let Ok(base) = url.join(&base) {+ return Ok(base.to_string());+ }+ }++ Ok(url.to_string())+}++fn _extract_base_href(html: &str, url: &str) -> Result<String, Box<dyn std::error::Error>> {+ let document = parse_html().one(html);+ let url = Url::parse(url)?;+ _extract_base_href_from_document(&document, &url)+}++/// Extract the base href from HTML document.+#[napi]+pub fn extract_base_href(html: String, url: String) -> napi::Result<String> {+ _extract_base_href(&html, &url).map_err(to_napi_err)+}++/// Extract all links from HTML document.+#[napi]+pub fn extract_links(html: Option<String>) -> napi::Result<Vec<String>> {+ let html = match html {+ Some(h) => h,+ None => return Ok(Vec::new()),+ };++ let document = parse_html().one(html.as_str());++ let anchors: Vec<_> = document+ .select("a[href]")+ .map_err(|_| to_napi_err("Failed to select links"))?+ .collect();++ let mut out: Vec<String> = Vec::new();++ for anchor in anchors {+ let mut href = match anchor.attributes.borrow().get("href") {+ Some(x) => x.to_string(),+ None => continue,+ };++ if href.starts_with("http:/") && !href.starts_with("http://") {+ href = format!("http://{}", &href[6..]);+ } else if href.starts_with("https:/") && !href.starts_with("https://") {+ href = format!("https://{}", &href[7..]);+ }++ out.push(href);+ }++ Ok(out)+}++macro_rules! insert_meta_name {+ ($out:ident, $document:ident, $metaName:expr, $outName:expr) => {+ if let Some(x) = $document+ .select(&format!("meta[name=\"{}\"]", $metaName))+ .map_err(|_| "Failed to select meta name")?+ .next()+ .and_then(|description| {+ description+ .attributes+ .borrow()+ .get("content")+ .map(|x| x.to_string())+ })+ {+ $out.insert(($outName).to_string(), Value::String(x));+ }+ };+}++macro_rules! insert_meta_property {+ ($out:ident, $document:ident, $metaName:expr, $outName:expr) => {+ if let Some(x) = $document+ .select(&format!("meta[property=\"{}\"]", $metaName))+ .map_err(|_| "Failed to select meta property")?+ .next()+ .and_then(|description| {+ description+ .attributes+ .borrow()+ .get("content")+ .map(|x| x.to_string())+ })+ {+ $out.insert(($outName).to_string(), Value::String(x));+ }+ };+}++fn _extract_metadata(html: &str) -> Result<HashMap<String, Value>, Box<dyn std::error::Error>> {+ let document = parse_html().one(html);+ let mut out = HashMap::<String, Value>::new();++ if let Some(title) = document+ .select("title")+ .map_err(|_| "Failed to select title")?+ .next()+ {+ out.insert("title".to_string(), Value::String(title.text_contents()));+ }++ if let Some(favicon_link) = document+ .select("link[rel=\"icon\"]")+ .map_err(|_| "Failed to select favicon")?+ .next()+ .and_then(|x| x.attributes.borrow().get("href").map(|x| x.to_string()))+ .or_else(|| {+ document+ .select("link[rel*=\"icon\"]")+ .ok()+ .and_then(|mut x| {+ x.next()+ .and_then(|x| x.attributes.borrow().get("href").map(|x| x.to_string()))+ })+ })+ {+ out.insert("favicon".to_string(), Value::String(favicon_link));+ }++ if let Some(lang) = document+ .select("html[lang]")+ .map_err(|_| "Failed to select lang")?+ .next()+ .and_then(|x| x.attributes.borrow().get("lang").map(|x| x.to_string()))+ {+ out.insert("language".to_string(), Value::String(lang));+ }++ insert_meta_property!(out, document, "og:title", "ogTitle");+ insert_meta_property!(out, document, "og:description", "ogDescription");+ insert_meta_property!(out, document, "og:url", "ogUrl");+ insert_meta_property!(out, document, "og:image", "ogImage");+ insert_meta_property!(out, document, "og:audio", "ogAudio");+ insert_meta_property!(out, document, "og:determiner", "ogDeterminer");+ insert_meta_property!(out, document, "og:locale", "ogLocale");++ for meta in document+ .select("meta[property=\"og:locale:alternate\"]")+ .map_err(|_| "Failed to select og locale alternate")?+ {+ let attrs = meta.attributes.borrow();++ if let Some(content) = attrs.get("content") {+ if let Some(v) = out.get_mut("ogLocaleAlternate") {+ match v {+ Value::Array(x) => x.push(Value::String(content.to_string())),+ _ => unreachable!(),+ }+ } else {+ out.insert(+ "ogLocaleAlternate".to_string(),+ Value::Array(vec![Value::String(content.to_string())]),+ );+ }+ }+ }++ insert_meta_property!(out, document, "og:site_name", "ogSiteName");+ insert_meta_property!(out, document, "og:video", "ogVideo");+ insert_meta_name!(out, document, "article:section", "articleSection");+ insert_meta_name!(out, document, "article:tag", "articleTag");+ insert_meta_property!(out, document, "article:published_time", "publishedTime");+ insert_meta_property!(out, document, "article:modified_time", "modifiedTime");+ insert_meta_name!(out, document, "dcterms.keywords", "dcTermsKeywords");+ insert_meta_name!(out, document, "dc.description", "dcDescription");+ insert_meta_name!(out, document, "dc.subject", "dcSubject");+ insert_meta_name!(out, document, "dcterms.subject", "dcTermsSubject");+ insert_meta_name!(out, document, "dcterms.audience", "dcTermsAudience");+ insert_meta_name!(out, document, "dc.type", "dcType");+ insert_meta_name!(out, document, "dcterms.type", "dcTermsType");+ insert_meta_name!(out, document, "dc.date", "dcDate");+ insert_meta_name!(out, document, "dc.date.created", "dcDateCreated");+ insert_meta_name!(out, document, "dcterms.created", "dcTermsCreated");++ for meta in document+ .select("meta")+ .map_err(|_| "Failed to select meta")?+ {+ let meta = meta.as_node().as_element().unwrap();+ let attrs = meta.attributes.borrow();++ if let Some(name) = attrs+ .get("name")+ .or_else(|| attrs.get("property"))+ .or_else(|| attrs.get("itemprop"))+ {+ if let Some(content) = attrs.get("content") {+ if let Some(v) = out.get(name) {+ match v {+ Value::String(existing) => {+ if name == "description" {+ out.insert(+ name.to_string(),+ Value::String(format!("{}, {}", existing, content)),+ );+ } else if name != "title" {+ out.insert(+ name.to_string(),+ Value::Array(vec![+ Value::String(existing.clone()),+ Value::String(content.to_string()),+ ]),+ );+ }+ }+ Value::Array(existing_array) => {+ if name == "description" {+ let mut values: Vec<String> = existing_array+ .iter()+ .filter_map(|v| match v {+ Value::String(s) => Some(s.clone()),+ _ => None,+ })+ .collect();+ values.push(content.to_string());+ out.insert(name.to_string(), Value::String(values.join(", ")));+ } else {+ match out.get_mut(name) {+ Some(Value::Array(x)) => x.push(Value::String(content.to_string())),+ _ => unreachable!(),+ }+ }+ }+ _ => unreachable!(),+ }+ } else {+ out.insert(name.to_string(), Value::String(content.to_string()));+ }+ }+ }+ }++ Ok(out)+}++/// Extract metadata from HTML document.+#[napi]+pub fn extract_metadata(html: Option<String>) -> napi::Result<HashMap<String, Value>> {+ let html = match html {+ Some(h) => h,+ None => return Ok(HashMap::new()),+ };++ _extract_metadata(&html).map_err(to_napi_err)+}++const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [+ "header",+ "footer",+ "nav",+ "aside",+ ".header",+ ".top",+ ".navbar",+ "#header",+ ".footer",+ ".bottom",+ "#footer",+ ".sidebar",+ ".side",+ ".aside",+ "#sidebar",+ ".modal",+ ".popup",+ "#modal",+ ".overlay",+ ".ad",+ ".ads",+ ".advert",+ "#ad",+ ".lang-selector",+ ".language",+ "#language-selector",+ ".social",+ ".social-media",+ ".social-links",+ "#social",+ ".menu",+ ".navigation",+ "#nav",+ ".breadcrumbs",+ "#breadcrumbs",+ ".share",+ "#share",+ ".widget",+ "#widget",+ ".cookie",+ "#cookie",+ ".fc-decoration",+];++const FORCE_INCLUDE_MAIN_TAGS: [&str; 13] = [+ "#main",+ ".swoogo-cols",+ ".swoogo-text",+ ".swoogo-table-div",+ ".swoogo-space",+ ".swoogo-alert",+ ".swoogo-sponsors",+ ".swoogo-title",+ ".swoogo-tabs",+ ".swoogo-logo",+ ".swoogo-image",+ ".swoogo-button",+ ".swoogo-agenda",+];++#[derive(Deserialize, Serialize)]+#[napi(object)]+pub struct TransformHtmlOptions {+ pub html: String,+ pub url: String,+ #[serde(default)]+ pub include_tags: Vec<String>,+ #[serde(default)]+ pub exclude_tags: Vec<String>,+ pub only_main_content: bool,+ pub omce_signatures: Option<Vec<String>>,+}++struct ImageSource {+ url: String,+ size: f64,+ is_x: bool,+}++fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn std::error::Error>> {+ let mut document = parse_html().one(opts.html.as_ref());+ let url = Url::parse(&_extract_base_href_from_document(+ &document,+ &Url::parse(&opts.url)?,+ )?)?;++ if !opts.include_tags.is_empty() {+ let new_document = parse_html().one("<div></div>");+ let root = new_document+ .select_first("div")+ .map_err(|_| "Failed to select root element")?;++ for x in opts.include_tags.iter() {+ let matching_nodes: Vec<_> = document+ .select(x)+ .map_err(|_| "Failed to include_tags tags")?+ .collect();+ for tag in matching_nodes {+ root.as_node().append(tag.as_node().clone());+ }+ }++ document = new_document;+ }++ while let Ok(x) = document.select_first("head") {+ x.as_node().detach();+ }+ while let Ok(x) = document.select_first("meta") {+ x.as_node().detach();+ }+ while let Ok(x) = document.select_first("noscript") {+ x.as_node().detach();+ }+ while let Ok(x) = document.select_first("style") {+ x.as_node().detach();+ }+ while let Ok(x) = document.select_first("script") {+ x.as_node().detach();+ }++ // OMCE first+ if opts.only_main_content {+ if let Some(signatures) = opts.omce_signatures.as_ref() {+ let mut nodes_to_drop: Vec<NodeRef> = Vec::new();++ let modes = signatures+ .iter()+ .map(|x| Into::<SignatureMode>::into(x.split(':').nth(1).unwrap().to_string()))+ .collect::<HashSet<_>>();++ for mode in modes {+ let matcher = format!(":{}:", Into::<String>::into(mode.clone()));+ let signatures = signatures+ .iter()+ .filter(|x| x.contains(&matcher))+ .cloned()+ .collect::<HashSet<_>>();++ for edge in document.traverse() {+ match edge {+ NodeEdge::Start(_) => {}+ NodeEdge::End(node) => {+ if node.as_element().is_none() {+ continue;+ }+ if node.text_contents().trim().is_empty() {+ continue;+ }++ let signature = get_node_signature(&node, mode.clone());+ if signatures.contains(&signature) {+ nodes_to_drop.push(node);+ }+ }+ }+ }+ }++ for node in nodes_to_drop {+ node.detach();+ }+ }+ }++ for x in opts.exclude_tags.iter() {+ while let Ok(x) = document.select_first(x) {+ x.as_node().detach();+ }+ }++ if opts.only_main_content {+ for x in EXCLUDE_NON_MAIN_TAGS.iter() {+ let x: Vec<_> = document+ .select(x)+ .map_err(|_| "Failed to select tags")?+ .collect();+ for tag in x {+ if !FORCE_INCLUDE_MAIN_TAGS.iter().any(|x| {+ tag+ .as_node()+ .select(x)+ .is_ok_and(|mut x| x.next().is_some())+ }) {+ tag.as_node().detach();+ }+ }+ }+ }++ let srcset_images: Vec<_> = document+ .select("img[srcset]")+ .map_err(|_| "Failed to select srcset images")?+ .collect();+ for img in srcset_images {+ let mut sizes: Vec<ImageSource> = img+ .attributes+ .borrow()+ .get("srcset")+ .ok_or("Failed to get srcset")?+ .split(',')+ .filter_map(|x| {+ let tok: Vec<&str> = x.trim().split(' ').collect();+ let last_token = tok[tok.len() - 1];+ let (last_token, last_token_used) = if tok.len() > 1+ && !last_token.is_empty()+ && (last_token.ends_with('x') || last_token.ends_with('w'))+ {+ (last_token, true)+ } else {+ ("1x", false)+ };++ if let Some((last_index, _)) = last_token.char_indices().last() {+ if let Ok(parsed_size) = last_token[..last_index].parse() {+ Some(ImageSource {+ url: if last_token_used {+ tok[0..tok.len() - 1].join(" ")+ } else {+ tok.join(" ")+ },+ size: parsed_size,+ is_x: last_token.ends_with('x'),+ })+ } else {+ None+ }+ } else {+ None+ }+ })+ .collect();++ if sizes.iter().all(|x| x.is_x) {+ if let Some(src) = img.attributes.borrow().get("src").map(|x| x.to_string()) {+ sizes.push(ImageSource {+ url: src,+ size: 1.0,+ is_x: true,+ });+ }+ }++ sizes.sort_by(|a, b| {+ b.size+ .partial_cmp(&a.size)+ .unwrap_or(std::cmp::Ordering::Equal)+ });++ if let Some(biggest) = sizes.first() {+ img+ .attributes+ .borrow_mut()+ .insert("src", biggest.url.clone());+ }+ }++ let src_images: Vec<_> = document+ .select("img[src]")+ .map_err(|_| "Failed to select src images")?+ .collect();+ for img in src_images {+ let old = img+ .attributes+ .borrow()+ .get("src")+ .map(|x| x.to_string())+ .ok_or("Failed to get src")?;+ if let Ok(new) = url.join(&old) {+ img.attributes.borrow_mut().insert("src", new.to_string());+ }+ }++ let href_anchors: Vec<_> = document+ .select("a[href]")+ .map_err(|_| "Failed to select href anchors")?+ .collect();+ for anchor in href_anchors {+ let old = anchor+ .attributes+ .borrow()+ .get("href")+ .map(|x| x.to_string())+ .ok_or("Failed to get href")?;+ if let Ok(new) = url.join(&old) {+ anchor+ .attributes+ .borrow_mut()+ .insert("href", new.to_string());+ }+ }++ Ok(document.to_string())+}++/// Transform and clean HTML content based on provided options.+#[napi]+pub fn transform_html(opts: TransformHtmlOptions) -> napi::Result<String> {+ _transform_html_inner(opts).map_err(to_napi_err)+}++fn _get_inner_json(html: &str) -> Result<String, ()> {+ Ok(parse_html().one(html).select_first("body")?.text_contents())+}++/// Extract inner text content from HTML body.+#[napi]+pub fn get_inner_json(html: String) -> napi::Result<String> {+ _get_inner_json(&html).map_err(|_| to_napi_err("Failed to get inner JSON"))+}++#[derive(Deserialize, Serialize)]+#[napi(object)]+pub struct AttributeSelector {+ pub selector: String,+ pub attribute: String,+}++#[derive(Deserialize, Serialize)]+#[napi(object)]+pub struct ExtractAttributesOptions {+ pub selectors: Vec<AttributeSelector>,+}++#[derive(Serialize)]+#[napi(object)]+pub struct ExtractedAttributeResult {+ pub selector: String,+ pub attribute: String,+ pub values: Vec<String>,+}++fn _extract_attributes(+ html: &str,+ options: &ExtractAttributesOptions,+) -> Result<Vec<ExtractedAttributeResult>, Box<dyn std::error::Error>> {+ let document = parse_html().one(html);+ let mut results = Vec::new();++ for selector_config in &options.selectors {+ let mut values = Vec::new();++ let elements: Vec<_> = match document.select(&selector_config.selector).map_err(|_| {+ format!(+ "Failed to select with selector: {}",+ selector_config.selector+ )+ }) {+ Ok(x) => x.collect(),+ Err(_) => Vec::new(), // invalid selector => empty list+ };++ for element in elements {+ if let Some(attr_value) = element+ .attributes+ .borrow()+ .get(selector_config.attribute.as_str())+ {+ values.push(attr_value.to_string());+ continue;+ }++ if !selector_config.attribute.starts_with("data-") {+ let data_attr = format!("data-{}", selector_config.attribute);+ if let Some(attr_value) = element.attributes.borrow().get(data_attr.as_str()) {+ values.push(attr_value.to_string());+ }+ }+ }++ results.push(ExtractedAttributeResult {+ selector: selector_config.selector.clone(),+ attribute: selector_config.attribute.clone(),+ values,+ });+ }++ Ok(results)+}++/// Extract specified attributes from HTML elements matching selectors.+#[napi]+pub fn extract_attributes(+ html: String,+ options: ExtractAttributesOptions,+) -> napi::Result<Vec<ExtractedAttributeResult>> {+ _extract_attributes(&html, &options).map_err(to_napi_err)+}++fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn std::error::Error>> {+ let document = parse_html().one(html);+ let base_url = Url::parse(base_url)?;+ let base_href = _extract_base_href_from_document(&document, &base_url)?;+ let base_href_url = Url::parse(&base_href)?;+ let mut images = HashSet::<String>::new();++ let resolve_image_url = |src: &str| -> Result<String, Box<dyn std::error::Error>> {+ if src.starts_with("data:") || src.starts_with("blob:") {+ return Ok(src.to_string());+ }+ if src.starts_with("http://") || src.starts_with("https://") {+ return Ok(src.to_string());+ }+ if src.starts_with("//") {+ let resolved = base_url.join(src)?;+ return Ok(resolved.to_string());+ }+ let resolved = base_href_url.join(src)?;+ Ok(resolved.to_string())+ };++ // <img>+ let img_elements: Vec<_> = match document+ .select("img")+ .map_err(|_| "Failed to select img tags")+ {+ Ok(x) => x.collect(),+ Err(e) => return Err(e.into()),+ };++ for img in img_elements {+ let attrs = img.attributes.borrow();++ if let Some(src) = attrs.get("src") {+ if let Ok(resolved) = resolve_image_url(src) {+ images.insert(resolved);+ }+ }++ if let Some(data_src) = attrs.get("data-src") {+ if let Ok(resolved) = resolve_image_url(data_src) {+ images.insert(resolved);+ }+ }++ if let Some(srcset) = attrs.get("srcset") {+ for part in srcset.split(',') {+ if let Some(url) = part.trim().split_whitespace().next() {+ if !url.is_empty() {+ if let Ok(resolved) = resolve_image_url(url) {+ images.insert(resolved);+ }+ }+ }+ }+ }+ }++ // <picture><source>+ let source_elements: Vec<_> = match document+ .select("picture source")+ .map_err(|_| "Failed to select picture source")+ {+ Ok(x) => x.collect(),+ Err(_) => Vec::new(),+ };++ for source in source_elements {+ if let Some(srcset) = source.attributes.borrow().get("srcset") {+ for part in srcset.split(',') {+ if let Some(url) = part.trim().split_whitespace().next() {+ if !url.is_empty() {+ if let Ok(resolved) = resolve_image_url(url) {+ images.insert(resolved);+ }+ }+ }+ }+ }+ }++ // OG/Twitter images+ let meta_selectors = [+ "meta[property=\"og:image\"]",+ "meta[property=\"og:image:url\"]",+ "meta[property=\"og:image:secure_url\"]",+ "meta[name=\"twitter:image\"]",+ "meta[name=\"twitter:image:src\"]",+ "meta[itemprop=\"image\"]",+ ];++ for selector in &meta_selectors {+ if let Ok(elements) = document.select(selector) {+ for element in elements {+ if let Some(content) = element.attributes.borrow().get("content") {+ if let Ok(resolved) = resolve_image_url(content) {+ images.insert(resolved);+ }+ }+ }+ }+ }++ // icons+ let link_selectors = [+ "link[rel*=\"icon\"]",+ "link[rel*=\"apple-touch-icon\"]",+ "link[rel*=\"image_src\"]",+ ];++ for selector in &link_selectors {+ if let Ok(elements) = document.select(selector) {+ for element in elements {+ if let Some(href) = element.attributes.borrow().get("href") {+ if let Ok(resolved) = resolve_image_url(href) {+ images.insert(resolved);+ }+ }+ }+ }+ }++ // <video poster="">+ if let Ok(video_elements) = document.select("video[poster]") {+ for video in video_elements {+ if let Some(poster) = video.attributes.borrow().get("poster") {+ if let Ok(resolved) = resolve_image_url(poster) {+ images.insert(resolved);+ }+ }+ }+ }++ let filtered_images: Vec<String> = images+ .into_iter()+ .filter(|url| !url.to_lowercase().starts_with("javascript:"))+ .filter(|url| !url.is_empty())+ .filter(|url| url.starts_with("data:") || url.starts_with("blob:") || Url::parse(url).is_ok())+ .collect();++ Ok(filtered_images)+}++/// Extract all image URLs from HTML document.+#[napi]+pub fn extract_images(html: String, base_url: String) -> napi::Result<Vec<String>> {+ _extract_images(&html, &base_url).map_err(to_napi_err)+}diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rsnew file mode 100644index 000000000..06cfebea6--- /dev/null+++ b/apps/api/native/src/lib.rs@@ -0,0 +1,14 @@+#![deny(clippy::all)]++pub use crate::crawler::*;+pub use crate::html::*;+pub use crate::pdf::*;+pub use crate::utils::*;++mod crawler;+mod html;+mod pdf;+mod utils;++pub use napi::bindgen_prelude::*;+pub use serde::{Deserialize, Serialize};diff --git a/apps/api/native/src/pdf.rs b/apps/api/native/src/pdf.rsnew file mode 100644index 000000000..8b144d042--- /dev/null+++ b/apps/api/native/src/pdf.rs@@ -0,0 +1,64 @@+use napi::bindgen_prelude::*;+use napi_derive::napi;+use serde::Serialize;++#[derive(Debug, Clone, Serialize)]+#[napi(object)]+pub struct PDFMetadata {+ pub num_pages: i32,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub title: Option<String>,+}++fn _get_pdf_metadata(path: &str) -> std::result::Result<PDFMetadata, String> {+ let doc = match lopdf::Document::load(path) {+ Ok(x) => x,+ Err(_) => {+ return Err("Failed to load PDF".to_string());+ }+ };++ let num_pages = doc.get_pages().len() as i32;++ let title = doc+ .trailer+ .get(b"Info")+ .and_then(|info| {+ info+ .as_dict()+ .and_then(|info| info.get(b"Title"))+ .and_then(|title| {+ title+ .as_str()+ .map(|s| String::from_utf8_lossy(s).to_string())+ })+ })+ .ok()+ .or_else(|| {+ doc.objects.iter().find_map(|(_i, obj)| {+ obj+ .as_dict()+ .and_then(|obj| obj.get(b"Title"))+ .and_then(|title| title.as_str())+ .map(|s| String::from_utf8_lossy(s).to_string())+ .ok()+ })+ })+ .map(|x| {+ x.trim_matches(|x| char::is_whitespace(x) || x == '\0')+ .to_string()+ });++ Ok(PDFMetadata { num_pages, title })+}++/// Extract metadata from PDF file.+#[napi]+pub fn get_pdf_metadata(path: String) -> Result<PDFMetadata> {+ _get_pdf_metadata(&path).map_err(|e| {+ Error::new(+ Status::GenericFailure,+ format!("Failed to get PDF metadata: {}", e),+ )+ })+}diff --git a/apps/api/native/src/utils.rs b/apps/api/native/src/utils.rsnew file mode 100644index 000000000..3d78d06c5--- /dev/null+++ b/apps/api/native/src/utils.rs@@ -0,0 +1,5 @@+use napi::bindgen_prelude::*;++pub fn to_napi_err<E: std::fmt::Display>(error: E) -> Error {+ Error::new(Status::GenericFailure, error.to_string())+}diff --git a/apps/api/native/tsconfig.json b/apps/api/native/tsconfig.jsonnew file mode 100644index 000000000..5056788c0--- /dev/null+++ b/apps/api/native/tsconfig.json@@ -0,0 +1,14 @@+{+ "compilerOptions": {+ "target": "ESNext",+ "strict": true,+ "moduleResolution": "node",+ "module": "CommonJS",+ "noUnusedLocals": true,+ "noUnusedParameters": true,+ "esModuleInterop": true,+ "allowSyntheticDefaultImports": true+ },+ "include": ["."],+ "exclude": ["node_modules", "bench", "__test__"]+}diff --git a/apps/api/native/wasi-worker-browser.mjs b/apps/api/native/wasi-worker-browser.mjsnew file mode 100644index 000000000..8b1b17221--- /dev/null+++ b/apps/api/native/wasi-worker-browser.mjs@@ -0,0 +1,32 @@+import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'++const handler = new MessageHandler({+ onLoad({ wasmModule, wasmMemory }) {+ const wasi = new WASI({+ print: function () {+ // eslint-disable-next-line no-console+ console.log.apply(console, arguments)+ },+ printErr: function() {+ // eslint-disable-next-line no-console+ console.error.apply(console, arguments)+ },+ })+ return instantiateNapiModuleSync(wasmModule, {+ childThread: true,+ wasi,+ overwriteImports(importObject) {+ importObject.env = {+ ...importObject.env,+ ...importObject.napi,+ ...importObject.emnapi,+ memory: wasmMemory,+ }+ },+ })+ },+})++globalThis.onmessage = function (e) {+ handler.handle(e)+}diff --git a/apps/api/package.json b/apps/api/package.jsonindex a5b3a7a7e..be164054e 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -159,7 +159,8 @@"ws": "^8.18.0","x402-express": "^0.5.3","xml2js": "^0.6.2",- "zod": "^3.24.2"+ "zod": "^3.24.2",+ "@mendable/firecrawl-rs": "workspace:*"},"nodemonConfig": {"ignore": [@@ -179,5 +180,6 @@},"lint-staged": {"*.{js,jsx,ts,tsx,json,css,md}": "prettier --write"- }+ },+ "packageManager": "[email protected]"}diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 715614fe4..b04f60409 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -3,6 +3,7 @@ lockfileVersion: '9.0'settings:autoInstallPeers: trueexcludeLinksFromLockfile: false+ injectWorkspacePackages: trueoverrides:brace-expansion@>=2.0.0 <=2.0.1: 2.0.2@@ -56,6 +57,9 @@ importers:'@google-cloud/storage':specifier: ^7.16.0version: 7.16.0([email protected])+ '@mendable/firecrawl-rs':+ specifier: workspace:*+ version: link:native'@nangohq/node':specifier: ^0.40.8version: 0.40.8@@ -391,6 +395,42 @@ importers:specifier: ^5.8.3version: 5.8.3+ native:+ devDependencies:+ '@emnapi/core':+ specifier: ^1.5.0+ version: 1.5.0+ '@emnapi/runtime':+ specifier: ^1.5.0+ version: 1.5.0+ '@napi-rs/cli':+ specifier: ^3.1.5+ version: 3.1.5(@emnapi/[email protected])(@types/[email protected])+ '@oxc-node/core':+ specifier: ^0.0.32+ version: 0.0.32+ '@taplo/cli':+ specifier: ^0.7.0+ version: 0.7.0+ '@tybys/wasm-util':+ specifier: ^0.10.0+ version: 0.10.0+ chalk:+ specifier: ^5.6.0+ version: 5.6.0+ npm-run-all2:+ specifier: ^8.0.4+ version: 8.0.4+ oxlint:+ specifier: ^1.14.0+ version: 1.14.0+ prettier:+ specifier: ^3.6.2+ version: 3.6.2+ typescript:+ specifier: ^5.9.2+ version: 5.9.2+packages:'@adraffy/[email protected]':@@ -983,6 +1023,15 @@ packages:peerDependencies:'@noble/ciphers': ^1.0.0+ '@emnapi/[email protected]':+ resolution: {integrity: sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==}++ '@emnapi/[email protected]':+ resolution: {integrity: sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==}++ '@emnapi/[email protected]':+ resolution: {integrity: sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==}+'@esbuild/[email protected]':resolution: {integrity: sha512-urAvrUedIqEiFR3FYSLTWQgLu5tb+m0qZw0NBEasUeo6wuqatkMDaRT+1uABiGXEu5vqgPd7FGE1BhsAIy9QVA==}engines: {node: '>=18'}@@ -1180,6 +1229,136 @@ packages:engines: {node: '>=6'}hasBin: true+ '@inquirer/[email protected]':+ resolution: {integrity: sha512-E+KExNurKcUJJdxmjglTl141EwxWyAHplvsYJQgSwXf8qiNWkTxTuCCqmhFEmbIXd4zLaGMfQFJ6WrZ7fSeV3g==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-j1a5VstaK5KQy8Mu8cHmuQvN1Zc62TbLhjJxwHvKPPKEoowSF6h/0UdOpA9DNdWZ+9Inq73+puRq1df6OJ8Sag==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-NyDSjPqhSvpZEMZrLCYUquWNl+XC/moEcVFqS55IEYIYsY0a1cUCevSqk7ctOlnm/RaSBU5psFryNlxcmGrjaA==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-yeQN3AXjCm7+Hmq5L6Dm2wEDeBRdAZuyZ4I7tWSSanbxDzqM0KqzoDbKM7p4ebllAYdoQuPJS6N71/3L281i6w==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-xUjteYtavH7HwDMzq4Cn2X4Qsh5NozoDHCJTdoXg9HfZ4w3R6mxV1B9tL7DGJX2eq/zqtsFjhm0/RJIMGlh3ag==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-Oau4yL24d2B5IL4ma4UpbQigkVhzPDXLoqy1ggK4gnHg/stmkffJE4oOXHXF3uz0UEpywG68KcyXsyYpA1Re/Q==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-lGPVU3yO9ZNqA7vTYz26jny41lE7yoQansmqdMLBEfqaGsmdg7V3W9mK9Pvb5IL4EVZ9GnSDGMO/cJXud5dMaw==}+ engines: {node: '>=18'}++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-hqOvBZj/MhQCpHUuD3MVq18SSoDNHy7wEnQ8mtvs71K8OPZVXJinOzcvQna33dNYLYE4LkA9BlhAhK6MJcsVbw==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-7exgBm52WXZRczsydCVftozFTrrwbG5ySE0GqUd2zLNSBXyIucs2Wnm7ZKLe/aUu6NUg9dg7Q80QIHCdZJiY4A==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-zXvzAGxPQTNk/SbT3carAD4Iqi6A2JS2qtcqQjsL22uvD+JfQzUrDEtPjLL7PLn8zlSNyPdY02IiQjzoL9TStA==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-MuxVZ1en1g5oGamXV3DWP89GEkdD54alcfhHd7InUW5BifAdKQEK9SLFa/5hlWbvuhMPlobF0WAx7Okq988Jxg==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-KOZqa3QNr3f0pMnufzL7K+nweFFCCBs6LCXZzXDrVGTyssjLeudn5ySktZYv1XiSqobyHRYYK0c6QsOxJEhXKA==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-TkMUY+A2p2EYVY3GCTItYGvqT6LiLzHBnqsU1rJbrpXUijFfM6zvUx0R4civofVwFCmJZcKqOVwwWAjplKkhxA==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-nwous24r31M+WyDEHV+qckXkepvihxhnyIaod2MG7eCE6G0Zm/HUF6jgN8GXgf4U7AU6SLseKdanY195cwvU6w==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true++ '@inquirer/[email protected]':+ resolution: {integrity: sha512-lg9Whz8onIHRthWaN1Q9EGLa/0LFJjyM8mEUbL1eTi6yMGvBf8gvyDLtxSXztQsxMvhxxNpJYrwa1YHdq+w4Jw==}+ engines: {node: '>=18'}+ peerDependencies:+ '@types/node': '>=18'+ peerDependenciesMeta:+ '@types/node':+ optional: true+'@ioredis/[email protected]':resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}@@ -1404,6 +1583,342 @@ packages:resolution: {integrity: sha512-w3btGjeTsYG32hPfbDfziYQRfqGWML3AK88ZE6sOu6qZHLbx7WWf1X/TsTogSxJ2BhnP4UoVQ4F935/G7ZWztA==}engines: {node: '>=18.0'}+ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-Wn6ZPw27qJiEWglGjkaAa70AHuLtyPya6FvjINYJ5U20uvbRhoB0Ta2+bFTAFfUb9R+wvuFvog9JQdy65OmFAQ==}+ engines: {node: '>= 16'}+ hasBin: true+ peerDependencies:+ '@emnapi/runtime': ^1.1.0+ emnapi: ^1.1.0+ peerDependenciesMeta:+ '@emnapi/runtime':+ optional: true+ emnapi:+ optional: true++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-ENPfLe4937bsKVTDA6zdABx4pq9w0tHqRrJHyaGxgaPq03a2Bd1unD5XSKjXJjebsABJ+MjAv1A2OvCgK9yehg==}+ peerDependencies:+ '@napi-rs/cross-toolchain-arm64-target-aarch64': ^1.0.3+ '@napi-rs/cross-toolchain-arm64-target-armv7': ^1.0.3+ '@napi-rs/cross-toolchain-arm64-target-ppc64le': ^1.0.3+ '@napi-rs/cross-toolchain-arm64-target-s390x': ^1.0.3+ '@napi-rs/cross-toolchain-arm64-target-x86_64': ^1.0.3+ '@napi-rs/cross-toolchain-x64-target-aarch64': ^1.0.3+ '@napi-rs/cross-toolchain-x64-target-armv7': ^1.0.3+ '@napi-rs/cross-toolchain-x64-target-ppc64le': ^1.0.3+ '@napi-rs/cross-toolchain-x64-target-s390x': ^1.0.3+ '@napi-rs/cross-toolchain-x64-target-x86_64': ^1.0.3+ peerDependenciesMeta:+ '@napi-rs/cross-toolchain-arm64-target-aarch64':+ optional: true+ '@napi-rs/cross-toolchain-arm64-target-armv7':+ optional: true+ '@napi-rs/cross-toolchain-arm64-target-ppc64le':+ optional: true+ '@napi-rs/cross-toolchain-arm64-target-s390x':+ optional: true+ '@napi-rs/cross-toolchain-arm64-target-x86_64':+ optional: true+ '@napi-rs/cross-toolchain-x64-target-aarch64':+ optional: true+ '@napi-rs/cross-toolchain-x64-target-armv7':+ optional: true+ '@napi-rs/cross-toolchain-x64-target-ppc64le':+ optional: true+ '@napi-rs/cross-toolchain-x64-target-s390x':+ optional: true+ '@napi-rs/cross-toolchain-x64-target-x86_64':+ optional: true++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-Up4gpyw2SacmyKWWEib06GhiDdF+H+CCU0LAV8pnM4aJIDqKKd5LHSlBht83Jut6frkB0vwEPmAkv4NjQ5u//Q==}+ engines: {node: '>= 10'}+ cpu: [arm]+ os: [android]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-uwa8sLlWEzkAM0MWyoZJg0JTD3BkPknvejAFG2acUA1raXM8jLrqujWCdOStisXhqQjZ2nDMp3FV6cs//zjfuQ==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [android]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-0Y0TQLQ2xAjVabrMDem1NhIssOZzF/y/dqetc6OT8mD3xMTDtF8u5BqZoX3MyPc9FzpsZw4ksol+w7DsxHrpMA==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [darwin]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-vR2IUyJY3En+V1wJkwmbGWcYiT8pHloTAWdW4pG24+51GIq+intst6Uf6D/r46citObGZrlX0QvMarOkQeHWpw==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [darwin]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-XpnYQC5SVovO35tF0xGkbHYjsS6kqyNCjuaLQ2dbEblFRr5cAZVvsJ/9h7zj/5FluJPJRDojVNxGyRhTp4z2lw==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [freebsd]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-ic1ZZMoRfRMwtSwxkyw4zIlbDZGC6davC9r+2oX6x9QiF247BRqqT94qGeL5ZP4Vtz0Hyy7TEViWhx5j6Bpzvw==}+ engines: {node: '>= 10'}+ cpu: [arm]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-asEp7FPd7C1Yi6DQb45a3KPHKOFBSfGuJWXcAd4/bL2Fjetb2n/KK2z14yfW8YC/Fv6x3rBM0VAZKmJuz4tysg==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-yWjcPDgJ2nIL3KNvi4536dlT/CcCWO0DUyEOlBs/SacG7BeD6IjGh6yYzd3/X1Y3JItCbZoDoLUH8iB1lTXo3w==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-0XRhKuIU/9ZjT4WDIG/qnX7Xz7mSQHYZo9Gb3MP2gcvBgr6BA4zywQ9k3gmQaPn9ECE+CZg2V7DV7kT+x2pUMQ==}+ engines: {node: '>= 10'}+ cpu: [ppc64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-QrqDIPEUUB23GCpyQj/QFyMlr8SGxxyExeZz9OWFnHfb70kXdTLWrHS/hEI1Ru+lSbQ/6xRqeoGyQ4Aqdg+/RA==}+ engines: {node: '>= 10'}+ cpu: [riscv64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-k8RVM5aMhW86E9H0QXdquwojew4H3SwPxbRVbl49/COJQWCUjGi79X6mYruMnMPEznZinUiT1jgKbFo2A00NdA==}+ engines: {node: '>= 10'}+ cpu: [s390x]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-6rMtBgnIq2Wcl1rQdZsnM+rtCcVCbws1nF8S2NzaUsVaZv8bjrPiAa0lwg4Eqnn1d9lgwqT+cZgm5m+//K08Kw==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-eiadGBKi7Vd0bCArBUOO/qqRYPHt/VQVvGyYvDFt6C2ZSIjlD+HuOl+2oS1sjf4CFjK4eDIog6EdXnL0NE6iyQ==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-+VyHHlr68dvey6fXc2hehw9gHVFIW3TtGF1XkcbAu65qVXsA9D/T+uuoRVqhE+JCyFHFrO0ixRbZDRK1XJt1sA==}+ engines: {node: '>=14.0.0'}+ cpu: [wasm32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-eewnqvIyyhHi3KaZtBOJXohLvwwN27gfS2G/YDWdfHlbz1jrmfeHAmzMsP5qv8vGB+T80TMHNkro4kYjeh6Deg==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-OeacFVRCJOKNU/a0ephUfYZ2Yt+NvaHze/4TgOwJ0J0P4P7X1mHzN+ig9Iyd74aQDXYqc7kaCXA2dpAOcH87Cg==}+ engines: {node: '>= 10'}+ cpu: [ia32]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-T4I1SamdSmtyZgDXGAGP+y5LEK5vxHUFwe8mz6D4R7Sa5/WCxTcCIgPJ9BD7RkpO17lzhlaM2vmVvMy96Lvk9Q==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-zS5LuN1OBPAyZpda2ZZgYOEDC+xecUdAGnrvbYzjnLXkrq/OBC3B9qcRvlxbDR3k5H/gVfvef1/jyUqPknqjbg==}+ engines: {node: '>= 10'}++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-h2Ryndraj/YiKgMV/r5by1cDusluYIRT0CaE0/PekQ4u+Wpy2iUVqvzVU98ZPnhXaNeYxEvVJHNGafpOfaD0TA==}+ engines: {node: '>= 10'}+ cpu: [arm]+ os: [android]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-DJFyQHr1ZxNZorm/gzc1qBNLF/FcKzcH0V0Vwan5P+o0aE2keQIGEjJ09FudkF9v6uOuJjHCVDdK6S6uHtShAw==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [android]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-Zz2sXRzjIX4e532zD6xm2SjXEym6MkvfCvL2RMpG2+UwNVDVscHNcz3d47Pf3sysP2e2af7fBB3TIoK2f6trPw==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [darwin]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-EI+CptIMNweT0ms9S3mkP/q+J6FNZ1Q6pvpJOEcWglRfyfQpLqjlC0O+dptruTPE8VamKYuqdjxfqD8hifZDOA==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [darwin]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-J0PIqX+pl6lBIAckL/c87gpodLbjZB1OtIK+RDscKC9NLdpVv6VGOxzUV/fYev/hctcE8EfkLbgFOfpmVQPg2g==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [freebsd]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-SLgIQo3f3EjkZ82ZwvrEgFvMdDAhsxCYjyoSuWfHCz0U16qx3SuGCp8+FYOPYCECHN3ZlGjXnoAIt9ERd0dEUg==}+ engines: {node: '>= 10'}+ cpu: [arm]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-d014cdle52EGaH6GpYTQOP9Py7glMO1zz/+ynJPjjzYFSxvdYx0byrjumZk2UQdIyGZiJO2MEFpCkEEKFSgPYA==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-L/y1/26q9L/uBqiW/JdOb/Dc94egFvNALUZV2WCGKQXc6UByPBMgdiEyW2dtoYxYYYYc+AKD+jr+wQPcvX2vrQ==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-EPE1K/80RQvPbLRJDJs1QmCIcH+7WRi0F73+oTe1582y9RtfGRuzAkzeBuAGRXAQEjRQw/RjtNqr6UTJ+8UuWQ==}+ engines: {node: '>= 10'}+ cpu: [ppc64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-B2jhWiB1ffw1nQBqLUP1h4+J1ovAxBOoe5N2IqDMOc63fsPZKNqF1PvO/dIem8z7LL4U4bsfmhy3gBfu547oNQ==}+ engines: {node: '>= 10'}+ cpu: [s390x]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-tbZDHnb9617lTnsDMGo/eAMZxnsQFnaRe+MszRqHguKfMwkisc9CCJnks/r1o84u5fECI+J/HOrKXgczq/3Oww==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-dV6cODlzbO8u6Anmv2N/ilQHq/AWz0xyltuXoLU3yUyXbZcnWYZuB2rL8OBGPmqNcD+x9NdScBNXh7vWN0naSQ==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-jIa9nb2HzOrfH0F8QQ9g3WE4aMH5vSI5/1NYVNm9ysCmNjCCtMXCAhlI3WKCdm/DwHf0zLqdrrtDFXODcNaqMw==}+ engines: {node: '>=14.0.0'}+ cpu: [wasm32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-vfpG71OB0ijtjemp3WTdmBKJm9R70KM8vsSExMsIQtV0lVzP07oM1CW6JbNRPXNLhRoue9ofYLiUDk8bE0Hckg==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-hGPyPW60YSpOSgzfy68DLBHgi6HxkAM+L59ZZZPMQ0TOXjQg+p2EW87+TjZfJOkSpbYiEkULwa/f4a2hcVjsqQ==}+ engines: {node: '>= 10'}+ cpu: [ia32]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-L6Ed1DxXK9YSCMyvpR8MiNAyKNkQLjsHsHK9E0qnHa8NzLFqzDKhvs5LfnWxM2kJ+F7m/e5n9zPm24kHb3LsVw==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-7cmzIu+Vbupriudo7UudoMRH2OA3cTw67vva8MxeoAe5S7vPFI7z0vp0pMXiA25S8IUJefImQ90FeJjl8fjEaQ==}+ engines: {node: '>= 10'}++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-rZxtMsLwjdXkMUGC3WwsPwLNVqVqnTJT6MNIB6e+5fhMcSCPP0AOsNWuMQ5mdCq6HNjs/ZeWAEchpqeprqBD2Q==}++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-lr07E/l571Gft5v4aA1dI8koJEmF1F0UigBbsqg9OWNzg80H3lDPO+auv85y3T/NHE3GirDk7x/D3sLO57vayw==}+ engines: {node: '>= 10'}+ cpu: [arm]+ os: [android]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-WDR7S+aRLV6LtBJAg5fmjKkTZIdrEnnQxgdsb7Cf8pYiMWBHLU+LC49OUVppQ2YSPY0+GeYm9yuZWW3kLjJ7Bg==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [android]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-qWTI+EEkiN0oIn/N2gQo7+TVYil+AJ20jjuzD2vATS6uIjVz+Updeqmszi7zq7rdFTLp6Ea3/z4kDKIfZwmR9g==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [darwin]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-bA6hubqtHROR5UI3tToAF/c6TDmaAgF0SWgo4rADHtQ4wdn0JeogvOk50gs2TYVhKPE2ZD2+qqt7oBKB+sxW3A==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [darwin]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-90+KLBkD9hZEjPQW1MDfwSt5J1L46EUKacpCZWyRuL6iIEO5CgWU0V/JnEgFsDOGyyYtiTvHc5bUdUTWd4I9Vg==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [freebsd]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-rG0QlS65x9K/u3HrKafDf8cFKj5wV2JHGfl8abWgKew0GVPyp6vfsDweOwHbWAjcHtp2LHi6JHoW80/MTHm52Q==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-jAasbIvjZXCgX0TCuEFQr+4D6Lla/3AAVx2LmDuMjgG4xoIXzjKWl7c4chuaD+TI+prWT0X6LJcdzFT+ROKGHQ==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-Plgk5rPqqK2nocBGajkMVbGm010Z7dnUgq0wtnYRZbzWWxwWcXfZMPa8EYxrK4eE8SzpI7VlZP1tdVsdjgGwMw==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-GW7AzGuWxtQkyHknHWYFdR0CHmW6is8rG2Rf4V6GNmMpmwtXt/ItWYWtBe4zqJWycMNazpfZKSw/BpT7/MVCXQ==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [linux]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-/nQVSTrqSsn7YdAc2R7Ips/tnw5SPUcl3D7QrXCNGPqjbatIspnaexvaOYNyKMU6xPu+pc0BTnKVmqhlJJCPLA==}+ engines: {node: '>=14.0.0'}+ cpu: [wasm32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-PFi7oJIBu5w7Qzh3dwFea3sHRO3pojMsaEnUIy22QvsW+UJfNQwJCryVrpoUt8m4QyZXI+saEq/0r4GwdoHYFQ==}+ engines: {node: '>= 10'}+ cpu: [arm64]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-gXkuYzxQsgkj05Zaq+KQTkHIN83dFAwMcTKa2aQcpYPRImFm2AQzEyLtpXmyCWzJ0F9ZYAOmbSyrNew8/us6bw==}+ engines: {node: '>= 10'}+ cpu: [ia32]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-rEAf05nol3e3eei2sRButmgXP+6ATgm0/38MKhz9Isne82T4rPIMYsCIFj0kOisaGeVwoi2fnm7O9oWp5YVnYQ==}+ engines: {node: '>= 10'}+ cpu: [x64]+ os: [win32]++ '@napi-rs/[email protected]':+ resolution: {integrity: sha512-enkZYyuCdo+9jneCPE/0fjIta4wWnvVN9hBo2HuiMpRF0q3lzv1J6b/cl7i0mxZUKhBrV3aCKDBQnCOhwKbPmQ==}+ engines: {node: '>= 10'}+'@noble/[email protected]':resolution: {integrity: sha512-rONPWMC7PeExE077uLE4oqWrZ1IvAfz3oH9LibVAcVCopJiA9R62uavnbEzdkVmJYI6M6Zgkbeb07+tWjlq2XA==}engines: {node: ^14.21.3 || >=16}@@ -1447,6 +1962,58 @@ packages:resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==}engines: {node: ^14.21.3 || >=16}+ '@octokit/[email protected]':+ resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==}+ engines: {node: '>= 20'}++ '@octokit/[email protected]':+ resolution: {integrity: sha512-oNXsh2ywth5aowwIa7RKtawnkdH6LgU1ztfP9AIUCQCvzysB+WeU8o2kyyosDPwBZutPpjZDKPQGIzzrfTWweQ==}+ engines: {node: '>= 20'}++ '@octokit/[email protected]':+ resolution: {integrity: sha512-hoYicJZaqISMAI3JfaDr1qMNi48OctWuOih1m80bkYow/ayPw6Jj52tqWJ6GEoFTk1gBqfanSoI1iY99Z5+ekQ==}+ engines: {node: '>= 20'}++ '@octokit/[email protected]':+ resolution: {integrity: sha512-j1nQNU1ZxNFx2ZtKmL4sMrs4egy5h65OMDmSbVyuCzjOcwsHq6EaYjOTGXPQxgfiN8dJ4CriYHk6zF050WEULg==}+ engines: {node: '>= 20'}++ '@octokit/[email protected]':+ resolution: {integrity: sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA==}++ '@octokit/[email protected]':+ resolution: {integrity: sha512-q9iQGlZlxAVNRN2jDNskJW/Cafy7/XE52wjZ5TTvyhyOD904Cvx//DNyoO3J/MXJ0ve3rPoNWKEg5iZrisQSuw==}+ engines: {node: '>= 20'}+ peerDependencies:+ '@octokit/core': '>=6'++ '@octokit/[email protected]':+ resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==}+ engines: {node: '>= 20'}+ peerDependencies:+ '@octokit/core': '>=6'++ '@octokit/[email protected]':+ resolution: {integrity: sha512-kJVUQk6/dx/gRNLWUnAWKFs1kVPn5O5CYZyssyEoNYaFedqZxsfYs7DwI3d67hGz4qOwaJ1dpm07hOAD1BXx6g==}+ engines: {node: '>= 20'}+ peerDependencies:+ '@octokit/core': '>=6'++ '@octokit/[email protected]':+ resolution: {integrity: sha512-KRA7VTGdVyJlh0cP5Tf94hTiYVVqmt2f3I6mnimmaVz4UG3gQV/k4mDJlJv3X67iX6rmN7gSHCF8ssqeMnmhZg==}+ engines: {node: '>= 20'}++ '@octokit/[email protected]':+ resolution: {integrity: sha512-V6jhKokg35vk098iBqp2FBKunk3kMTXlmq+PtbV9Gl3TfskWlebSofU9uunVKhUN7xl+0+i5vt0TGTG8/p/7HA==}+ engines: {node: '>= 20'}++ '@octokit/[email protected]':+ resolution: {integrity: sha512-z6tmTu9BTnw51jYGulxrlernpsQYXpui1RK21vmXn8yF5bp6iX16yfTtJYGK5Mh1qDkvDOmp2n8sRMcQmR8jiA==}+ engines: {node: '>= 20'}++ '@octokit/[email protected]':+ resolution: {integrity: sha512-1y6DgTy8Jomcpu33N+p5w58l6xyt55Ar2I91RPiIA0xCJBXyUAhXCcmZaDWSANiha7R9a6qJJ2CRomGPZ6f46g==}+'@one-ini/[email protected]':resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==}@@ -2091,6 +2658,134 @@ packages:peerDependencies:'@opentelemetry/api': ^1.1.0+ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-Ykkz+xYJ1Hd+vjVSXljyIFRW/ZRMp2tQfPy9T2jj9F2pm5CAchqJHSXEbB3FbhdUXusUqhWLmu70JdVhS4SBcQ==}+ cpu: [arm]+ os: [android]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-csPEZPGTlPtkoMnDeG+hrYX2AJIh8dPKW7TW5E0MdNMIkJ8aFIymBH2QF+xSMsFl9Gpbl5ZxYLkRIrth+Mhzhg==}+ cpu: [arm64]+ os: [android]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-wlURG+gBge3ovdBtMQ/KvyFsLVXnQ4h5vlIxBYMCTkwCzwvfXbcSOLLZMxypOeXPlH/cdVqOigvh9XBbc3KAfg==}+ cpu: [arm64]+ os: [darwin]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-tFlUz54cemNp6N7+suP+uwCXErzol7ibMslxOHbXtGWwRVckV3Bb1rQyCajFSQYCr/du1ZdDCVsTfuPmXQlejg==}+ cpu: [x64]+ os: [darwin]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-yqikvHq0VbnPasRwOkDiMe5dhLZexvZSBzNroBVGbMWNaHBvkP+GP4EFZ3Y5pcNPC3og3xP4J6+N8bbbEZk3vg==}+ cpu: [x64]+ os: [freebsd]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-Qbv937NEH4gNNC1W3Lx6G4Mh5mETMYSf2jbTxCzcfng74BVorpQd0pKB72bvEA0RQtIkXoEfCW24VeyR3yA84Q==}+ cpu: [arm]+ os: [linux]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-w+6SmMWEaVap5RC9axCvg1ZOFdQsNLlLYrYrKX5siZvcVs6jPNyg3O9rbO85tBFPvuAvSLr66mBJwW70MQJv0w==}+ cpu: [arm64]+ os: [linux]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-AZeVCYaECJbQaj6bxCZK77ApbAQUKNOq4wWoRIkVZnn9/ay5wqHLmkvWdOku0aw2WkGiteLp5nCXSDZjw1g0/A==}+ cpu: [arm64]+ os: [linux]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-sz8r3BPLWZC5vZ/5UCb2MhcEFT8bRIeo3IFnksh6K9umAyjgByxnBZP/EyL4twcjQqbrDC64Ox0Pm5vtTIiFWw==}+ cpu: [ppc64]+ os: [linux]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-YLpDVEcsvj03z6SqqcxA1J+ocTR+pP6dv0oIrGXaATwQtx0C+nt6WrVESaXjJpQDAc/5I+LkVDlimeAJb4iLNQ==}+ cpu: [s390x]+ os: [linux]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-FCBaQq4Y08AOLvl83AVzGeRAp/7RI49sDb44sE3/XzxdAVHYyhp0wA84h/NBi8Cj8XhSJdr6KosskHl1cVB0Qg==}+ cpu: [x64]+ os: [linux]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-QKvj6d7VGKK3udW8TXfB0NYNjCloKxZznl58eaoFMoO+bOwuOWJX/8tN37FjkghH4Sb38vRK24+BdqebAHjBdA==}+ cpu: [x64]+ os: [linux]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-k8wqQZ04J4l44v6iB7VFVNc+tz0BBJV2nxSwhLTHpciAFZnOG4BJUqw+2OsObUreR3z5zarebkVFyEJ3QaUH5g==}+ cpu: [arm64]+ os: [openharmony]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-O8Bj67iC0lm0xSlpmHseU+tQytrqdhDFUc0MzYmPSvs0i60u3CNGiw/BWuO6xDpBfhp/IQkWSGBrecZZt1hMFg==}+ engines: {node: '>=14.0.0'}+ cpu: [wasm32]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-mGFu9B3E/jBqvy6FzTZHjUZUOOst1dnN/LIX1Bi8wbSrB+1TdOBykOcNJ3F+BppdaTCWqKb6ieLU6BvvnxboaA==}+ cpu: [arm64]+ os: [win32]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-JEk4P/c/x6T4MHPBtty7IdCx/hxqhdhLrrJ0pIQ8Ue1Kkx74Aq6NDPg6TWtHAxmn+4cw6c1TLzGPXW+1dwbFcA==}+ cpu: [ia32]+ os: [win32]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-6G0YmfxjD3Ec8G619VXfJ3luryJb1Fq649H8ZcfLO8810WLwtzZiAXZCsCLs8lPsLTEpTeFR+AWuc8v4IRHekg==}+ cpu: [x64]+ os: [win32]++ '@oxc-node/[email protected]':+ resolution: {integrity: sha512-2lbEquSd7qU5SZwbu2ngxs/vaa5sgRB5FE6TSPIPp6wfy7+11M0OrzD3g9TxH5CcsawecF2pC8nNpRVjBgBhOg==}++ '@oxlint/[email protected]':+ resolution: {integrity: sha512-rcTw0QWeOc6IeVp+Up7WtcwdS9l4j7TOq4tihF0Ud/fl+VUVdvDCPuZ9QTnLXJhwMXiyQRWdxRyI6XBwf80ncQ==}+ cpu: [arm64]+ os: [darwin]++ '@oxlint/[email protected]':+ resolution: {integrity: sha512-TWFSEmyl2/DN4HoXNwQl0y/y3EXFJDctfv5MiDtVOV1GJKX80cGSIxMxXb08Q3CCWqteqEijmfSMo5TG8X1H/A==}+ cpu: [x64]+ os: [darwin]++ '@oxlint/[email protected]':+ resolution: {integrity: sha512-N1FqdKfwhVWPpMElv8qlGqdEefTbDYaRVhdGWOjs/2f7FESa5vX0cvA7ToqzkoXyXZI5DqByWiPML33njK30Kg==}+ cpu: [arm64]+ os: [linux]++ '@oxlint/[email protected]':+ resolution: {integrity: sha512-v/BPuiateLBb7Gz1STb69EWjkgKdlPQ1NM56z+QQur21ly2hiMkBX2n0zEhqfu9PQVRUizu6AlsYuzcPY/zsIQ==}+ cpu: [arm64]+ os: [linux]++ '@oxlint/[email protected]':+ resolution: {integrity: sha512-gUTp8KIrSYt97dn+tRRC3LKnH4xlHKCwrPwiDcGmLbCxojuN9/H5mnIhPKEfwNuZNdoKGS/ABuq3neVyvRCRtQ==}+ cpu: [x64]+ os: [linux]++ '@oxlint/[email protected]':+ resolution: {integrity: sha512-DpN6cW2HPjYXeENG0JBbmubO8LtfKt6qJqEMBw9gUevbyBaX+k+Jn7sYgh6S23wGOkzmTNphBsf/7ulj4nIVYA==}+ cpu: [x64]+ os: [linux]++ '@oxlint/[email protected]':+ resolution: {integrity: sha512-oXxJksnUTUMgJ0NvjKS1mrCXAy1ttPgIVacRSlxQ+1XHy+aJDMM7I8fsCtoKoEcTIpPaD98eqUqlLYs0H2MGjA==}+ cpu: [arm64]+ os: [win32]++ '@oxlint/[email protected]':+ resolution: {integrity: sha512-iRYy2rhTQKFztyx0jtNMRBnFpzsRwFdjWQ7sKKzJpmbijA3Tw3DCqlGT7QRgoVRF0+X/ccNGvvsrgMohPVfLeQ==}+ cpu: [x64]+ os: [win32]+'@paralleldrive/[email protected]':resolution: {integrity: sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==}@@ -2638,6 +3333,10 @@ packages:peerDependencies:react: ^18 || ^19+ '@taplo/[email protected]':+ resolution: {integrity: sha512-Ck3zFhQhIhi02Hl6T4ZmJsXdnJE+wXcJz5f8klxd4keRYgenMnip3JDPMGDRLbnC/2iGd8P0sBIQqI3KxfVjBg==}+ hasBin: true+'@tootallnate/[email protected]':resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==}engines: {node: '>= 10'}@@ -2657,6 +3356,9 @@ packages:'@tsconfig/[email protected]':resolution: {integrity: sha512-0IKu9GHYF1NGTJiYgfWwqnOQSlnE9V9R7YohHNNf0/fj/SyOZWzdd06JFr0fLpg1Mqw0kGbYg8w5xdkSqLKM9g==}+ '@tybys/[email protected]':+ resolution: {integrity: sha512-VyyPYFlOMNylG45GoAe0xDoLwWuowvf92F9kySqzYh8vmYm7D2u4iUJKa1tOUpS70Ku13ASrOkS4ScXFsTaCNQ==}+'@types/[email protected]':resolution: {integrity: sha512-AX+AbjH/rH5ezX1fbK8onC/a+HyQHo7QGmvoxAE42n22OsciAxvZoZNEr22tbXs8WfP1nIsBjKDpgPm3HjOZbA==}@@ -3161,6 +3863,9 @@ packages:resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}+ resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==}+resolution: {integrity: sha512-y/ie+Faknx7sZA5MfGA2xKlu0GDv8RWrXGsmlteyJQ2lvoKv9GBK/fpRMc2qlSoBAgNxrixICFCBefIq8WCQpQ==}@@ -3298,10 +4003,6 @@ packages:resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}engines: {node: '>=10'}- resolution: {integrity: sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==}- engines: {node: ^12.17.0 || ^14.13 || >=16.0.0}-resolution: {integrity: sha512-46QrSQFyVSEyYAgQ22hQ+zDa60YHA4fBstHmtSApj1Y5vKtG27fWowW03jCk5KcbXEWPZUIR894aARCA/G1kfQ==}engines: {node: ^12.17.0 || ^14.13 || >=16.0.0}@@ -3310,6 +4011,9 @@ packages:resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==}engines: {node: '>=10'}+ resolution: {integrity: sha512-bNFETTG/pM5ryzQ9Ad0lJOTa6HWD/YsScAR3EnCPZRPlQh77JocYktSHOUHelyhm8IARL+o4c4F1bP5KVOjiRA==}+resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==}@@ -3343,6 +4047,15 @@ packages:resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==}engines: {node: '>=18'}+ resolution: {integrity: sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==}+ engines: {node: '>= 12'}++ resolution: {integrity: sha512-CXkMQxU6s9GklO/1f714dkKBMu1lopS1WFF0B8o4AxPykR1hpozxSiUZ5ZUeBjfPgCWqbcNOtZVFhB8Lkfp1+Q==}+ peerDependencies:+ typanion: '*'+resolution: {integrity: sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==}@@ -3793,6 +4506,9 @@ packages:resolution: {integrity: sha512-X13Q/ZSc+vsO1q600bvNK4bxgXMkHcf//RxCmYDaRY5DAcT+eoXjY5hoAPGMdRnWQjvyLEcyauG3b6hz76LNqg==}+ resolution: {integrity: sha512-E0iGnTtbDhkeczB0T+mxmoVlT4YNweEKBLq7oaU4p11mecdsZpNWOglI4895Vh4usbQ+LsJiuLuI2L0Vdmfm2w==}+resolution: {integrity: sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==}@@ -3905,6 +4621,9 @@ packages:resolution: {integrity: sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==}engines: {node: '> 0.1.90'}+ resolution: {integrity: sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg==}+resolution: {integrity: sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==}@@ -3966,6 +4685,10 @@ packages:resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}engines: {node: '>=8'}+ resolution: {integrity: sha512-YyZM99iHrqLKjmt4LJDj58KI+fYyufRLBSYcqycxf//KpBk9FoewoGX0450m9nB44qrZnovzC2oeP5hUibxc/g==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==}@@ -4361,6 +5084,10 @@ packages:resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}+ resolution: {integrity: sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==}+ engines: {node: '>=16'}+resolution: {integrity: sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==}@@ -4601,6 +5328,10 @@ packages:resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}+ resolution: {integrity: sha512-lR4MXjGNgkJc7tkQ97kb2nuEMnNCyU//XYVH0MKTGcXEiSudQ5MKGKen3C5QubYy0vmq+JGitUg92uuywGEwIA==}+ engines: {node: ^18.17.0 || >=20.5.0}+resolution: {integrity: sha512-NEdLrtrq1jUZyfjkr9OCz9EzCNhnRyWtt1PAnvnhwy6e8XETS0Dtc+ZNCO2gvuAoKsIn2+vCSowXTYE4CkgnAQ==}engines: {node: '>=10.0.0'}@@ -4718,6 +5449,10 @@ packages:resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==}engines: {node: '>=8'}+ resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==}+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}+resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==}@@ -4825,6 +5560,10 @@ packages:resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}engines: {node: '>= 0.6'}+ resolution: {integrity: sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==}+ engines: {node: '>= 0.10.0'}+resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==}@@ -4941,6 +5680,10 @@ packages:resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==}hasBin: true+ resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==}+ engines: {node: ^18.17.0 || >=20.5.0}+resolution: {integrity: sha512-21t+ozMQDAL/UGgQVBbZ/xXvNO10++ZPuTmKRO8k9V3AClVRht49ahtDjfY8l1q6nSHOrE5ASfthzH3ol6R/hg==}engines: {node: '>=20.17'}@@ -5023,6 +5766,11 @@ packages:resolution: {integrity: sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==}engines: {node: ^18.17.0 || >=20.5.0}+ resolution: {integrity: sha512-wdbB5My48XKp2ZfJUlhnLVihzeuA1hgBnqB2J9ahV77wLS+/YAJAlN8I+X3DIFIPZ3m5L7nplmlbhNiFDmXRDA==}+ engines: {node: ^20.5.0 || >=22.0.0, npm: '>= 10'}+ hasBin: true+resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==}engines: {node: '>=8'}@@ -5119,6 +5867,16 @@ packages:typescript:optional: true+ resolution: {integrity: sha512-oo0nq3zF9hmgATGc9esoMahLuEESOodUxEDeHDA2K7tbYcSfcmReE9G2QNppnq9rOSQHLTwlMtzGAjjttYaufQ==}+ engines: {node: '>=8.*'}+ hasBin: true+ peerDependencies:+ oxlint-tsgolint: '>=0.1.5'+ peerDependenciesMeta:+ oxlint-tsgolint:+ optional: true+resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==}engines: {node: '>=6'}@@ -5127,10 +5885,18 @@ packages:resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==}engines: {node: '>=10'}+ resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==}+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}+resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==}engines: {node: '>=8'}+ resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==}+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}+resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}engines: {node: '>=6'}@@ -5171,6 +5937,10 @@ packages:resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}engines: {node: '>=8'}+ resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==}+ engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}+resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}engines: {node: '>=0.10.0'}@@ -5226,6 +5996,10 @@ packages:resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}engines: {node: '>=8.6'}+ resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==}+ engines: {node: '>=12'}+resolution: {integrity: sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==}engines: {node: '>=0.10'}@@ -5249,8 +6023,8 @@ packages:resolution: {integrity: sha512-dMACeu63HtRLmCG8VKdy4cShCPKaYDR4youZqoSWLxl5Gu99HUw8bw75thbPv9Nip+H+QYX8o3ZJbTdVZZ2TVg==}hasBin: true- resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==}+ resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==}engines: {node: '>= 6'}@@ -5432,6 +6206,10 @@ packages:resolution: {integrity: sha512-SEbJV7tohp3DAAILbEMPXavBjAnMN0tVnh4+9G8ihV4Pq3HYF9h8QNez9zkJ1ILkv9G2BjdzwctznGZXgu/HGw==}engines: {node: ^18.17.0 || >=20.5.0}+ resolution: {integrity: sha512-qpt8EwugBWDw2cgE2W+/3oxC+KTez2uSVR8JU9Q36TXPAGCaozfQUs59v4j4GFpWTaw0i6hAZSvOmu1J0uOEUg==}+ engines: {node: ^18.17.0 || >=20.5.0}+resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}@@ -5581,6 +6359,11 @@ packages:engines: {node: '>=10'}hasBin: true+ resolution: {integrity: sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==}+ engines: {node: '>=10'}+ hasBin: true+resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==}engines: {node: '>= 0.8.0'}@@ -5615,6 +6398,10 @@ packages:resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}engines: {node: '>=8'}+ resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw==}engines: {node: '>= 4.0.0'}@@ -5986,6 +6773,9 @@ packages:resolution: {integrity: sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==}+ resolution: {integrity: sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==}+resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==}engines: {node: '>=4'}@@ -6007,6 +6797,11 @@ packages:engines: {node: '>=14.17'}hasBin: true+ resolution: {integrity: sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==}+ engines: {node: '>=14.17'}+ hasBin: true+resolution: {integrity: sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA==}@@ -6026,6 +6821,13 @@ packages:resolution: {integrity: sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw==}engines: {node: '>=20.18.1'}+ resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==}+ engines: {node: '>=18'}++ resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==}+resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==}engines: {node: '>= 4.0.0'}@@ -6259,6 +7061,11 @@ packages:engines: {node: '>= 8'}hasBin: true+ resolution: {integrity: sha512-JEdGzHwwkrbWoGOlIHqQ5gtprKGOenpDHpxE9zVR1bWbOtYRyPPHMe9FaP6x61CmNaTThSkb0DAJte5jD+DmzQ==}+ engines: {node: ^18.17.0 || >=20.5.0}+ hasBin: true+resolution: {integrity: sha512-qxSTKswC6llEMZKgCQdaWgDuMJQnhuvF5f2Nk3SNXc4byfQ+voo2mX1Px9dkNOuR8p0KAjfPG29PuYUSIb+vSA==}engines: {node: '>= 12.0.0'}@@ -6442,6 +7249,14 @@ packages:resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}engines: {node: '>=10'}+ resolution: {integrity: sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==}+ engines: {node: '>=12.20'}++ resolution: {integrity: sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==}peerDependencies:@@ -6693,17 +7508,17 @@ snapshots:'@aws-sdk/types': 3.679.0'@aws-sdk/util-locate-window': 3.679.0'@smithy/util-utf8': 2.3.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-crypto/[email protected]':dependencies:'@aws-crypto/util': 5.2.0'@aws-sdk/types': 3.679.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-crypto/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@aws-crypto/[email protected]':dependencies:@@ -6715,7 +7530,7 @@ snapshots:dependencies:'@aws-sdk/types': 3.679.0'@smithy/util-utf8': 2.3.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:@@ -6759,7 +7574,7 @@ snapshots:'@smithy/util-middleware': 3.0.8'@smithy/util-retry': 3.0.8'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- aws-crt@@ -6807,7 +7622,7 @@ snapshots:'@smithy/util-utf8': 3.0.0'@smithy/util-waiter': 3.1.7'@types/uuid': 9.0.8- tslib: 2.6.3+ tslib: 2.8.1uuid: 9.0.1transitivePeerDependencies:- aws-crt@@ -6853,7 +7668,7 @@ snapshots:'@smithy/util-middleware': 3.0.8'@smithy/util-retry': 3.0.8'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- aws-crt@@ -6896,7 +7711,7 @@ snapshots:'@smithy/util-middleware': 3.0.8'@smithy/util-retry': 3.0.8'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- aws-crt@@ -6941,7 +7756,7 @@ snapshots:'@smithy/util-middleware': 3.0.8'@smithy/util-retry': 3.0.8'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- aws-crt@@ -6957,7 +7772,7 @@ snapshots:'@smithy/types': 3.6.0'@smithy/util-middleware': 3.0.8fast-xml-parser: 4.4.1- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:@@ -6965,7 +7780,7 @@ snapshots:'@aws-sdk/types': 3.679.0'@smithy/property-provider': 3.1.8'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- aws-crt@@ -6975,7 +7790,7 @@ snapshots:'@aws-sdk/types': 3.679.0'@smithy/property-provider': 3.1.8'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:@@ -6988,7 +7803,7 @@ snapshots:'@smithy/smithy-client': 3.4.2'@smithy/types': 3.6.0'@smithy/util-stream': 3.2.1- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected](@aws-sdk/[email protected](@aws-sdk/[email protected]))(@aws-sdk/[email protected])':dependencies:@@ -7004,7 +7819,7 @@ snapshots:'@smithy/property-provider': 3.1.8'@smithy/shared-ini-file-loader': 3.1.9'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- '@aws-sdk/client-sso-oidc'- aws-crt@@ -7022,7 +7837,7 @@ snapshots:'@smithy/property-provider': 3.1.8'@smithy/shared-ini-file-loader': 3.1.9'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- '@aws-sdk/client-sso-oidc'- '@aws-sdk/client-sts'@@ -7035,7 +7850,7 @@ snapshots:'@smithy/property-provider': 3.1.8'@smithy/shared-ini-file-loader': 3.1.9'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1dependencies:@@ -7046,7 +7861,7 @@ snapshots:'@smithy/property-provider': 3.1.8'@smithy/shared-ini-file-loader': 3.1.9'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- '@aws-sdk/client-sso-oidc'- aws-crt@@ -7058,7 +7873,7 @@ snapshots:'@aws-sdk/types': 3.679.0'@smithy/property-provider': 3.1.8'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1dependencies:@@ -7078,7 +7893,7 @@ snapshots:'@smithy/credential-provider-imds': 3.2.5'@smithy/property-provider': 3.1.8'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1transitivePeerDependencies:- '@aws-sdk/client-sso-oidc'- aws-crt@@ -7088,20 +7903,20 @@ snapshots:'@aws-sdk/types': 3.679.0'@smithy/protocol-http': 4.1.5'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:'@aws-sdk/types': 3.679.0'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:'@aws-sdk/types': 3.679.0'@smithy/protocol-http': 4.1.5'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:@@ -7111,12 +7926,12 @@ snapshots:'@smithy/core': 2.5.1'@smithy/protocol-http': 4.1.5'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:'@smithy/protocol-http': 1.2.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:@@ -7125,12 +7940,12 @@ snapshots:'@smithy/types': 3.6.0'@smithy/util-config-provider': 3.0.0'@smithy/util-middleware': 3.0.8- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:'@smithy/signature-v4': 1.1.0- tslib: 2.6.3+ tslib: 2.8.1dependencies:@@ -7139,30 +7954,30 @@ snapshots:'@smithy/property-provider': 3.1.8'@smithy/shared-ini-file-loader': 3.1.9'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:'@aws-sdk/types': 3.679.0'@smithy/types': 3.6.0'@smithy/util-endpoints': 2.1.4- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:'@aws-sdk/types': 3.679.0'@smithy/types': 3.6.0bowser: 2.11.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:@@ -7170,11 +7985,11 @@ snapshots:'@aws-sdk/types': 3.679.0'@smithy/node-config-provider': 3.1.9'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@aws-sdk/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@babel/[email protected]':dependencies:@@ -7548,6 +8363,19 @@ snapshots:dependencies:'@noble/ciphers': 1.3.0+ '@emnapi/[email protected]':+ dependencies:+ '@emnapi/wasi-threads': 1.1.0+ tslib: 2.8.1++ '@emnapi/[email protected]':+ dependencies:+ tslib: 2.8.1++ '@emnapi/[email protected]':+ dependencies:+ tslib: 2.8.1+'@esbuild/[email protected]':optional: true@@ -7688,6 +8516,129 @@ snapshots:protobufjs: 7.5.3yargs: 17.7.2+ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/figures': 1.0.13+ '@inquirer/type': 3.0.8(@types/[email protected])+ ansi-escapes: 4.3.2+ yoctocolors-cjs: 2.1.3+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/type': 3.0.8(@types/[email protected])+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/figures': 1.0.13+ '@inquirer/type': 3.0.8(@types/[email protected])+ ansi-escapes: 4.3.2+ cli-width: 4.1.0+ mute-stream: 2.0.0+ signal-exit: 4.1.0+ wrap-ansi: 6.2.0+ yoctocolors-cjs: 2.1.3+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/external-editor': 1.0.1(@types/[email protected])+ '@inquirer/type': 3.0.8(@types/[email protected])+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/type': 3.0.8(@types/[email protected])+ yoctocolors-cjs: 2.1.3+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ chardet: 2.1.0+ iconv-lite: 0.6.3+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected]': {}++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/type': 3.0.8(@types/[email protected])+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/type': 3.0.8(@types/[email protected])+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/type': 3.0.8(@types/[email protected])+ ansi-escapes: 4.3.2+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/checkbox': 4.2.2(@types/[email protected])+ '@inquirer/confirm': 5.1.16(@types/[email protected])+ '@inquirer/editor': 4.2.18(@types/[email protected])+ '@inquirer/expand': 4.0.18(@types/[email protected])+ '@inquirer/input': 4.2.2(@types/[email protected])+ '@inquirer/number': 3.0.18(@types/[email protected])+ '@inquirer/password': 4.0.18(@types/[email protected])+ '@inquirer/rawlist': 4.1.6(@types/[email protected])+ '@inquirer/search': 3.1.1(@types/[email protected])+ '@inquirer/select': 4.3.2(@types/[email protected])+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/type': 3.0.8(@types/[email protected])+ yoctocolors-cjs: 2.1.3+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/figures': 1.0.13+ '@inquirer/type': 3.0.8(@types/[email protected])+ yoctocolors-cjs: 2.1.3+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.2.0(@types/[email protected])+ '@inquirer/figures': 1.0.13+ '@inquirer/type': 3.0.8(@types/[email protected])+ ansi-escapes: 4.3.2+ yoctocolors-cjs: 2.1.3+ optionalDependencies:+ '@types/node': 20.14.1++ '@inquirer/[email protected](@types/[email protected])':+ optionalDependencies:+ '@types/node': 20.14.1+'@ioredis/[email protected]': {}'@isaacs/[email protected]':@@ -7860,7 +8811,7 @@ snapshots:jest-regex-util: 29.6.3jest-util: 29.7.0micromatch: 4.0.8- pirates: 4.0.6+ pirates: 4.0.7slash: 3.0.0write-file-atomic: 4.0.2transitivePeerDependencies:@@ -8030,7 +8981,7 @@ snapshots:'@ethereumjs/tx': 4.2.0'@types/debug': 4.1.12debug: 4.4.1- semver: 7.6.2+ semver: 7.7.2superstruct: 1.0.4transitivePeerDependencies:- supports-color@@ -8044,7 +8995,7 @@ snapshots:'@types/debug': 4.1.12debug: 4.4.1pony-cause: 2.1.11- semver: 7.6.2+ semver: 7.7.2uuid: 9.0.1transitivePeerDependencies:- supports-color@@ -8058,36 +9009,280 @@ snapshots:'@types/debug': 4.1.12debug: 4.4.1pony-cause: 2.1.11- semver: 7.6.2+ semver: 7.7.2uuid: 9.0.1transitivePeerDependencies:- supports-color- '@mixmark-io/[email protected]': {}+ '@mixmark-io/[email protected]': {}++ '@msgpackr-extract/[email protected]':+ optional: true++ '@msgpackr-extract/[email protected]':+ optional: true++ '@msgpackr-extract/[email protected]':+ optional: true++ '@msgpackr-extract/[email protected]':+ optional: true++ '@msgpackr-extract/[email protected]':+ optional: true++ '@msgpackr-extract/[email protected]':+ optional: true++ '@nangohq/[email protected]':+ dependencies:+ axios: 1.11.0+ transitivePeerDependencies:+ - debug++ dependencies:+ '@inquirer/prompts': 7.8.4(@types/[email protected])+ '@napi-rs/cross-toolchain': 1.0.3+ '@napi-rs/wasm-tools': 1.0.1+ '@octokit/rest': 22.0.0+ clipanion: 4.0.0-rc.4([email protected])+ colorette: 2.0.20+ debug: 4.4.1+ es-toolkit: 1.39.10+ find-up: 7.0.0+ js-yaml: 4.1.0+ semver: 7.7.2+ typanion: 3.14.0+ optionalDependencies:+ '@emnapi/runtime': 1.5.0+ transitivePeerDependencies:+ - '@napi-rs/cross-toolchain-arm64-target-aarch64'+ - '@napi-rs/cross-toolchain-arm64-target-armv7'+ - '@napi-rs/cross-toolchain-arm64-target-ppc64le'+ - '@napi-rs/cross-toolchain-arm64-target-s390x'+ - '@napi-rs/cross-toolchain-arm64-target-x86_64'+ - '@napi-rs/cross-toolchain-x64-target-aarch64'+ - '@napi-rs/cross-toolchain-x64-target-armv7'+ - '@napi-rs/cross-toolchain-x64-target-ppc64le'+ - '@napi-rs/cross-toolchain-x64-target-s390x'+ - '@napi-rs/cross-toolchain-x64-target-x86_64'+ - '@types/node'+ - supports-color++ '@napi-rs/[email protected]':+ dependencies:+ '@napi-rs/lzma': 1.4.5+ '@napi-rs/tar': 1.1.0+ debug: 4.4.1+ transitivePeerDependencies:+ - supports-color++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ dependencies:+ '@napi-rs/wasm-runtime': 1.0.3+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optionalDependencies:+ '@napi-rs/lzma-android-arm-eabi': 1.4.5+ '@napi-rs/lzma-android-arm64': 1.4.5+ '@napi-rs/lzma-darwin-arm64': 1.4.5+ '@napi-rs/lzma-darwin-x64': 1.4.5+ '@napi-rs/lzma-freebsd-x64': 1.4.5+ '@napi-rs/lzma-linux-arm-gnueabihf': 1.4.5+ '@napi-rs/lzma-linux-arm64-gnu': 1.4.5+ '@napi-rs/lzma-linux-arm64-musl': 1.4.5+ '@napi-rs/lzma-linux-ppc64-gnu': 1.4.5+ '@napi-rs/lzma-linux-riscv64-gnu': 1.4.5+ '@napi-rs/lzma-linux-s390x-gnu': 1.4.5+ '@napi-rs/lzma-linux-x64-gnu': 1.4.5+ '@napi-rs/lzma-linux-x64-musl': 1.4.5+ '@napi-rs/lzma-wasm32-wasi': 1.4.5+ '@napi-rs/lzma-win32-arm64-msvc': 1.4.5+ '@napi-rs/lzma-win32-ia32-msvc': 1.4.5+ '@napi-rs/lzma-win32-x64-msvc': 1.4.5++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ dependencies:+ '@napi-rs/wasm-runtime': 1.0.3+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optionalDependencies:+ '@napi-rs/tar-android-arm-eabi': 1.1.0+ '@napi-rs/tar-android-arm64': 1.1.0+ '@napi-rs/tar-darwin-arm64': 1.1.0+ '@napi-rs/tar-darwin-x64': 1.1.0+ '@napi-rs/tar-freebsd-x64': 1.1.0+ '@napi-rs/tar-linux-arm-gnueabihf': 1.1.0+ '@napi-rs/tar-linux-arm64-gnu': 1.1.0+ '@napi-rs/tar-linux-arm64-musl': 1.1.0+ '@napi-rs/tar-linux-ppc64-gnu': 1.1.0+ '@napi-rs/tar-linux-s390x-gnu': 1.1.0+ '@napi-rs/tar-linux-x64-gnu': 1.1.0+ '@napi-rs/tar-linux-x64-musl': 1.1.0+ '@napi-rs/tar-wasm32-wasi': 1.1.0+ '@napi-rs/tar-win32-arm64-msvc': 1.1.0+ '@napi-rs/tar-win32-ia32-msvc': 1.1.0+ '@napi-rs/tar-win32-x64-msvc': 1.1.0++ '@napi-rs/[email protected]':+ dependencies:+ '@emnapi/core': 1.5.0+ '@emnapi/runtime': 1.5.0+ '@tybys/wasm-util': 0.10.0+ optional: true++ '@napi-rs/[email protected]':+ optional: true- '@msgpackr-extract/[email protected]':+ '@napi-rs/[email protected]':optional: true- '@msgpackr-extract/[email protected]':+ '@napi-rs/[email protected]':optional: true- '@msgpackr-extract/[email protected]':+ '@napi-rs/[email protected]':optional: true- '@msgpackr-extract/[email protected]':+ '@napi-rs/[email protected]':optional: true- '@msgpackr-extract/[email protected]':+ '@napi-rs/[email protected]':optional: true- '@msgpackr-extract/[email protected]':+ '@napi-rs/[email protected]':optional: true- '@nangohq/[email protected]':+ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':dependencies:- axios: 1.11.0- transitivePeerDependencies:- - debug+ '@napi-rs/wasm-runtime': 1.0.3+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optional: true++ '@napi-rs/[email protected]':+ optionalDependencies:+ '@napi-rs/wasm-tools-android-arm-eabi': 1.0.1+ '@napi-rs/wasm-tools-android-arm64': 1.0.1+ '@napi-rs/wasm-tools-darwin-arm64': 1.0.1+ '@napi-rs/wasm-tools-darwin-x64': 1.0.1+ '@napi-rs/wasm-tools-freebsd-x64': 1.0.1+ '@napi-rs/wasm-tools-linux-arm64-gnu': 1.0.1+ '@napi-rs/wasm-tools-linux-arm64-musl': 1.0.1+ '@napi-rs/wasm-tools-linux-x64-gnu': 1.0.1+ '@napi-rs/wasm-tools-linux-x64-musl': 1.0.1+ '@napi-rs/wasm-tools-wasm32-wasi': 1.0.1+ '@napi-rs/wasm-tools-win32-arm64-msvc': 1.0.1+ '@napi-rs/wasm-tools-win32-ia32-msvc': 1.0.1+ '@napi-rs/wasm-tools-win32-x64-msvc': 1.0.1'@noble/[email protected]': {}@@ -8121,6 +9316,68 @@ snapshots:'@noble/[email protected]': {}+ '@octokit/[email protected]': {}++ '@octokit/[email protected]':+ dependencies:+ '@octokit/auth-token': 6.0.0+ '@octokit/graphql': 9.0.1+ '@octokit/request': 10.0.3+ '@octokit/request-error': 7.0.0+ '@octokit/types': 14.1.0+ before-after-hook: 4.0.0+ universal-user-agent: 7.0.3++ '@octokit/[email protected]':+ dependencies:+ '@octokit/types': 14.1.0+ universal-user-agent: 7.0.3++ '@octokit/[email protected]':+ dependencies:+ '@octokit/request': 10.0.3+ '@octokit/types': 14.1.0+ universal-user-agent: 7.0.3++ '@octokit/[email protected]': {}++ '@octokit/[email protected](@octokit/[email protected])':+ dependencies:+ '@octokit/core': 7.0.3+ '@octokit/types': 14.1.0++ '@octokit/[email protected](@octokit/[email protected])':+ dependencies:+ '@octokit/core': 7.0.3++ '@octokit/[email protected](@octokit/[email protected])':+ dependencies:+ '@octokit/core': 7.0.3+ '@octokit/types': 14.1.0++ '@octokit/[email protected]':+ dependencies:+ '@octokit/types': 14.1.0++ '@octokit/[email protected]':+ dependencies:+ '@octokit/endpoint': 11.0.0+ '@octokit/request-error': 7.0.0+ '@octokit/types': 14.1.0+ fast-content-type-parse: 3.0.0+ universal-user-agent: 7.0.3++ '@octokit/[email protected]':+ dependencies:+ '@octokit/core': 7.0.3+ '@octokit/plugin-paginate-rest': 13.1.1(@octokit/[email protected])+ '@octokit/plugin-request-log': 6.0.0(@octokit/[email protected])+ '@octokit/plugin-rest-endpoint-methods': 16.0.0(@octokit/[email protected])++ '@octokit/[email protected]':+ dependencies:+ '@octokit/openapi-types': 25.1.0+'@one-ini/[email protected]': {}'@openrouter/[email protected]([email protected])':@@ -9041,6 +10298,105 @@ snapshots:'@opentelemetry/api': 1.9.0'@opentelemetry/core': 2.0.1(@opentelemetry/[email protected])+ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ dependencies:+ '@napi-rs/wasm-runtime': 1.0.3+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ optional: true++ '@oxc-node/[email protected]':+ dependencies:+ pirates: 4.0.7+ optionalDependencies:+ '@oxc-node/core-android-arm-eabi': 0.0.32+ '@oxc-node/core-android-arm64': 0.0.32+ '@oxc-node/core-darwin-arm64': 0.0.32+ '@oxc-node/core-darwin-x64': 0.0.32+ '@oxc-node/core-freebsd-x64': 0.0.32+ '@oxc-node/core-linux-arm-gnueabihf': 0.0.32+ '@oxc-node/core-linux-arm64-gnu': 0.0.32+ '@oxc-node/core-linux-arm64-musl': 0.0.32+ '@oxc-node/core-linux-ppc64-gnu': 0.0.32+ '@oxc-node/core-linux-s390x-gnu': 0.0.32+ '@oxc-node/core-linux-x64-gnu': 0.0.32+ '@oxc-node/core-linux-x64-musl': 0.0.32+ '@oxc-node/core-openharmony-arm64': 0.0.32+ '@oxc-node/core-wasm32-wasi': 0.0.32+ '@oxc-node/core-win32-arm64-msvc': 0.0.32+ '@oxc-node/core-win32-ia32-msvc': 0.0.32+ '@oxc-node/core-win32-x64-msvc': 0.0.32++ '@oxlint/[email protected]':+ optional: true++ '@oxlint/[email protected]':+ optional: true++ '@oxlint/[email protected]':+ optional: true++ '@oxlint/[email protected]':+ optional: true++ '@oxlint/[email protected]':+ optional: true++ '@oxlint/[email protected]':+ optional: true++ '@oxlint/[email protected]':+ optional: true++ '@oxlint/[email protected]':+ optional: true+'@paralleldrive/[email protected]':dependencies:'@noble/hashes': 1.8.0@@ -9545,7 +10901,7 @@ snapshots:'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9553,7 +10909,7 @@ snapshots:'@smithy/types': 3.6.0'@smithy/util-config-provider': 3.0.0'@smithy/util-middleware': 3.0.8- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9564,7 +10920,7 @@ snapshots:'@smithy/util-middleware': 3.0.8'@smithy/util-stream': 3.2.1'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9572,14 +10928,14 @@ snapshots:'@smithy/property-provider': 3.1.8'@smithy/types': 3.6.0'@smithy/url-parser': 3.0.8- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@aws-crypto/crc32': 3.0.0'@smithy/types': 1.2.0'@smithy/util-hex-encoding': 1.1.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9587,7 +10943,7 @@ snapshots:'@smithy/querystring-builder': 3.0.8'@smithy/types': 3.6.0'@smithy/util-base64': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9595,37 +10951,37 @@ snapshots:'@smithy/querystring-builder': 3.0.8'@smithy/types': 3.6.0'@smithy/util-base64': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0'@smithy/util-buffer-from': 3.0.0'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/protocol-http': 4.1.5'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9636,7 +10992,7 @@ snapshots:'@smithy/types': 3.6.0'@smithy/url-parser': 3.0.8'@smithy/util-middleware': 3.0.8- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9647,25 +11003,25 @@ snapshots:'@smithy/types': 3.6.0'@smithy/util-middleware': 3.0.8'@smithy/util-retry': 3.0.8- tslib: 2.6.3+ tslib: 2.8.1uuid: 9.0.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/property-provider': 3.1.8'@smithy/shared-ini-file-loader': 3.1.9'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9673,33 +11029,33 @@ snapshots:'@smithy/protocol-http': 4.1.5'@smithy/querystring-builder': 3.0.8'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 1.2.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0'@smithy/util-uri-escape': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9708,7 +11064,7 @@ snapshots:'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9719,7 +11075,7 @@ snapshots:'@smithy/util-middleware': 1.1.0'@smithy/util-uri-escape': 1.1.0'@smithy/util-utf8': 1.1.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9730,7 +11086,7 @@ snapshots:'@smithy/util-middleware': 3.0.8'@smithy/util-uri-escape': 3.0.0'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9740,54 +11096,54 @@ snapshots:'@smithy/protocol-http': 4.1.5'@smithy/types': 3.6.0'@smithy/util-stream': 3.2.1- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/querystring-parser': 3.0.8'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/util-buffer-from': 3.0.0'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/is-array-buffer': 1.1.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/is-array-buffer': 2.2.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/is-array-buffer': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9795,7 +11151,7 @@ snapshots:'@smithy/smithy-client': 3.4.2'@smithy/types': 3.6.0bowser: 2.11.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9805,36 +11161,36 @@ snapshots:'@smithy/property-provider': 3.1.8'@smithy/smithy-client': 3.4.2'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/node-config-provider': 3.1.9'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/service-error-classification': 3.0.8'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:@@ -9845,36 +11201,36 @@ snapshots:'@smithy/util-buffer-from': 3.0.0'@smithy/util-hex-encoding': 3.0.0'@smithy/util-utf8': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/util-buffer-from': 1.1.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/util-buffer-from': 2.2.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/util-buffer-from': 3.0.0- tslib: 2.6.3+ tslib: 2.8.1'@smithy/[email protected]':dependencies:'@smithy/abort-controller': 3.1.6'@smithy/types': 3.6.0- tslib: 2.6.3+ tslib: 2.8.1'@socket.io/[email protected]': {}@@ -9944,13 +11300,13 @@ snapshots:'@solana/[email protected]([email protected])':dependencies:- chalk: 5.4.1+ chalk: 5.6.0commander: 12.1.0typescript: 5.8.3'@solana/[email protected]([email protected])':dependencies:- chalk: 5.4.1+ chalk: 5.6.0commander: 14.0.0typescript: 5.8.3@@ -10073,6 +11429,8 @@ snapshots:'@tanstack/query-core': 5.83.1react: 18.3.1+ '@taplo/[email protected]': {}+'@tootallnate/[email protected]': {}'@tsconfig/[email protected]': {}@@ -10085,6 +11443,10 @@ snapshots:'@tsconfig/[email protected]': {}+ '@tybys/[email protected]':+ dependencies:+ tslib: 2.8.1+'@types/[email protected]': {}'@types/[email protected]':@@ -11111,6 +12473,8 @@ snapshots:+ [email protected]: {}+@@ -11282,12 +12646,12 @@ snapshots:ansi-styles: 4.3.0supports-color: 7.2.0- [email protected]: {}-+ [email protected]: {}+@@ -11328,6 +12692,12 @@ snapshots:slice-ansi: 5.0.0string-width: 7.2.0+ [email protected]: {}++ dependencies:+ typanion: 3.14.0+dependencies:string-width: 4.2.3@@ -11742,6 +13112,8 @@ snapshots:+ [email protected]: {}+@@ -11922,6 +13294,8 @@ snapshots:+ [email protected]: {}+@@ -11984,6 +13358,12 @@ snapshots:locate-path: 5.0.0path-exists: 4.0.0+ dependencies:+ locate-path: 7.2.0+ path-exists: 5.0.0+ unicorn-magic: 0.1.0+@@ -12419,6 +13799,8 @@ snapshots:+ [email protected]: {}+dependencies:node-fetch: 2.7.0([email protected])@@ -12895,6 +14277,8 @@ snapshots:+ [email protected]: {}+dependencies:'@metamask/safe-event-emitter': 2.0.0@@ -12915,7 +14299,7 @@ snapshots:dependencies:'@types/diff-match-patch': 1.0.36- chalk: 5.4.1+ chalk: 5.6.0diff-match-patch: 1.0.5@@ -13024,6 +14408,10 @@ snapshots:dependencies:p-locate: 4.1.0+ dependencies:+ p-locate: 6.0.0+@@ -13102,7 +14490,7 @@ snapshots:dependencies:- semver: 7.6.2+ semver: 7.7.2@@ -13137,6 +14525,8 @@ snapshots:+ [email protected]: {}+@@ -13226,6 +14616,8 @@ snapshots:+ [email protected]: {}+@@ -13283,6 +14675,17 @@ snapshots:+ dependencies:+ ansi-styles: 6.2.1+ cross-spawn: 7.0.6+ memorystream: 0.3.1+ picomatch: 4.0.3+ pidtree: 0.6.0+ read-package-json-fast: 4.0.0+ shell-quote: 1.8.3+ which: 5.0.0+dependencies:path-key: 3.1.1@@ -13421,6 +14824,17 @@ snapshots:transitivePeerDependencies:- zod+ optionalDependencies:+ '@oxlint/darwin-arm64': 1.14.0+ '@oxlint/darwin-x64': 1.14.0+ '@oxlint/linux-arm64-gnu': 1.14.0+ '@oxlint/linux-arm64-musl': 1.14.0+ '@oxlint/linux-x64-gnu': 1.14.0+ '@oxlint/linux-x64-musl': 1.14.0+ '@oxlint/win32-arm64': 1.14.0+ '@oxlint/win32-x64': 1.14.0+dependencies:p-try: 2.2.0@@ -13429,10 +14843,18 @@ snapshots:dependencies:yocto-queue: 0.1.0+ dependencies:+ yocto-queue: 1.2.1+dependencies:p-limit: 2.3.0+ dependencies:+ p-limit: 4.0.0+@@ -13472,6 +14894,8 @@ snapshots:+ [email protected]: {}+@@ -13525,6 +14949,8 @@ snapshots:+ [email protected]: {}+@@ -13552,7 +14978,7 @@ snapshots:sonic-boom: 2.8.0thread-stream: 0.15.2- [email protected]: {}+ [email protected]: {}dependencies:@@ -13720,6 +15146,11 @@ snapshots:+ dependencies:+ json-parse-even-better-errors: 4.0.0+ npm-normalize-package-bin: 4.0.0+dependencies:core-util-is: 1.0.3@@ -13873,6 +15304,8 @@ snapshots:+ [email protected]: {}+dependencies:debug: 2.6.9@@ -13927,6 +15360,8 @@ snapshots:+ [email protected]: {}+@@ -14320,6 +15755,8 @@ snapshots:dependencies:'@mixmark-io/domino': 2.2.0+ [email protected]: {}+@@ -14337,6 +15774,8 @@ snapshots:+ [email protected]: {}+@@ -14351,6 +15790,10 @@ snapshots:+ [email protected]: {}++ [email protected]: {}+@@ -14602,6 +16045,10 @@ snapshots:dependencies:isexe: 2.0.0+ dependencies:+ isexe: 3.1.1+dependencies:logform: 2.6.1@@ -14832,6 +16279,10 @@ snapshots:+ [email protected]: {}++ [email protected]: {}+dependencies:zod: 3.24.2diff --git a/apps/api/pnpm-workspace.yaml b/apps/api/pnpm-workspace.yamlnew file mode 100644index 000000000..ce4f0560b--- /dev/null+++ b/apps/api/pnpm-workspace.yaml@@ -0,0 +1,3 @@+packages:+ - "native"+injectWorkspacePackages: truediff --git a/apps/api/sharedLibs/crawler/.gitignore b/apps/api/sharedLibs/crawler/.gitignoredeleted file mode 100644index 1de565933..000000000--- a/apps/api/sharedLibs/crawler/.gitignore+++ /dev/null@@ -1 +0,0 @@-target\ No newline at end of filediff --git a/apps/api/sharedLibs/crawler/Cargo.lock b/apps/api/sharedLibs/crawler/Cargo.lockdeleted file mode 100644index 8a753c488..000000000--- a/apps/api/sharedLibs/crawler/Cargo.lock+++ /dev/null@@ -1,514 +0,0 @@-# This file is automatically @generated by Cargo.-# It is not intended for manual editing.-version = 4--[[package]]-name = "aho-corasick"-version = "1.1.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"-dependencies = [- "memchr",-]--[[package]]-name = "anyhow"-version = "1.0.98"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"--[[package]]-name = "bstr"-version = "0.2.17"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"-dependencies = [- "lazy_static",- "memchr",- "regex-automata 0.1.10",-]--[[package]]-name = "crawler"-version = "0.1.0"-dependencies = [- "libc",- "regex",- "roxmltree",- "serde",- "serde_json",- "texting_robots",- "url",-]--[[package]]-name = "displaydoc"-version = "0.2.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "form_urlencoded"-version = "1.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"-dependencies = [- "percent-encoding",-]--[[package]]-name = "icu_collections"-version = "2.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47"-dependencies = [- "displaydoc",- "potential_utf",- "yoke",- "zerofrom",- "zerovec",-]--[[package]]-name = "icu_locale_core"-version = "2.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a"-dependencies = [- "displaydoc",- "litemap",- "tinystr",- "writeable",- "zerovec",-]--[[package]]-name = "icu_normalizer"-version = "2.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979"-dependencies = [- "displaydoc",- "icu_collections",- "icu_normalizer_data",- "icu_properties",- "icu_provider",- "smallvec",- "zerovec",-]--[[package]]-name = "icu_normalizer_data"-version = "2.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3"--[[package]]-name = "icu_properties"-version = "2.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b"-dependencies = [- "displaydoc",- "icu_collections",- "icu_locale_core",- "icu_properties_data",- "icu_provider",- "potential_utf",- "zerotrie",- "zerovec",-]--[[package]]-name = "icu_properties_data"-version = "2.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632"--[[package]]-name = "icu_provider"-version = "2.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af"-dependencies = [- "displaydoc",- "icu_locale_core",- "stable_deref_trait",- "tinystr",- "writeable",- "yoke",- "zerofrom",- "zerotrie",- "zerovec",-]--[[package]]-name = "idna"-version = "1.0.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"-dependencies = [- "idna_adapter",- "smallvec",- "utf8_iter",-]--[[package]]-name = "idna_adapter"-version = "1.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"-dependencies = [- "icu_normalizer",- "icu_properties",-]--[[package]]-name = "itoa"-version = "1.0.15"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"--[[package]]-name = "lazy_static"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"--[[package]]-name = "libc"-version = "0.2.174"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"--[[package]]-name = "litemap"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"--[[package]]-name = "memchr"-version = "2.7.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"--[[package]]-name = "minimal-lexical"-version = "0.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"--[[package]]-name = "nom"-version = "7.1.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"-dependencies = [- "memchr",- "minimal-lexical",-]--[[package]]-name = "percent-encoding"-version = "2.3.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"--[[package]]-name = "potential_utf"-version = "0.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585"-dependencies = [- "zerovec",-]--[[package]]-name = "proc-macro2"-version = "1.0.95"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"-dependencies = [- "unicode-ident",-]--[[package]]-name = "quote"-version = "1.0.40"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"-dependencies = [- "proc-macro2",-]--[[package]]-name = "regex"-version = "1.11.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"-dependencies = [- "aho-corasick",- "memchr",- "regex-automata 0.4.9",- "regex-syntax",-]--[[package]]-name = "regex-automata"-version = "0.1.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"--[[package]]-name = "regex-automata"-version = "0.4.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"-dependencies = [- "aho-corasick",- "memchr",- "regex-syntax",-]--[[package]]-name = "regex-syntax"-version = "0.8.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"--[[package]]-name = "roxmltree"-version = "0.20.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"--[[package]]-name = "ryu"-version = "1.0.20"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"--[[package]]-name = "serde"-version = "1.0.219"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"-dependencies = [- "serde_derive",-]--[[package]]-name = "serde_derive"-version = "1.0.219"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "serde_json"-version = "1.0.140"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"-dependencies = [- "itoa",- "memchr",- "ryu",- "serde",-]--[[package]]-name = "smallvec"-version = "1.15.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"--[[package]]-name = "stable_deref_trait"-version = "1.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"--[[package]]-name = "syn"-version = "2.0.104"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"-dependencies = [- "proc-macro2",- "quote",- "unicode-ident",-]--[[package]]-name = "synstructure"-version = "0.13.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "texting_robots"-version = "0.2.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a"-dependencies = [- "anyhow",- "bstr",- "lazy_static",- "nom",- "percent-encoding",- "regex",- "thiserror",- "url",-]--[[package]]-name = "thiserror"-version = "1.0.69"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"-dependencies = [- "thiserror-impl",-]--[[package]]-name = "thiserror-impl"-version = "1.0.69"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "tinystr"-version = "0.8.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b"-dependencies = [- "displaydoc",- "zerovec",-]--[[package]]-name = "unicode-ident"-version = "1.0.18"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"--[[package]]-name = "url"-version = "2.5.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"-dependencies = [- "form_urlencoded",- "idna",- "percent-encoding",-]--[[package]]-name = "utf8_iter"-version = "1.0.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"--[[package]]-name = "writeable"-version = "0.6.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"--[[package]]-name = "yoke"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc"-dependencies = [- "serde",- "stable_deref_trait",- "yoke-derive",- "zerofrom",-]--[[package]]-name = "yoke-derive"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"-dependencies = [- "proc-macro2",- "quote",- "syn",- "synstructure",-]--[[package]]-name = "zerofrom"-version = "0.1.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"-dependencies = [- "zerofrom-derive",-]--[[package]]-name = "zerofrom-derive"-version = "0.1.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"-dependencies = [- "proc-macro2",- "quote",- "syn",- "synstructure",-]--[[package]]-name = "zerotrie"-version = "0.2.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595"-dependencies = [- "displaydoc",- "yoke",- "zerofrom",-]--[[package]]-name = "zerovec"-version = "0.11.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"-dependencies = [- "yoke",- "zerofrom",- "zerovec-derive",-]--[[package]]-name = "zerovec-derive"-version = "0.11.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"-dependencies = [- "proc-macro2",- "quote",- "syn",-]diff --git a/apps/api/sharedLibs/crawler/Cargo.toml b/apps/api/sharedLibs/crawler/Cargo.tomldeleted file mode 100644index 1e43b72e9..000000000--- a/apps/api/sharedLibs/crawler/Cargo.toml+++ /dev/null@@ -1,16 +0,0 @@-[package]-name = "crawler"-version = "0.1.0"-edition = "2024"--[dependencies]-libc = "0.2.0"-serde = { version = "1.0", features = ["derive"] }-serde_json = "1.0"-url = "2.5.4"-regex = "1.11.1"-texting_robots = "0.2.2"-roxmltree = "0.20"--[lib]-crate-type = ["cdylib"]diff --git a/apps/api/sharedLibs/crawler/src/lib.rs b/apps/api/sharedLibs/crawler/src/lib.rsdeleted file mode 100644index 0ee510f98..000000000--- a/apps/api/sharedLibs/crawler/src/lib.rs+++ /dev/null@@ -1,635 +0,0 @@-use std::{collections::HashMap, usize};-use serde::{Deserialize, Serialize};-use std::ffi::{CStr, CString};-use url::Url;-use regex::Regex;-use texting_robots::Robot;--#[derive(Deserialize)]-struct FilterLinksCall {- links: Vec<String>,- limit: Option<i64>,- max_depth: u32,- base_url: String,- initial_url: String,- regex_on_full_url: bool,- excludes: Vec<String>,- includes: Vec<String>,- allow_backward_crawling: bool,- ignore_robots_txt: bool,- robots_txt: String,-}--#[derive(Serialize)]-struct FilterLinksResult {- links: Vec<String>,- denial_reasons: HashMap<String, String>,-}--#[derive(Serialize, Debug)]-struct SitemapUrl {- loc: Vec<String>,-}--#[derive(Serialize, Debug)]-struct SitemapEntry {- loc: Vec<String>,-}--#[derive(Serialize, Debug)]-#[serde(untagged)]-enum ParsedSitemap {- Urlset { urlset: SitemapUrlset },- SitemapIndex { sitemapindex: SitemapIndex },-}--#[derive(Serialize, Debug)]-struct SitemapUrlset {- url: Vec<SitemapUrl>,-}--#[derive(Serialize, Debug)]-struct SitemapIndex {- sitemap: Vec<SitemapEntry>,-}--#[derive(Serialize, Debug)]-struct SitemapInstruction {- action: String,- urls: Vec<String>,- count: usize,-}--#[derive(Serialize, Debug)]-struct SitemapProcessingResult {- instructions: Vec<SitemapInstruction>,- total_count: usize,-}--fn _is_file(url: &Url) -> bool {- let file_extensions = vec![- ".png",- ".jpg",- ".jpeg",- ".gif",- ".css",- ".js",- ".ico",- ".svg",- ".tiff",- ".zip",- ".exe",- ".dmg",- ".mp4",- ".mp3",- ".wav",- ".pptx",- ".xlsx",- ".avi",- ".flv",- ".woff",- ".ttf",- ".woff2",- ".webp",- ".inc",- ];- let url_without_query = url.path().to_lowercase();- file_extensions.iter().any(|ext| url_without_query.ends_with(ext))-}--fn _get_url_depth(path: &str) -> u32 {- path.split('/').filter(|x| *x != "" && *x != "index.php" && *x != "index.html").count() as u32-}--fn _filter_links(data: FilterLinksCall) -> Result<FilterLinksResult, Box<dyn std::error::Error>> {- let mut denial_reasons = HashMap::new();-- let limit = data.limit- .and_then(|x| if x < 0 { Some(0) } else { Some(x as usize) })- .unwrap_or(usize::MAX);-- if limit == 0 {- return Ok(FilterLinksResult {- links: Vec::with_capacity(0),- denial_reasons,- });- }-- let base_url = Url::parse(&data.base_url)?;- let initial_url = Url::parse(&data.initial_url)?;-- let excludes_regex = data.excludes.iter().map(|exclude| Regex::new(exclude)).collect::<Vec<Result<Regex, regex::Error>>>();- let excludes_regex = excludes_regex.into_iter().filter_map(|x| x.ok()).collect::<Vec<Regex>>();-- let includes_regex = data.includes.iter().map(|include| Regex::new(include)).collect::<Vec<Result<Regex, regex::Error>>>();- let includes_regex = includes_regex.into_iter().filter_map(|x| x.ok()).collect::<Vec<Regex>>();-- let links = data.links.into_iter()- .filter(|link| {- let url = match base_url.join(link) {- Ok(x) => x,- Err(_) => {- denial_reasons.insert(link.clone(), "URL_PARSE_ERROR".to_string());- return false;- }- };-- let path = url.path();- let depth = _get_url_depth(path);- if depth > data.max_depth {- denial_reasons.insert(link.clone(), "DEPTH_LIMIT".to_string());- return false;- }-- let exinc_path = if data.regex_on_full_url {- url.as_str()- } else {- url.path()- };-- if !excludes_regex.is_empty() && excludes_regex.iter().any(|regex| regex.is_match(exinc_path)) {- denial_reasons.insert(link.clone(), "EXCLUDE_PATTERN".to_string());- return false;- }-- if !includes_regex.is_empty() && !includes_regex.iter().any(|regex| regex.is_match(exinc_path)) {- denial_reasons.insert(link.clone(), "INCLUDE_PATTERN".to_string());- return false;- }-- if !data.allow_backward_crawling {- if !url.path().starts_with(initial_url.path()) {- denial_reasons.insert(link.clone(), "BACKWARD_CRAWLING".to_string());- return false;- }- }-- if !data.ignore_robots_txt {- match Robot::new("FireCrawlAgent", data.robots_txt.as_bytes()) {- Ok(robot) => {- let allowed = robot.allowed(url.as_str());- if !allowed {- match Robot::new("FirecrawlAgent", data.robots_txt.as_bytes()) {- Ok(robot_alt) => {- let allowed_alt = robot_alt.allowed(url.as_str());- if !allowed_alt {- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());- return false;- }- }- Err(_) => {- denial_reasons.insert(link.clone(), "ROBOTS_TXT".to_string());- return false;- }- }- }- }- Err(_) => {- }- }- }-- if _is_file(&url) {- denial_reasons.insert(link.clone(), "FILE_TYPE".to_string());- return false;- }-- true- })- .take(limit)- .collect::<Vec<_>>();-- Ok(FilterLinksResult {- links,- denial_reasons,- })-}--#[unsafe(no_mangle)]-pub unsafe extern "C" fn filter_links(data: *const libc::c_char) -> *mut libc::c_char {- let data = match serde_json::from_slice(unsafe { CStr::from_ptr(data).to_bytes() }) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as C string: {}", e)).unwrap().into_raw();- }- };-- let result = match _filter_links(data) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- let result = match serde_json::to_string(&result) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();- }- };-- CString::new(result).unwrap().into_raw()-}--fn _parse_sitemap_xml(xml_content: &str) -> Result<ParsedSitemap, Box<dyn std::error::Error>> {- let doc = roxmltree::Document::parse_with_options(- xml_content,- roxmltree::ParsingOptions { allow_dtd: true, ..Default::default() },- )?;- let root = doc.root_element();-- match root.tag_name().name() {- "sitemapindex" => {- let mut sitemaps = Vec::new();-- for sitemap_node in root.children().filter(|n| n.is_element() && n.tag_name().name() == "sitemap") {- if let Some(loc_node) = sitemap_node.children().find(|n| n.is_element() && n.tag_name().name() == "loc") {- if let Some(loc_text) = loc_node.text() {- sitemaps.push(SitemapEntry {- loc: vec![loc_text.to_string()],- });- }- }- }-- Ok(ParsedSitemap::SitemapIndex {- sitemapindex: SitemapIndex { sitemap: sitemaps }- })- },- "urlset" => {- let mut urls = Vec::new();-- for url_node in root.children().filter(|n| n.is_element() && n.tag_name().name() == "url") {- if let Some(loc_node) = url_node.children().find(|n| n.is_element() && n.tag_name().name() == "loc") {- if let Some(loc_text) = loc_node.text() {- urls.push(SitemapUrl {- loc: vec![loc_text.to_string()],- });- }- }- }-- Ok(ParsedSitemap::Urlset {- urlset: SitemapUrlset { url: urls }- })- },- _ => {- Err("Invalid sitemap format: root element must be 'sitemapindex' or 'urlset'".into())- }- }-}--#[unsafe(no_mangle)]-pub unsafe extern "C" fn parse_sitemap_xml(data: *const libc::c_char) -> *mut libc::c_char {- let xml_content = match unsafe { CStr::from_ptr(data).to_str() } {- Ok(s) => s,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as UTF-8 string: {}", e)).unwrap().into_raw();- }- };-- let result = match _parse_sitemap_xml(xml_content) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- let result = match serde_json::to_string(&result) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();- }- };-- CString::new(result).unwrap().into_raw()-}--fn _process_sitemap(xml_content: &str) -> Result<SitemapProcessingResult, Box<dyn std::error::Error>> {- let parsed = _parse_sitemap_xml(xml_content)?;- let mut instructions = Vec::new();- let mut total_count = 0;-- match parsed {- ParsedSitemap::SitemapIndex { sitemapindex } => {- let sitemap_urls: Vec<String> = sitemapindex.sitemap- .iter()- .filter_map(|sitemap| {- if !sitemap.loc.is_empty() {- Some(sitemap.loc[0].trim().to_string())- } else {- None- }- })- .collect();-- if !sitemap_urls.is_empty() {- instructions.push(SitemapInstruction {- action: "recurse".to_string(),- urls: sitemap_urls.clone(),- count: sitemap_urls.len(),- });- total_count += sitemap_urls.len();- }- },- ParsedSitemap::Urlset { urlset } => {- let mut xml_sitemaps = Vec::new();- let mut valid_urls = Vec::new();-- for url_entry in urlset.url {- if !url_entry.loc.is_empty() {- let url = url_entry.loc[0].trim();- if url.to_lowercase().ends_with(".xml") {- xml_sitemaps.push(url.to_string());- } else if let Ok(parsed_url) = Url::parse(url) {- if !_is_file(&parsed_url) {- valid_urls.push(url.to_string());- }- }- }- }-- if !xml_sitemaps.is_empty() {- instructions.push(SitemapInstruction {- action: "recurse".to_string(),- urls: xml_sitemaps.clone(),- count: xml_sitemaps.len(),- });- total_count += xml_sitemaps.len();- }-- if !valid_urls.is_empty() {- instructions.push(SitemapInstruction {- action: "process".to_string(),- urls: valid_urls.clone(),- count: valid_urls.len(),- });- total_count += valid_urls.len();- }- }- }-- Ok(SitemapProcessingResult {- instructions,- total_count,- })-}--#[unsafe(no_mangle)]-pub unsafe extern "C" fn process_sitemap(data: *const libc::c_char) -> *mut libc::c_char {- let xml_content = match unsafe { CStr::from_ptr(data).to_str() } {- Ok(s) => s,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Failed to parse input data as UTF-8 string: {}", e)).unwrap().into_raw();- }- };-- let result = match _process_sitemap(xml_content) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- let result = match serde_json::to_string(&result) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Failed to serialize result as JSON {}", e)).unwrap().into_raw();- }- };-- CString::new(result).unwrap().into_raw()-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_parse_sitemap_xml_urlset() {- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">- <url>- <loc>https://example.com/page1</loc>- </url>- <url>- <loc>https://example.com/page2</loc>- </url>-</urlset>"#;-- let result = _parse_sitemap_xml(xml_content).unwrap();- match result {- ParsedSitemap::Urlset { urlset } => {- assert_eq!(urlset.url.len(), 2);- assert_eq!(urlset.url[0].loc[0], "https://example.com/page1");- assert_eq!(urlset.url[1].loc[0], "https://example.com/page2");- }- _ => panic!("Expected Urlset variant"),- }- }-- #[test]- fn test_parse_sitemap_xml_sitemapindex() {- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>-<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">- <sitemap>- <loc>https://example.com/sitemap1.xml</loc>- </sitemap>- <sitemap>- <loc>https://example.com/sitemap2.xml</loc>- </sitemap>-</sitemapindex>"#;-- let result = _parse_sitemap_xml(xml_content).unwrap();- match result {- ParsedSitemap::SitemapIndex { sitemapindex } => {- assert_eq!(sitemapindex.sitemap.len(), 2);- assert_eq!(sitemapindex.sitemap[0].loc[0], "https://example.com/sitemap1.xml");- assert_eq!(sitemapindex.sitemap[1].loc[0], "https://example.com/sitemap2.xml");- }- _ => panic!("Expected SitemapIndex variant"),- }- }-- #[test]- fn test_parse_sitemap_xml_invalid_root() {- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>-<invalid xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">- <url>- <loc>https://example.com/page1</loc>- </url>-</invalid>"#;-- let result = _parse_sitemap_xml(xml_content);- assert!(result.is_err());- assert!(result.unwrap_err().to_string().contains("Invalid sitemap format"));- }-- #[test]- fn test_parse_sitemap_xml_malformed() {- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">- <url>- <loc>https://example.com/page1</loc>- </url>-</urlset"#; // Missing closing >-- let result = _parse_sitemap_xml(xml_content);- assert!(result.is_err());- }-- #[test]- fn test_process_sitemap_urlset() {- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">- <url>- <loc>https://example.com/page1</loc>- </url>- <url>- <loc>https://example.com/sitemap2.xml</loc>- </url>- <url>- <loc>https://example.com/image.png</loc>- </url>-</urlset>"#;-- let result = _process_sitemap(xml_content).unwrap();- assert_eq!(result.instructions.len(), 2);-- let recurse_instruction = result.instructions.iter().find(|i| i.action == "recurse").unwrap();- assert_eq!(recurse_instruction.urls.len(), 1);- assert_eq!(recurse_instruction.urls[0], "https://example.com/sitemap2.xml");-- let process_instruction = result.instructions.iter().find(|i| i.action == "process").unwrap();- assert_eq!(process_instruction.urls.len(), 1);- assert_eq!(process_instruction.urls[0], "https://example.com/page1");- }-- #[test]- fn test_process_sitemap_sitemapindex() {- let xml_content = r#"<?xml version="1.0" encoding="UTF-8"?>-<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">- <sitemap>- <loc>https://example.com/sitemap1.xml</loc>- </sitemap>- <sitemap>- <loc>https://example.com/sitemap2.xml</loc>- </sitemap>-</sitemapindex>"#;-- let result = _process_sitemap(xml_content).unwrap();- assert_eq!(result.instructions.len(), 1);- assert_eq!(result.instructions[0].action, "recurse");- assert_eq!(result.instructions[0].urls.len(), 2);- assert_eq!(result.instructions[0].urls[0], "https://example.com/sitemap1.xml");- assert_eq!(result.instructions[0].urls[1], "https://example.com/sitemap2.xml");- }-- #[test]- fn test_filter_links_normal_robots_txt() {- let data = FilterLinksCall {- links: vec!["https://example.com/allowed".to_string(), "https://example.com/disallowed".to_string()],- limit: Some(10),- includes: vec![],- excludes: vec![],- ignore_robots_txt: false,- robots_txt: "User-agent: *\nDisallow: /disallowed".to_string(),- max_depth: 10,- base_url: "https://example.com".to_string(),- initial_url: "https://example.com".to_string(),- regex_on_full_url: false,- allow_backward_crawling: true,- };-- let result = _filter_links(data).unwrap();- assert_eq!(result.links.len(), 1);- assert_eq!(result.links[0], "https://example.com/allowed");- assert!(result.denial_reasons.contains_key("https://example.com/disallowed"));- assert_eq!(result.denial_reasons.get("https://example.com/disallowed").unwrap(), "ROBOTS_TXT");- }-- #[test]- fn test_filter_links_malformed_robots_txt() {- let data = FilterLinksCall {- links: vec!["https://example.com/test".to_string()],- limit: Some(10),- includes: vec![],- excludes: vec![],- ignore_robots_txt: false,- robots_txt: "Invalid robots.txt content with \x00 null bytes and malformed syntax".to_string(),- max_depth: 10,- base_url: "https://example.com".to_string(),- initial_url: "https://example.com".to_string(),- regex_on_full_url: false,- allow_backward_crawling: true,- };-- let result = _filter_links(data);- assert!(result.is_ok());- let result = result.unwrap();- assert_eq!(result.links.len(), 1);- assert_eq!(result.links[0], "https://example.com/test");- }-- #[test]- fn test_filter_links_non_utf8_robots_txt() {- let mut non_utf8_bytes = vec![0xFF, 0xFE];- non_utf8_bytes.extend_from_slice(b"User-agent: *\nDisallow: /blocked");- let non_utf8_string = String::from_utf8_lossy(&non_utf8_bytes).to_string();-- let data = FilterLinksCall {- links: vec!["https://example.com/allowed".to_string()],- limit: Some(10),- includes: vec![],- excludes: vec![],- ignore_robots_txt: false,- robots_txt: non_utf8_string,- max_depth: 10,- base_url: "https://example.com".to_string(),- initial_url: "https://example.com".to_string(),- regex_on_full_url: false,- allow_backward_crawling: true,- };-- let result = _filter_links(data);- assert!(result.is_ok());- let result = result.unwrap();- assert_eq!(result.links.len(), 1);- assert_eq!(result.links[0], "https://example.com/allowed");- }-- #[test]- fn test_filter_links_char_boundary_issue() {- let problematic_content = "User-agent: *\nDisallow: /\u{a0}test";-- let data = FilterLinksCall {- links: vec!["https://example.com/test".to_string()],- limit: Some(10),- includes: vec![],- excludes: vec![],- ignore_robots_txt: false,- robots_txt: problematic_content.to_string(),- max_depth: 10,- base_url: "https://example.com".to_string(),- initial_url: "https://example.com".to_string(),- regex_on_full_url: false,- allow_backward_crawling: true,- };-- let result = _filter_links(data);- assert!(result.is_ok());- let result = result.unwrap();- assert_eq!(result.links.len(), 1);- assert_eq!(result.links[0], "https://example.com/test");- }--}--/// Frees a string allocated in Rust-land.-///-/// # Safety-/// ptr must be a non-freed string pointer returned by Rust code.-#[unsafe(no_mangle)]-pub unsafe extern "C" fn free_string(ptr: *mut libc::c_char) {- drop(unsafe { CString::from_raw(ptr) })-}diff --git a/apps/api/sharedLibs/html-transformer/.gitignore b/apps/api/sharedLibs/html-transformer/.gitignoredeleted file mode 100644index 1de565933..000000000--- a/apps/api/sharedLibs/html-transformer/.gitignore+++ /dev/null@@ -1 +0,0 @@-target\ No newline at end of filediff --git a/apps/api/sharedLibs/html-transformer/Cargo.lock b/apps/api/sharedLibs/html-transformer/Cargo.lockdeleted file mode 100644index 4cfb944ce..000000000--- a/apps/api/sharedLibs/html-transformer/Cargo.lock+++ /dev/null@@ -1,1396 +0,0 @@-# This file is automatically @generated by Cargo.-# It is not intended for manual editing.-version = 4--[[package]]-name = "allocator-api2"-version = "0.2.21"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"--[[package]]-name = "autocfg"-version = "1.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"--[[package]]-name = "base64"-version = "0.22.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"--[[package]]-name = "bitflags"-version = "1.3.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"--[[package]]-name = "bitflags"-version = "2.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"--[[package]]-name = "block-buffer"-version = "0.10.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"-dependencies = [- "generic-array",-]--[[package]]-name = "byteorder"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"--[[package]]-name = "cfg-if"-version = "1.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"--[[package]]-name = "convert_case"-version = "0.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"--[[package]]-name = "cpufeatures"-version = "0.2.17"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"-dependencies = [- "libc",-]--[[package]]-name = "crypto-common"-version = "0.1.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"-dependencies = [- "generic-array",- "typenum",-]--[[package]]-name = "cssparser"-version = "0.27.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"-dependencies = [- "cssparser-macros",- "dtoa-short",- "itoa 0.4.8",- "matches",- "phf 0.8.0",- "proc-macro2",- "quote",- "smallvec",- "syn 1.0.109",-]--[[package]]-name = "cssparser"-version = "0.29.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f93d03419cb5950ccfd3daf3ff1c7a36ace64609a1a8746d493df1ca0afde0fa"-dependencies = [- "cssparser-macros",- "dtoa-short",- "itoa 1.0.14",- "matches",- "phf 0.10.1",- "proc-macro2",- "quote",- "smallvec",- "syn 1.0.109",-]--[[package]]-name = "cssparser-macros"-version = "0.6.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"-dependencies = [- "quote",- "syn 2.0.96",-]--[[package]]-name = "derive_more"-version = "0.99.18"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"-dependencies = [- "convert_case",- "proc-macro2",- "quote",- "rustc_version",- "syn 2.0.96",-]--[[package]]-name = "digest"-version = "0.10.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"-dependencies = [- "block-buffer",- "crypto-common",-]--[[package]]-name = "displaydoc"-version = "0.2.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",-]--[[package]]-name = "dtoa"-version = "1.0.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"--[[package]]-name = "dtoa-short"-version = "0.3.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"-dependencies = [- "dtoa",-]--[[package]]-name = "encoding_rs"-version = "0.8.35"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"-dependencies = [- "cfg-if",-]--[[package]]-name = "equivalent"-version = "1.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"--[[package]]-name = "foldhash"-version = "0.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"--[[package]]-name = "form_urlencoded"-version = "1.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"-dependencies = [- "percent-encoding",-]--[[package]]-name = "futf"-version = "0.1.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"-dependencies = [- "mac",- "new_debug_unreachable",-]--[[package]]-name = "fxhash"-version = "0.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"-dependencies = [- "byteorder",-]--[[package]]-name = "generic-array"-version = "0.14.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"-dependencies = [- "typenum",- "version_check",-]--[[package]]-name = "getrandom"-version = "0.1.16"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"-dependencies = [- "cfg-if",- "libc",- "wasi 0.9.0+wasi-snapshot-preview1",-]--[[package]]-name = "getrandom"-version = "0.2.15"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"-dependencies = [- "cfg-if",- "libc",- "wasi 0.11.0+wasi-snapshot-preview1",-]--[[package]]-name = "getrandom"-version = "0.3.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"-dependencies = [- "cfg-if",- "libc",- "r-efi",- "wasi 0.14.2+wasi-0.2.4",-]--[[package]]-name = "hashbrown"-version = "0.12.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"--[[package]]-name = "hashbrown"-version = "0.15.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"-dependencies = [- "allocator-api2",- "equivalent",- "foldhash",-]--[[package]]-name = "hex"-version = "0.4.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"--[[package]]-name = "html-transformer"-version = "0.1.0"-dependencies = [- "kuchikiki",- "libc",- "lol_html",- "nodesig",- "serde",- "serde_json",- "url",-]--[[package]]-name = "html5ever"-version = "0.26.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7"-dependencies = [- "log",- "mac",- "markup5ever",- "proc-macro2",- "quote",- "syn 1.0.109",-]--[[package]]-name = "icu_collections"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"-dependencies = [- "displaydoc",- "yoke",- "zerofrom",- "zerovec",-]--[[package]]-name = "icu_locid"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"-dependencies = [- "displaydoc",- "litemap",- "tinystr",- "writeable",- "zerovec",-]--[[package]]-name = "icu_locid_transform"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"-dependencies = [- "displaydoc",- "icu_locid",- "icu_locid_transform_data",- "icu_provider",- "tinystr",- "zerovec",-]--[[package]]-name = "icu_locid_transform_data"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"--[[package]]-name = "icu_normalizer"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"-dependencies = [- "displaydoc",- "icu_collections",- "icu_normalizer_data",- "icu_properties",- "icu_provider",- "smallvec",- "utf16_iter",- "utf8_iter",- "write16",- "zerovec",-]--[[package]]-name = "icu_normalizer_data"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"--[[package]]-name = "icu_properties"-version = "1.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"-dependencies = [- "displaydoc",- "icu_collections",- "icu_locid_transform",- "icu_properties_data",- "icu_provider",- "tinystr",- "zerovec",-]--[[package]]-name = "icu_properties_data"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"--[[package]]-name = "icu_provider"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"-dependencies = [- "displaydoc",- "icu_locid",- "icu_provider_macros",- "stable_deref_trait",- "tinystr",- "writeable",- "yoke",- "zerofrom",- "zerovec",-]--[[package]]-name = "icu_provider_macros"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",-]--[[package]]-name = "idna"-version = "1.0.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"-dependencies = [- "idna_adapter",- "smallvec",- "utf8_iter",-]--[[package]]-name = "idna_adapter"-version = "1.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"-dependencies = [- "icu_normalizer",- "icu_properties",-]--[[package]]-name = "indexmap"-version = "1.9.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"-dependencies = [- "autocfg",- "hashbrown 0.12.3",-]--[[package]]-name = "itoa"-version = "0.4.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"--[[package]]-name = "itoa"-version = "1.0.14"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"--[[package]]-name = "kuchikiki"-version = "0.8.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f29e4755b7b995046f510a7520c42b2fed58b77bd94d5a87a8eb43d2fd126da8"-dependencies = [- "cssparser 0.27.2",- "html5ever",- "indexmap",- "matches",- "selectors 0.22.0",-]--[[package]]-name = "libc"-version = "0.2.169"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"--[[package]]-name = "litemap"-version = "0.7.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"--[[package]]-name = "lock_api"-version = "0.4.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"-dependencies = [- "autocfg",- "scopeguard",-]--[[package]]-name = "log"-version = "0.4.25"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"--[[package]]-name = "lol_html"-version = "2.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3b1058123f6262982b891dccc395cff0144d9439de366460b47fab719258b96e"-dependencies = [- "bitflags 2.8.0",- "cfg-if",- "cssparser 0.29.6",- "encoding_rs",- "hashbrown 0.15.2",- "memchr",- "mime",- "selectors 0.24.0",- "thiserror",-]--[[package]]-name = "mac"-version = "0.1.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"--[[package]]-name = "markup5ever"-version = "0.11.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016"-dependencies = [- "log",- "phf 0.10.1",- "phf_codegen 0.10.0",- "string_cache",- "string_cache_codegen",- "tendril",-]--[[package]]-name = "matches"-version = "0.1.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"--[[package]]-name = "memchr"-version = "2.7.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"--[[package]]-name = "mime"-version = "0.3.17"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"--[[package]]-name = "new_debug_unreachable"-version = "1.0.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"--[[package]]-name = "nodesig"-version = "1.0.0"-source = "git+https://github.com/firecrawl/nodesig#b8ebc4a710810dbc36e189d9a0df87b877702b3c"-dependencies = [- "base64",- "hex",- "kuchikiki",- "rand 0.9.1",- "sha2",-]--[[package]]-name = "nodrop"-version = "0.1.14"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"--[[package]]-name = "once_cell"-version = "1.20.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"--[[package]]-name = "parking_lot"-version = "0.12.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"-dependencies = [- "lock_api",- "parking_lot_core",-]--[[package]]-name = "parking_lot_core"-version = "0.9.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"-dependencies = [- "cfg-if",- "libc",- "redox_syscall",- "smallvec",- "windows-targets",-]--[[package]]-name = "percent-encoding"-version = "2.3.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"--[[package]]-name = "phf"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"-dependencies = [- "phf_macros 0.8.0",- "phf_shared 0.8.0",- "proc-macro-hack",-]--[[package]]-name = "phf"-version = "0.10.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"-dependencies = [- "phf_macros 0.10.0",- "phf_shared 0.10.0",- "proc-macro-hack",-]--[[package]]-name = "phf_codegen"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"-dependencies = [- "phf_generator 0.8.0",- "phf_shared 0.8.0",-]--[[package]]-name = "phf_codegen"-version = "0.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"-dependencies = [- "phf_generator 0.10.0",- "phf_shared 0.10.0",-]--[[package]]-name = "phf_generator"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"-dependencies = [- "phf_shared 0.8.0",- "rand 0.7.3",-]--[[package]]-name = "phf_generator"-version = "0.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"-dependencies = [- "phf_shared 0.10.0",- "rand 0.8.5",-]--[[package]]-name = "phf_macros"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"-dependencies = [- "phf_generator 0.8.0",- "phf_shared 0.8.0",- "proc-macro-hack",- "proc-macro2",- "quote",- "syn 1.0.109",-]--[[package]]-name = "phf_macros"-version = "0.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0"-dependencies = [- "phf_generator 0.10.0",- "phf_shared 0.10.0",- "proc-macro-hack",- "proc-macro2",- "quote",- "syn 1.0.109",-]--[[package]]-name = "phf_shared"-version = "0.8.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"-dependencies = [- "siphasher",-]--[[package]]-name = "phf_shared"-version = "0.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"-dependencies = [- "siphasher",-]--[[package]]-name = "ppv-lite86"-version = "0.2.20"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"-dependencies = [- "zerocopy",-]--[[package]]-name = "precomputed-hash"-version = "0.1.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"--[[package]]-name = "proc-macro-hack"-version = "0.5.20+deprecated"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"--[[package]]-name = "proc-macro2"-version = "1.0.93"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"-dependencies = [- "unicode-ident",-]--[[package]]-name = "quote"-version = "1.0.38"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"-dependencies = [- "proc-macro2",-]--[[package]]-name = "r-efi"-version = "5.3.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"--[[package]]-name = "rand"-version = "0.7.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"-dependencies = [- "getrandom 0.1.16",- "libc",- "rand_chacha 0.2.2",- "rand_core 0.5.1",- "rand_hc",- "rand_pcg",-]--[[package]]-name = "rand"-version = "0.8.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"-dependencies = [- "libc",- "rand_chacha 0.3.1",- "rand_core 0.6.4",-]--[[package]]-name = "rand"-version = "0.9.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"-dependencies = [- "rand_chacha 0.9.0",- "rand_core 0.9.3",-]--[[package]]-name = "rand_chacha"-version = "0.2.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"-dependencies = [- "ppv-lite86",- "rand_core 0.5.1",-]--[[package]]-name = "rand_chacha"-version = "0.3.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"-dependencies = [- "ppv-lite86",- "rand_core 0.6.4",-]--[[package]]-name = "rand_chacha"-version = "0.9.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"-dependencies = [- "ppv-lite86",- "rand_core 0.9.3",-]--[[package]]-name = "rand_core"-version = "0.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"-dependencies = [- "getrandom 0.1.16",-]--[[package]]-name = "rand_core"-version = "0.6.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"-dependencies = [- "getrandom 0.2.15",-]--[[package]]-name = "rand_core"-version = "0.9.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"-dependencies = [- "getrandom 0.3.3",-]--[[package]]-name = "rand_hc"-version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"-dependencies = [- "rand_core 0.5.1",-]--[[package]]-name = "rand_pcg"-version = "0.2.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"-dependencies = [- "rand_core 0.5.1",-]--[[package]]-name = "redox_syscall"-version = "0.5.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"-dependencies = [- "bitflags 2.8.0",-]--[[package]]-name = "rustc_version"-version = "0.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"-dependencies = [- "semver",-]--[[package]]-name = "ryu"-version = "1.0.18"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"--[[package]]-name = "scopeguard"-version = "1.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"--[[package]]-name = "selectors"-version = "0.22.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"-dependencies = [- "bitflags 1.3.2",- "cssparser 0.27.2",- "derive_more",- "fxhash",- "log",- "matches",- "phf 0.8.0",- "phf_codegen 0.8.0",- "precomputed-hash",- "servo_arc 0.1.1",- "smallvec",- "thin-slice",-]--[[package]]-name = "selectors"-version = "0.24.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0c37578180969d00692904465fb7f6b3d50b9a2b952b87c23d0e2e5cb5013416"-dependencies = [- "bitflags 1.3.2",- "cssparser 0.29.6",- "derive_more",- "fxhash",- "log",- "phf 0.8.0",- "phf_codegen 0.8.0",- "precomputed-hash",- "servo_arc 0.2.0",- "smallvec",-]--[[package]]-name = "semver"-version = "1.0.25"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03"--[[package]]-name = "serde"-version = "1.0.217"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"-dependencies = [- "serde_derive",-]--[[package]]-name = "serde_derive"-version = "1.0.217"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",-]--[[package]]-name = "serde_json"-version = "1.0.137"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b"-dependencies = [- "itoa 1.0.14",- "memchr",- "ryu",- "serde",-]--[[package]]-name = "servo_arc"-version = "0.1.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"-dependencies = [- "nodrop",- "stable_deref_trait",-]--[[package]]-name = "servo_arc"-version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d52aa42f8fdf0fed91e5ce7f23d8138441002fa31dca008acf47e6fd4721f741"-dependencies = [- "nodrop",- "stable_deref_trait",-]--[[package]]-name = "sha2"-version = "0.10.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"-dependencies = [- "cfg-if",- "cpufeatures",- "digest",-]--[[package]]-name = "siphasher"-version = "0.3.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"--[[package]]-name = "smallvec"-version = "1.13.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"--[[package]]-name = "stable_deref_trait"-version = "1.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"--[[package]]-name = "string_cache"-version = "0.8.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"-dependencies = [- "new_debug_unreachable",- "once_cell",- "parking_lot",- "phf_shared 0.10.0",- "precomputed-hash",- "serde",-]--[[package]]-name = "string_cache_codegen"-version = "0.5.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"-dependencies = [- "phf_generator 0.10.0",- "phf_shared 0.10.0",- "proc-macro2",- "quote",-]--[[package]]-name = "syn"-version = "1.0.109"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"-dependencies = [- "proc-macro2",- "quote",- "unicode-ident",-]--[[package]]-name = "syn"-version = "2.0.96"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"-dependencies = [- "proc-macro2",- "quote",- "unicode-ident",-]--[[package]]-name = "synstructure"-version = "0.13.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",-]--[[package]]-name = "tendril"-version = "0.4.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"-dependencies = [- "futf",- "mac",- "utf-8",-]--[[package]]-name = "thin-slice"-version = "0.1.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"--[[package]]-name = "thiserror"-version = "2.0.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"-dependencies = [- "thiserror-impl",-]--[[package]]-name = "thiserror-impl"-version = "2.0.11"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",-]--[[package]]-name = "tinystr"-version = "0.7.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"-dependencies = [- "displaydoc",- "zerovec",-]--[[package]]-name = "typenum"-version = "1.18.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"--[[package]]-name = "unicode-ident"-version = "1.0.15"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "11cd88e12b17c6494200a9c1b683a04fcac9573ed74cd1b62aeb2727c5592243"--[[package]]-name = "url"-version = "2.5.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"-dependencies = [- "form_urlencoded",- "idna",- "percent-encoding",-]--[[package]]-name = "utf-8"-version = "0.7.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"--[[package]]-name = "utf16_iter"-version = "1.0.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"--[[package]]-name = "utf8_iter"-version = "1.0.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"--[[package]]-name = "version_check"-version = "0.9.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"--[[package]]-name = "wasi"-version = "0.9.0+wasi-snapshot-preview1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"--[[package]]-name = "wasi"-version = "0.11.0+wasi-snapshot-preview1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"--[[package]]-name = "wasi"-version = "0.14.2+wasi-0.2.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"-dependencies = [- "wit-bindgen-rt",-]--[[package]]-name = "windows-targets"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"-dependencies = [- "windows_aarch64_gnullvm",- "windows_aarch64_msvc",- "windows_i686_gnu",- "windows_i686_gnullvm",- "windows_i686_msvc",- "windows_x86_64_gnu",- "windows_x86_64_gnullvm",- "windows_x86_64_msvc",-]--[[package]]-name = "windows_aarch64_gnullvm"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"--[[package]]-name = "windows_aarch64_msvc"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"--[[package]]-name = "windows_i686_gnu"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"--[[package]]-name = "windows_i686_gnullvm"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"--[[package]]-name = "windows_i686_msvc"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"--[[package]]-name = "windows_x86_64_gnu"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"--[[package]]-name = "windows_x86_64_gnullvm"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"--[[package]]-name = "windows_x86_64_msvc"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"--[[package]]-name = "wit-bindgen-rt"-version = "0.39.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"-dependencies = [- "bitflags 2.8.0",-]--[[package]]-name = "write16"-version = "1.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"--[[package]]-name = "writeable"-version = "0.5.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"--[[package]]-name = "yoke"-version = "0.7.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"-dependencies = [- "serde",- "stable_deref_trait",- "yoke-derive",- "zerofrom",-]--[[package]]-name = "yoke-derive"-version = "0.7.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",- "synstructure",-]--[[package]]-name = "zerocopy"-version = "0.7.35"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"-dependencies = [- "byteorder",- "zerocopy-derive",-]--[[package]]-name = "zerocopy-derive"-version = "0.7.35"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",-]--[[package]]-name = "zerofrom"-version = "0.1.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e"-dependencies = [- "zerofrom-derive",-]--[[package]]-name = "zerofrom-derive"-version = "0.1.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",- "synstructure",-]--[[package]]-name = "zerovec"-version = "0.10.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"-dependencies = [- "yoke",- "zerofrom",- "zerovec-derive",-]--[[package]]-name = "zerovec-derive"-version = "0.10.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.96",-]diff --git a/apps/api/sharedLibs/html-transformer/Cargo.toml b/apps/api/sharedLibs/html-transformer/Cargo.tomldeleted file mode 100644index 34d06a614..000000000--- a/apps/api/sharedLibs/html-transformer/Cargo.toml+++ /dev/null@@ -1,16 +0,0 @@-[package]-name = "html-transformer"-version = "0.1.0"-edition = "2021"--[dependencies]-libc = "0.2.0"-lol_html = "2.2.0"-kuchikiki = "0.8.2"-serde = { version = "1.0", features = ["derive"] }-serde_json = "1.0"-url = "2.5.4"-nodesig = { git = "https://github.com/firecrawl/nodesig" }--[lib]-crate-type = ["cdylib"]diff --git a/apps/api/sharedLibs/html-transformer/src/lib.rs b/apps/api/sharedLibs/html-transformer/src/lib.rsdeleted file mode 100644index 5a93e8a86..000000000--- a/apps/api/sharedLibs/html-transformer/src/lib.rs+++ /dev/null@@ -1,823 +0,0 @@-use std::{collections::{HashMap, HashSet}, ffi::{CStr, CString}};--use nodesig::{get_node_signature, SignatureMode};-use kuchikiki::{iter::NodeEdge, parse_html, traits::TendrilSink, NodeRef};-use serde::Deserialize;-use serde_json::Value;-use url::Url;--fn _extract_base_href_from_document(document: &NodeRef, url: &Url) -> Result<String, Box<dyn std::error::Error>> {- if let Some(base) = document.select("base[href]").map_err(|_| "Failed to select base href".to_string())?.next()- .and_then(|base| base.attributes.borrow().get("href").map(|x| x.to_string())) {-- if let Ok(base) = url.join(&base) {- return Ok(base.to_string());- }- }-- Ok(url.to_string())-}--fn _extract_base_href(html: &str, url: &str) -> Result<String, Box<dyn std::error::Error>> {- let document = parse_html().one(html);- let url = Url::parse(url)?;-- _extract_base_href_from_document(&document, &url)-}--/// Extracts base href from HTML-///-/// # Safety-/// Input must be a C HTML string and a C URL string. Output will be a string. Output string must be freed with free_string.-#[no_mangle]-pub unsafe extern "C" fn extract_base_href(html: *const libc::c_char, url: *const libc::c_char) -> *mut libc::c_char {- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();- }- };- let url = match unsafe { CStr::from_ptr(url) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input URL as C string").unwrap().into_raw();- }- };-- let base_href = match _extract_base_href(html, url) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- CString::new(base_href).unwrap().into_raw()-}--/// Extracts links from HTML-///-/// # Safety-/// Input options must be a C HTML string. Output will be a JSON string array. Output string must be freed with free_string.-#[no_mangle]-pub unsafe extern "C" fn extract_links(html: *const libc::c_char) -> *mut libc::c_char {- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();- }- };-- let document = parse_html().one(html);-- let mut out: Vec<String> = Vec::new();-- let anchors: Vec<_> = match document.select("a[href]").map_err(|_| "Failed to select links") {- Ok(x) => x.collect(),- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- for anchor in anchors {- let mut href = match anchor.attributes.borrow().get("href") {- Some(x) => x.to_string(),- None => continue,- };-- if href.starts_with("http:/") && !href.starts_with("http://") {- href = format!("http://{}", &href[6..]);- } else if href.starts_with("https:/") && !href.starts_with("https://") {- href = format!("https://{}", &href[7..]);- }-- out.push(href);- }-- let links_out = match serde_json::ser::to_string(&out) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- CString::new(links_out).unwrap().into_raw()-}--macro_rules! insert_meta_name {- ($out:ident, $document:ident, $metaName:expr, $outName:expr) => {- if let Some(x) = $document.select(&format!("meta[name=\"{}\"]", $metaName)).map_err(|_| "Failed to select meta name")?.next().and_then(|description| description.attributes.borrow().get("content").map(|x| x.to_string())) {- $out.insert(($outName).to_string(), Value::String(x));- }- };-}--macro_rules! insert_meta_property {- ($out:ident, $document:ident, $metaName:expr, $outName:expr) => {- if let Some(x) = $document.select(&format!("meta[property=\"{}\"]", $metaName)).map_err(|_| "Failed to select meta property")?.next().and_then(|description| description.attributes.borrow().get("content").map(|x| x.to_string())) {- $out.insert(($outName).to_string(), Value::String(x));- }- };-}--fn _extract_metadata(html: &str) -> Result<String, Box<dyn std::error::Error>> {- let document = parse_html().one(html);- let mut out = HashMap::<String, Value>::new();-- if let Some(title) = document.select("title").map_err(|_| "Failed to select title")?.next() {- out.insert("title".to_string(), Value::String(title.text_contents()));- }- // insert_meta_name!(out, document, "description", "description");-- if let Some(favicon_link) = document.select("link[rel=\"icon\"]").map_err(|_| "Failed to select favicon")?.next()- .and_then(|x| x.attributes.borrow().get("href").map(|x| x.to_string()))- .or_else(|| document.select("link[rel*=\"icon\"]").unwrap().next()- .and_then(|x| x.attributes.borrow().get("href").map(|x| x.to_string()))) {- out.insert("favicon".to_string(), Value::String(favicon_link));- }-- if let Some(lang) = document.select("html[lang]").map_err(|_| "Failed to select lang")?.next().and_then(|x| x.attributes.borrow().get("lang").map(|x| x.to_string())) {- out.insert("language".to_string(), Value::String(lang));- }-- // insert_meta_name!(out, document, "keywords", "keywords");- // insert_meta_name!(out, document, "robots", "robots");- insert_meta_property!(out, document, "og:title", "ogTitle");- insert_meta_property!(out, document, "og:description", "ogDescription");- insert_meta_property!(out, document, "og:url", "ogUrl");- insert_meta_property!(out, document, "og:image", "ogImage");- insert_meta_property!(out, document, "og:audio", "ogAudio");- insert_meta_property!(out, document, "og:determiner", "ogDeterminer");- insert_meta_property!(out, document, "og:locale", "ogLocale");-- for meta in document.select("meta[property=\"og:locale:alternate\"]").map_err(|_| "Failed to select og locale alternate")? {- let attrs = meta.attributes.borrow();-- if let Some(content) = attrs.get("content") {- if let Some(v) = out.get_mut("ogLocaleAlternate") {- match v {- Value::Array(x) => {- x.push(Value::String(content.to_string()));- },- _ => unreachable!(),- }- } else {- out.insert("ogLocaleAlternate".to_string(), Value::Array(vec! [Value::String(content.to_string())]));- }- }- }-- insert_meta_property!(out, document, "og:site_name", "ogSiteName");- insert_meta_property!(out, document, "og:video", "ogVideo");- insert_meta_name!(out, document, "article:section", "articleSection");- insert_meta_name!(out, document, "article:tag", "articleTag");- insert_meta_property!(out, document, "article:published_time", "publishedTime");- insert_meta_property!(out, document, "article:modified_time", "modifiedTime");- insert_meta_name!(out, document, "dcterms.keywords", "dcTermsKeywords");- insert_meta_name!(out, document, "dc.description", "dcDescription");- insert_meta_name!(out, document, "dc.subject", "dcSubject");- insert_meta_name!(out, document, "dcterms.subject", "dcTermsSubject");- insert_meta_name!(out, document, "dcterms.audience", "dcTermsAudience");- insert_meta_name!(out, document, "dc.type", "dcType");- insert_meta_name!(out, document, "dcterms.type", "dcTermsType");- insert_meta_name!(out, document, "dc.date", "dcDate");- insert_meta_name!(out, document, "dc.date.created", "dcDateCreated");- insert_meta_name!(out, document, "dcterms.created", "dcTermsCreated");-- for meta in document.select("meta").map_err(|_| "Failed to select meta")? {- let meta = meta.as_node().as_element().unwrap();- let attrs = meta.attributes.borrow();-- if let Some(name) = attrs.get("name").or_else(|| attrs.get("property")).or_else(|| attrs.get("itemprop")) {- if let Some(content) = attrs.get("content") {- if let Some(v) = out.get(name) {- match v {- Value::String(existing) => {- if name == "description" {- out.insert(name.to_string(), Value::String(format!("{}, {}", existing, content)));- } else if name != "title" { // preserve title tag in metadata- out.insert(name.to_string(), Value::Array(vec! [Value::String(existing.clone()), Value::String(content.to_string())]));- }- },- Value::Array(existing_array) => {- if name == "description" {- let mut values: Vec<String> = existing_array.iter()- .filter_map(|v| match v {- Value::String(s) => Some(s.clone()),- _ => None,- })- .collect();- values.push(content.to_string());- out.insert(name.to_string(), Value::String(values.join(", ")));- } else {- match out.get_mut(name) {- Some(Value::Array(x)) => {- x.push(Value::String(content.to_string()));- },- _ => unreachable!(),- }- }- },- _ => unreachable!(),- }- } else {- out.insert(name.to_string(), Value::String(content.to_string()));- }- }- }- }-- Ok(serde_json::ser::to_string(&out)?)-}--/// Extracts metadata from HTML-///-/// # Safety-/// Input options must be a C HTML string. Output will be a JSON object. Output string must be freed with free_string.-#[no_mangle]-pub unsafe extern "C" fn extract_metadata(html: *const libc::c_char) -> *mut libc::c_char {- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();- }- };-- let metadata_out = match _extract_metadata(html) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- CString::new(metadata_out).unwrap().into_raw()-}--const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [- "header",- "footer",- "nav",- "aside",- ".header",- ".top",- ".navbar",- "#header",- ".footer",- ".bottom",- "#footer",- ".sidebar",- ".side",- ".aside",- "#sidebar",- ".modal",- ".popup",- "#modal",- ".overlay",- ".ad",- ".ads",- ".advert",- "#ad",- ".lang-selector",- ".language",- "#language-selector",- ".social",- ".social-media",- ".social-links",- "#social",- ".menu",- ".navigation",- "#nav",- ".breadcrumbs",- "#breadcrumbs",- ".share",- "#share",- ".widget",- "#widget",- ".cookie",- "#cookie",- ".fc-decoration",-];--const FORCE_INCLUDE_MAIN_TAGS: [&str; 13] = [- "#main",-- // swoogo event software as .widget in all of their content- ".swoogo-cols",- ".swoogo-text",- ".swoogo-table-div",- ".swoogo-space",- ".swoogo-alert",- ".swoogo-sponsors",- ".swoogo-title",- ".swoogo-tabs",- ".swoogo-logo",- ".swoogo-image",- ".swoogo-button",- ".swoogo-agenda",-];--#[derive(Deserialize)]-struct TranformHTMLOptions {- html: String,- url: String,- include_tags: Vec<String>,- exclude_tags: Vec<String>,- only_main_content: bool,- omce_signatures: Option<Vec<String>>,-}--struct ImageSource {- url: String,- size: f64,- is_x: bool,-}--fn _transform_html_inner(opts: TranformHTMLOptions) -> Result<String, Box<dyn std::error::Error>> {- let mut document = parse_html().one(opts.html.as_ref());- let url = Url::parse(&_extract_base_href_from_document(&document, &Url::parse(&opts.url)?)?)?;-- if !opts.include_tags.is_empty() {- let new_document = parse_html().one("<div></div>");- let root = new_document.select_first("div").map_err(|_| "Failed to select root element")?;-- for x in opts.include_tags.iter() {- let matching_nodes: Vec<_> = document.select(x).map_err(|_| "Failed to include_tags tags")?.collect();- for tag in matching_nodes {- root.as_node().append(tag.as_node().clone());- }- }-- document = new_document;- }-- while let Ok(x) = document.select_first("head") {- x.as_node().detach();- }-- while let Ok(x) = document.select_first("meta") {- x.as_node().detach();- }-- while let Ok(x) = document.select_first("noscript") {- x.as_node().detach();- }-- while let Ok(x) = document.select_first("style") {- x.as_node().detach();- }-- while let Ok(x) = document.select_first("script") {- x.as_node().detach();- }-- // The first operation applied must be OMCE.- if opts.only_main_content {- if let Some(signatures) = opts.omce_signatures.as_ref() {- let mut nodes_to_drop: Vec<NodeRef> = Vec::new();-- let modes = signatures.iter().map(|x| Into::<SignatureMode>::into(x.split(':').nth(1).unwrap().to_string())).collect::<HashSet<_>>();- for mode in modes {- let matcher = format!(":{}:", Into::<String>::into(mode));- let signatures = signatures.iter().filter(|x| x.contains(&matcher)).cloned().collect::<HashSet<_>>();-- for edge in document.traverse() {- match edge {- NodeEdge::Start(_) => {},- NodeEdge::End(node) => {- if node.as_element().is_none() {- continue;- }-- if node.text_contents().trim().is_empty() {- continue;- }-- let signature = get_node_signature(&node, mode);- if signatures.contains(&signature) {- nodes_to_drop.push(node);- }- }- }- }- }-- for node in nodes_to_drop {- node.detach();- }- }- }-- for x in opts.exclude_tags.iter() {- // TODO: implement weird version- while let Ok(x) = document.select_first(x) {- x.as_node().detach();- }- }-- if opts.only_main_content {- for x in EXCLUDE_NON_MAIN_TAGS.iter() {- let x: Vec<_> = document.select(x).map_err(|_| "Failed to select tags")?.collect();- for tag in x {- if !FORCE_INCLUDE_MAIN_TAGS.iter().any(|x| tag.as_node().select(x).is_ok_and(|mut x| x.next().is_some())) {- tag.as_node().detach();- }- }- }- }-- let srcset_images: Vec<_> = document.select("img[srcset]").map_err(|_| "Failed to select srcset images")?.collect();- for img in srcset_images {- let mut sizes: Vec<ImageSource> = img.attributes.borrow().get("srcset").ok_or("Failed to get srcset")?.split(",").filter_map(|x| {- let tok: Vec<&str> = x.trim().split(" ").collect();- let last_token = tok[tok.len() - 1]; // SAFETY: split is guaranteed to return at least one token- let (last_token, last_token_used) = if tok.len() > 1 && !last_token.is_empty() && (last_token.ends_with("x") || last_token.ends_with("w")) {- (last_token, true)- } else {- ("1x", false)- };-- // split off the last character of the last token and parse as size- if let Some((last_index, _)) = last_token.char_indices().last() {- if let Ok(parsed_size) = last_token[..last_index].parse() {- Some(ImageSource {- url: if last_token_used { tok[0..tok.len()-1].join(" ") } else { tok.join(" ") },- size: parsed_size,- is_x: last_token.ends_with("x")- })- } else {- None- }- } else {- None- }- }).collect();-- if sizes.iter().all(|x| x.is_x) {- if let Some(src) = img.attributes.borrow().get("src").map(|x| x.to_string()) {- sizes.push(ImageSource {- url: src,- size: 1.0,- is_x: true,- });- }- }-- sizes.sort_by(|a, b| b.size.partial_cmp(&a.size).unwrap_or(std::cmp::Ordering::Equal));-- if let Some(biggest) = sizes.first() {- img.attributes.borrow_mut().insert("src", biggest.url.clone());- }- }-- let src_images: Vec<_> = document.select("img[src]").map_err(|_| "Failed to select src images")?.collect();- for img in src_images {- let old = img.attributes.borrow().get("src").map(|x| x.to_string()).ok_or("Failed to get src")?;- if let Ok(new) = url.join(&old) {- img.attributes.borrow_mut().insert("src", new.to_string());- }- }-- let href_anchors: Vec<_> = document.select("a[href]").map_err(|_| "Failed to select href anchors")?.collect();- for anchor in href_anchors {- let old = anchor.attributes.borrow().get("href").map(|x| x.to_string()).ok_or("Failed to get href")?;- if let Ok(new) = url.join(&old) {- anchor.attributes.borrow_mut().insert("href", new.to_string());- }- }-- Ok(document.to_string())-}--/// Transforms rawHtml to html (formerly removeUnwantedElements)-///-/// # Safety-/// Input options must be a C JSON string. Output will be an HTML string. Output string must be freed with free_string.-#[no_mangle]-pub unsafe extern "C" fn transform_html(opts: *const libc::c_char) -> *mut libc::c_char {- let opts: TranformHTMLOptions = match unsafe { CStr::from_ptr(opts) }.to_str().map_err(|_| ()).and_then(|x| serde_json::de::from_str(x).map_err(|_| ())) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR").unwrap().into_raw();- }- };-- let out = match _transform_html_inner(opts) {- Ok(x) => x,- Err(e) => format!("RUSTFC:ERROR:{}", e),- };-- CString::new(out).unwrap().into_raw()-}--fn _get_inner_json(html: &str) -> Result<String, ()> {- Ok(parse_html().one(html).select_first("body")?.text_contents())-}--/// For JSON pages retrieved by browser engines, this function can be used to transform it back into valid JSON.-///-/// # Safety-/// Input must be a C HTML string. Output will be an HTML string. Output string must be freed with free_string.-#[no_mangle]-pub unsafe extern "C" fn get_inner_json(html: *const libc::c_char) -> *mut libc::c_char {- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();- }- };-- let out = match _get_inner_json(html) {- Ok(x) => x,- Err(_) => "RUSTFC:ERROR".to_string(),- };-- CString::new(out).unwrap().into_raw()-}--#[derive(Deserialize)]-struct AttributeSelector {- selector: String,- attribute: String,-}--#[derive(Deserialize)]-struct ExtractAttributesOptions {- selectors: Vec<AttributeSelector>,-}--fn _extract_attributes(html: &str, options: &ExtractAttributesOptions) -> Result<String, Box<dyn std::error::Error>> {- let document = parse_html().one(html);- let mut results = Vec::new();-- for selector_config in &options.selectors {- let mut values = Vec::new();-- let elements: Vec<_> = match document.select(&selector_config.selector).map_err(|_| format!("Failed to select with selector: {}", selector_config.selector)) {- Ok(x) => x.collect(),- Err(_) => {- // Return empty values for invalid selectors rather than failing completely- Vec::new()- }- };-- for element in elements {- // Try the attribute as-is first- if let Some(attr_value) = element.attributes.borrow().get(selector_config.attribute.as_str()) {- values.push(attr_value.to_string());- continue;- }-- // If not found and doesn't start with 'data-', try with 'data-' prefix- if !selector_config.attribute.starts_with("data-") {- let data_attr = format!("data-{}", selector_config.attribute);- if let Some(attr_value) = element.attributes.borrow().get(data_attr.as_str()) {- values.push(attr_value.to_string());- }- }- }-- results.push(serde_json::json!({- "selector": selector_config.selector,- "attribute": selector_config.attribute,- "values": values- }));- }-- Ok(serde_json::to_string(&results)?)-}--/// Extracts attributes from HTML elements using CSS selectors-///-/// # Safety-/// html must be a C HTML string. options must be a C JSON string with format:-/// {"selectors": [{"selector": "...", "attribute": "..."}]}-/// Output will be a JSON string. Output string must be freed with free_string.-#[no_mangle]-pub unsafe extern "C" fn extract_attributes(html: *const libc::c_char, options: *const libc::c_char) -> *mut libc::c_char {- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();- }- };-- let options_str = match unsafe { CStr::from_ptr(options) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input options as C string").unwrap().into_raw();- }- };-- let parsed_options: ExtractAttributesOptions = match serde_json::from_str(options_str) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Failed to parse options JSON: {}", e)).unwrap().into_raw();- }- };-- let result = match _extract_attributes(html, &parsed_options) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- CString::new(result).unwrap().into_raw()-}--fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn std::error::Error>> {- let document = parse_html().one(html);- let base_url = Url::parse(base_url)?;- let base_href = _extract_base_href_from_document(&document, &base_url)?;- let base_href_url = Url::parse(&base_href)?;- let mut images = HashSet::<String>::new();-- // Helper function to resolve image URLs- let resolve_image_url = |src: &str| -> Result<String, Box<dyn std::error::Error>> {- // Skip data URIs and blob URLs- if src.starts_with("data:") || src.starts_with("blob:") {- return Ok(src.to_string());- }-- // Handle absolute URLs- if src.starts_with("http://") || src.starts_with("https://") {- return Ok(src.to_string());- }-- // Handle protocol-relative URLs- if src.starts_with("//") {- let resolved = base_url.join(src)?;- return Ok(resolved.to_string());- }-- // Handle relative URLs- let resolved = base_href_url.join(src)?;- Ok(resolved.to_string())- };-- // Extract from img tags- let img_elements: Vec<_> = match document.select("img").map_err(|_| "Failed to select img tags") {- Ok(x) => x.collect(),- Err(e) => return Err(e.into()),- };-- for img in img_elements {- let attrs = img.attributes.borrow();-- // Extract from src attribute- if let Some(src) = attrs.get("src") {- if let Ok(resolved) = resolve_image_url(src) {- images.insert(resolved);- }- }-- // Extract from data-src (lazy loading)- if let Some(data_src) = attrs.get("data-src") {- if let Ok(resolved) = resolve_image_url(data_src) {- images.insert(resolved);- }- }-- // Extract from srcset (responsive images)- if let Some(srcset) = attrs.get("srcset") {- for part in srcset.split(',') {- if let Some(url) = part.trim().split_whitespace().next() {- if !url.is_empty() {- if let Ok(resolved) = resolve_image_url(url) {- images.insert(resolved);- }- }- }- }- }- }-- // Extract from picture source elements- let source_elements: Vec<_> = match document.select("picture source").map_err(|_| "Failed to select picture source") {- Ok(x) => x.collect(),- Err(_) => Vec::new(),- };-- for source in source_elements {- if let Some(srcset) = source.attributes.borrow().get("srcset") {- for part in srcset.split(',') {- if let Some(url) = part.trim().split_whitespace().next() {- if !url.is_empty() {- if let Ok(resolved) = resolve_image_url(url) {- images.insert(resolved);- }- }- }- }- }- }-- // Extract from meta tags (Open Graph, Twitter Cards)- let meta_selectors = [- "meta[property=\"og:image\"]",- "meta[property=\"og:image:url\"]",- "meta[property=\"og:image:secure_url\"]",- "meta[name=\"twitter:image\"]",- "meta[name=\"twitter:image:src\"]",- "meta[itemprop=\"image\"]"- ];-- for selector in &meta_selectors {- if let Ok(elements) = document.select(selector) {- for element in elements {- if let Some(content) = element.attributes.borrow().get("content") {- if let Ok(resolved) = resolve_image_url(content) {- images.insert(resolved);- }- }- }- }- }-- // Extract from link tags (favicons, apple-touch-icons)- let link_selectors = [- "link[rel*=\"icon\"]",- "link[rel*=\"apple-touch-icon\"]",- "link[rel*=\"image_src\"]"- ];-- for selector in &link_selectors {- if let Ok(elements) = document.select(selector) {- for element in elements {- if let Some(href) = element.attributes.borrow().get("href") {- if let Ok(resolved) = resolve_image_url(href) {- images.insert(resolved);- }- }- }- }- }-- // Extract from video poster attributes- if let Ok(video_elements) = document.select("video[poster]") {- for video in video_elements {- if let Some(poster) = video.attributes.borrow().get("poster") {- if let Ok(resolved) = resolve_image_url(poster) {- images.insert(resolved);- }- }- }- }-- // Filter out javascript: URLs for security and validate URLs- let filtered_images: Vec<String> = images.into_iter()- .filter(|url| !url.to_lowercase().starts_with("javascript:"))- .filter(|url| !url.is_empty())- .filter(|url| {- // Validate URLs (allow data URIs and valid URLs)- url.starts_with("data:") || url.starts_with("blob:") || Url::parse(url).is_ok()- })- .collect();-- Ok(filtered_images)-}--/// Extracts images from HTML-///-/// # Safety-/// Input must be a C HTML string and base URL string. Output will be a JSON string array. Output string must be freed with free_string.-#[no_mangle]-pub unsafe extern "C" fn extract_images(html: *const libc::c_char, base_url: *const libc::c_char) -> *mut libc::c_char {- let html = match unsafe { CStr::from_ptr(html) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input HTML as C string").unwrap().into_raw();- }- };-- let base_url = match unsafe { CStr::from_ptr(base_url) }.to_str().map_err(|_| ()) {- Ok(x) => x,- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input base URL as C string").unwrap().into_raw();- }- };-- let images = match _extract_images(html, base_url) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- let images_out = match serde_json::ser::to_string(&images) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:{}", e)).unwrap().into_raw();- }- };-- CString::new(images_out).unwrap().into_raw()-}--/// Frees a string allocated in Rust-land.-///-/// # Safety-/// ptr must be a non-freed string pointer returned by Rust code.-#[no_mangle]-pub unsafe extern "C" fn free_string(ptr: *mut libc::c_char) {- drop(unsafe { CString::from_raw(ptr) })-}diff --git a/apps/api/sharedLibs/pdf-parser/.gitignore b/apps/api/sharedLibs/pdf-parser/.gitignoredeleted file mode 100644index 1de565933..000000000--- a/apps/api/sharedLibs/pdf-parser/.gitignore+++ /dev/null@@ -1 +0,0 @@-target\ No newline at end of filediff --git a/apps/api/sharedLibs/pdf-parser/Cargo.lock b/apps/api/sharedLibs/pdf-parser/Cargo.lockdeleted file mode 100644index 5635c68aa..000000000--- a/apps/api/sharedLibs/pdf-parser/Cargo.lock+++ /dev/null@@ -1,1019 +0,0 @@-# This file is automatically @generated by Cargo.-# It is not intended for manual editing.-version = 4--[[package]]-name = "adler2"-version = "2.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"--[[package]]-name = "aes"-version = "0.8.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"-dependencies = [- "cfg-if",- "cipher",- "cpufeatures",-]--[[package]]-name = "android-tzdata"-version = "0.1.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"--[[package]]-name = "android_system_properties"-version = "0.1.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"-dependencies = [- "libc",-]--[[package]]-name = "autocfg"-version = "1.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"--[[package]]-name = "bitflags"-version = "2.9.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"--[[package]]-name = "block-buffer"-version = "0.10.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"-dependencies = [- "generic-array",-]--[[package]]-name = "block-padding"-version = "0.3.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"-dependencies = [- "generic-array",-]--[[package]]-name = "bumpalo"-version = "3.17.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"--[[package]]-name = "bytecount"-version = "0.6.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"--[[package]]-name = "cbc"-version = "0.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"-dependencies = [- "cipher",-]--[[package]]-name = "cc"-version = "1.2.23"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766"-dependencies = [- "shlex",-]--[[package]]-name = "cfg-if"-version = "1.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"--[[package]]-name = "chrono"-version = "0.4.41"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d"-dependencies = [- "android-tzdata",- "iana-time-zone",- "num-traits",- "windows-link",-]--[[package]]-name = "cipher"-version = "0.4.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"-dependencies = [- "crypto-common",- "inout",-]--[[package]]-name = "core-foundation-sys"-version = "0.8.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"--[[package]]-name = "cpufeatures"-version = "0.2.17"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"-dependencies = [- "libc",-]--[[package]]-name = "crc32fast"-version = "1.4.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"-dependencies = [- "cfg-if",-]--[[package]]-name = "crossbeam-deque"-version = "0.8.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"-dependencies = [- "crossbeam-epoch",- "crossbeam-utils",-]--[[package]]-name = "crossbeam-epoch"-version = "0.9.18"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"-dependencies = [- "crossbeam-utils",-]--[[package]]-name = "crossbeam-utils"-version = "0.8.21"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"--[[package]]-name = "crypto-common"-version = "0.1.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"-dependencies = [- "generic-array",- "typenum",-]--[[package]]-name = "deranged"-version = "0.4.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e"-dependencies = [- "powerfmt",-]--[[package]]-name = "digest"-version = "0.10.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"-dependencies = [- "block-buffer",- "crypto-common",-]--[[package]]-name = "ecb"-version = "0.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7"-dependencies = [- "cipher",-]--[[package]]-name = "either"-version = "1.15.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"--[[package]]-name = "encoding_rs"-version = "0.8.35"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"-dependencies = [- "cfg-if",-]--[[package]]-name = "equivalent"-version = "1.0.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"--[[package]]-name = "flate2"-version = "1.1.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"-dependencies = [- "crc32fast",- "miniz_oxide",-]--[[package]]-name = "generic-array"-version = "0.14.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"-dependencies = [- "typenum",- "version_check",-]--[[package]]-name = "getrandom"-version = "0.3.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"-dependencies = [- "cfg-if",- "libc",- "r-efi",- "wasi",-]--[[package]]-name = "hashbrown"-version = "0.15.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3"--[[package]]-name = "iana-time-zone"-version = "0.1.63"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8"-dependencies = [- "android_system_properties",- "core-foundation-sys",- "iana-time-zone-haiku",- "js-sys",- "log",- "wasm-bindgen",- "windows-core",-]--[[package]]-name = "iana-time-zone-haiku"-version = "0.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"-dependencies = [- "cc",-]--[[package]]-name = "indexmap"-version = "2.9.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"-dependencies = [- "equivalent",- "hashbrown",-]--[[package]]-name = "inout"-version = "0.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"-dependencies = [- "block-padding",- "generic-array",-]--[[package]]-name = "itoa"-version = "1.0.15"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"--[[package]]-name = "jiff"-version = "0.2.14"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93"-dependencies = [- "jiff-static",- "jiff-tzdb-platform",- "log",- "portable-atomic",- "portable-atomic-util",- "serde",- "windows-sys",-]--[[package]]-name = "jiff-static"-version = "0.2.14"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "jiff-tzdb"-version = "0.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c1283705eb0a21404d2bfd6eef2a7593d240bc42a0bdb39db0ad6fa2ec026524"--[[package]]-name = "jiff-tzdb-platform"-version = "0.1.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8"-dependencies = [- "jiff-tzdb",-]--[[package]]-name = "js-sys"-version = "0.3.77"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"-dependencies = [- "once_cell",- "wasm-bindgen",-]--[[package]]-name = "libc"-version = "0.2.172"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"--[[package]]-name = "log"-version = "0.4.27"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"--[[package]]-name = "lopdf"-version = "0.36.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "59fa2559e99ba0f26a12458aabc754432c805bbb8cba516c427825a997af1fb7"-dependencies = [- "aes",- "bitflags",- "cbc",- "chrono",- "ecb",- "encoding_rs",- "flate2",- "indexmap",- "itoa",- "jiff",- "log",- "md-5",- "nom",- "nom_locate",- "rand",- "rangemap",- "rayon",- "sha2",- "stringprep",- "thiserror",- "time",- "weezl",-]--[[package]]-name = "md-5"-version = "0.10.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"-dependencies = [- "cfg-if",- "digest",-]--[[package]]-name = "memchr"-version = "2.7.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"--[[package]]-name = "miniz_oxide"-version = "0.8.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"-dependencies = [- "adler2",-]--[[package]]-name = "nom"-version = "8.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"-dependencies = [- "memchr",-]--[[package]]-name = "nom_locate"-version = "5.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d"-dependencies = [- "bytecount",- "memchr",- "nom",-]--[[package]]-name = "num-conv"-version = "0.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"--[[package]]-name = "num-traits"-version = "0.2.19"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"-dependencies = [- "autocfg",-]--[[package]]-name = "once_cell"-version = "1.21.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"--[[package]]-name = "pdf-parser"-version = "0.1.0"-dependencies = [- "libc",- "lopdf",- "serde",- "serde_json",-]--[[package]]-name = "portable-atomic"-version = "1.11.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"--[[package]]-name = "portable-atomic-util"-version = "0.2.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"-dependencies = [- "portable-atomic",-]--[[package]]-name = "powerfmt"-version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"--[[package]]-name = "ppv-lite86"-version = "0.2.21"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"-dependencies = [- "zerocopy",-]--[[package]]-name = "proc-macro2"-version = "1.0.95"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"-dependencies = [- "unicode-ident",-]--[[package]]-name = "quote"-version = "1.0.40"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"-dependencies = [- "proc-macro2",-]--[[package]]-name = "r-efi"-version = "5.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"--[[package]]-name = "rand"-version = "0.9.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"-dependencies = [- "rand_chacha",- "rand_core",-]--[[package]]-name = "rand_chacha"-version = "0.9.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"-dependencies = [- "ppv-lite86",- "rand_core",-]--[[package]]-name = "rand_core"-version = "0.9.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"-dependencies = [- "getrandom",-]--[[package]]-name = "rangemap"-version = "1.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f60fcc7d6849342eff22c4350c8b9a989ee8ceabc4b481253e8946b9fe83d684"--[[package]]-name = "rayon"-version = "1.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"-dependencies = [- "either",- "rayon-core",-]--[[package]]-name = "rayon-core"-version = "1.12.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"-dependencies = [- "crossbeam-deque",- "crossbeam-utils",-]--[[package]]-name = "rustversion"-version = "1.0.21"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"--[[package]]-name = "ryu"-version = "1.0.20"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"--[[package]]-name = "serde"-version = "1.0.219"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"-dependencies = [- "serde_derive",-]--[[package]]-name = "serde_derive"-version = "1.0.219"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "serde_json"-version = "1.0.143"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a"-dependencies = [- "itoa",- "memchr",- "ryu",- "serde",-]--[[package]]-name = "sha2"-version = "0.10.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"-dependencies = [- "cfg-if",- "cpufeatures",- "digest",-]--[[package]]-name = "shlex"-version = "1.3.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"--[[package]]-name = "stringprep"-version = "0.1.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"-dependencies = [- "unicode-bidi",- "unicode-normalization",- "unicode-properties",-]--[[package]]-name = "syn"-version = "2.0.101"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"-dependencies = [- "proc-macro2",- "quote",- "unicode-ident",-]--[[package]]-name = "thiserror"-version = "2.0.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"-dependencies = [- "thiserror-impl",-]--[[package]]-name = "thiserror-impl"-version = "2.0.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "time"-version = "0.3.41"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40"-dependencies = [- "deranged",- "itoa",- "num-conv",- "powerfmt",- "serde",- "time-core",- "time-macros",-]--[[package]]-name = "time-core"-version = "0.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"--[[package]]-name = "time-macros"-version = "0.2.22"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49"-dependencies = [- "num-conv",- "time-core",-]--[[package]]-name = "tinyvec"-version = "1.9.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"-dependencies = [- "tinyvec_macros",-]--[[package]]-name = "tinyvec_macros"-version = "0.1.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"--[[package]]-name = "typenum"-version = "1.18.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"--[[package]]-name = "unicode-bidi"-version = "0.3.18"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"--[[package]]-name = "unicode-ident"-version = "1.0.18"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"--[[package]]-name = "unicode-normalization"-version = "0.1.24"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"-dependencies = [- "tinyvec",-]--[[package]]-name = "unicode-properties"-version = "0.1.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"--[[package]]-name = "version_check"-version = "0.9.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"--[[package]]-name = "wasi"-version = "0.14.2+wasi-0.2.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"-dependencies = [- "wit-bindgen-rt",-]--[[package]]-name = "wasm-bindgen"-version = "0.2.100"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"-dependencies = [- "cfg-if",- "once_cell",- "rustversion",- "wasm-bindgen-macro",-]--[[package]]-name = "wasm-bindgen-backend"-version = "0.2.100"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"-dependencies = [- "bumpalo",- "log",- "proc-macro2",- "quote",- "syn",- "wasm-bindgen-shared",-]--[[package]]-name = "wasm-bindgen-macro"-version = "0.2.100"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"-dependencies = [- "quote",- "wasm-bindgen-macro-support",-]--[[package]]-name = "wasm-bindgen-macro-support"-version = "0.2.100"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"-dependencies = [- "proc-macro2",- "quote",- "syn",- "wasm-bindgen-backend",- "wasm-bindgen-shared",-]--[[package]]-name = "wasm-bindgen-shared"-version = "0.2.100"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"-dependencies = [- "unicode-ident",-]--[[package]]-name = "weezl"-version = "0.1.10"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3"--[[package]]-name = "windows-core"-version = "0.61.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"-dependencies = [- "windows-implement",- "windows-interface",- "windows-link",- "windows-result",- "windows-strings",-]--[[package]]-name = "windows-implement"-version = "0.60.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "windows-interface"-version = "0.59.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"-dependencies = [- "proc-macro2",- "quote",- "syn",-]--[[package]]-name = "windows-link"-version = "0.1.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"--[[package]]-name = "windows-result"-version = "0.3.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"-dependencies = [- "windows-link",-]--[[package]]-name = "windows-strings"-version = "0.4.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"-dependencies = [- "windows-link",-]--[[package]]-name = "windows-sys"-version = "0.59.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"-dependencies = [- "windows-targets",-]--[[package]]-name = "windows-targets"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"-dependencies = [- "windows_aarch64_gnullvm",- "windows_aarch64_msvc",- "windows_i686_gnu",- "windows_i686_gnullvm",- "windows_i686_msvc",- "windows_x86_64_gnu",- "windows_x86_64_gnullvm",- "windows_x86_64_msvc",-]--[[package]]-name = "windows_aarch64_gnullvm"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"--[[package]]-name = "windows_aarch64_msvc"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"--[[package]]-name = "windows_i686_gnu"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"--[[package]]-name = "windows_i686_gnullvm"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"--[[package]]-name = "windows_i686_msvc"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"--[[package]]-name = "windows_x86_64_gnu"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"--[[package]]-name = "windows_x86_64_gnullvm"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"--[[package]]-name = "windows_x86_64_msvc"-version = "0.52.6"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"--[[package]]-name = "wit-bindgen-rt"-version = "0.39.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"-dependencies = [- "bitflags",-]--[[package]]-name = "zerocopy"-version = "0.8.25"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb"-dependencies = [- "zerocopy-derive",-]--[[package]]-name = "zerocopy-derive"-version = "0.8.25"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"-dependencies = [- "proc-macro2",- "quote",- "syn",-]diff --git a/apps/api/sharedLibs/pdf-parser/Cargo.toml b/apps/api/sharedLibs/pdf-parser/Cargo.tomldeleted file mode 100644index 7f3945f64..000000000--- a/apps/api/sharedLibs/pdf-parser/Cargo.toml+++ /dev/null@@ -1,13 +0,0 @@-[package]-name = "pdf-parser"-version = "0.1.0"-edition = "2021"--[dependencies]-libc = "0.2.0"-lopdf = "0.36.0"-serde = { version = "1.0", features = ["derive"] }-serde_json = "1.0"--[lib]-crate-type = ["cdylib"]diff --git a/apps/api/sharedLibs/pdf-parser/src/lib.rs b/apps/api/sharedLibs/pdf-parser/src/lib.rsdeleted file mode 100644index b83262f4d..000000000--- a/apps/api/sharedLibs/pdf-parser/src/lib.rs+++ /dev/null@@ -1,68 +0,0 @@-use std::ffi::{CStr, CString};-use serde::Serialize;--#[derive(Debug, Clone, Serialize)]-#[serde(rename_all = "camelCase")]-pub struct PDFMetadata {- num_pages: i32,-- #[serde(skip_serializing_if = "Option::is_none")]- title: Option<String>,-}--fn _get_pdf_metadata(path: &str) -> Result<PDFMetadata, String> {- let doc = match lopdf::Document::load(path) {- Ok(x) => x,- Err(_) => {- return Err("Failed to load PDF".to_string());- }- };-- let num_pages = doc.get_pages().len() as i32;- let title = doc.trailer.get(b"Info")- .and_then(|info| info.as_dict()- .and_then(|info| info.get(b"Title"))- .and_then(|title| title.as_str().map(|s| String::from_utf8_lossy(s).to_string()))- ).ok()- .or_else(|| doc.objects.iter()- .find_map(|(_i, obj)| obj.as_dict()- .and_then(|obj| obj.get(b"Title"))- .and_then(|title| title.as_str())- .map(|s| String::from_utf8_lossy(s).to_string())- .ok()- )- )- .map(|x| x.trim_matches(|x| char::is_whitespace(x) || x == '\0').to_string());-- Ok(PDFMetadata { num_pages, title })-}--/// Returns the metadata of a PDF file-///-/// # Safety-/// Input path must be a C string of a path pointing to a PDF file. Output will be a JSON string of the PDF metadata.-#[no_mangle]-pub unsafe extern "C" fn get_pdf_metadata(path: *const libc::c_char) -> *const libc::c_char {- let path: String = match unsafe { CStr::from_ptr(path) }.to_str().map_err(|_| ()) {- Ok(x) => x.to_string(),- Err(_) => {- return CString::new("RUSTFC:ERROR:Failed to parse input path as C string").unwrap().into_raw();- }- };-- let metadata = match _get_pdf_metadata(&path) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Failed to get PDF metadata: {}", e)).unwrap().into_raw();- }- };-- let json = match serde_json::to_string(&metadata) {- Ok(x) => x,- Err(e) => {- return CString::new(format!("RUSTFC:ERROR:Serde failed to serialize metadata: {}", e)).unwrap().into_raw();- }- };-- CString::new(json).unwrap().into_raw()-}diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.tsindex 46e398feb..bb033d5cf 100644--- a/apps/api/src/__tests__/snips/v1/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts@@ -1,3 +1,4 @@+import { filterLinks } from "@mendable/firecrawl-rs";import {asyncCrawl,asyncCrawlWaitForFinish,@@ -9,7 +10,6 @@ import {scrapeTimeout,} from "./lib";import { describe, it, expect } from "@jest/globals";-import { filterLinks } from "../../../lib/crawler";let identity: Identity;@@ -398,23 +398,23 @@ describe("Robots.txt FFI Integration tests", () => {"https://example.com/disallowed",],limit: 10,- max_depth: 10,- base_url: "https://example.com",- initial_url: "https://example.com",- regex_on_full_url: false,+ maxDepth: 10,+ baseUrl: "https://example.com",+ initialUrl: "https://example.com",+ regexOnFullUrl: false,excludes: [],includes: [],- allow_backward_crawling: true,- ignore_robots_txt: false,- robots_txt: "User-agent: *\nDisallow: /disallowed",+ allowBackwardCrawling: true,+ ignoreRobotsTxt: false,+ robotsTxt: "User-agent: *\nDisallow: /disallowed",});expect(result.links).toHaveLength(1);expect(result.links[0]).toBe("https://example.com/allowed");- expect(result.denial_reasons.has("https://example.com/disallowed")).toBe(+ expect("https://example.com/disallowed" in result.denialReasons).toBe(true,);- expect(result.denial_reasons.get("https://example.com/disallowed")).toBe(+ expect(result.denialReasons["https://example.com/disallowed"]).toBe("ROBOTS_TXT",);},@@ -427,15 +427,15 @@ describe("Robots.txt FFI Integration tests", () => {const result = await filterLinks({links: ["https://example.com/test"],limit: 10,- max_depth: 10,- base_url: "https://example.com",- initial_url: "https://example.com",- regex_on_full_url: false,+ maxDepth: 10,+ baseUrl: "https://example.com",+ initialUrl: "https://example.com",+ regexOnFullUrl: false,excludes: [],includes: [],- allow_backward_crawling: true,- ignore_robots_txt: false,- robots_txt:+ allowBackwardCrawling: true,+ ignoreRobotsTxt: false,+ robotsTxt:"Invalid robots.txt content with \x00 null bytes and malformed syntax",});@@ -453,15 +453,15 @@ describe("Robots.txt FFI Integration tests", () => {const result = await filterLinks({links: ["https://example.com/allowed"],limit: 10,- max_depth: 10,- base_url: "https://example.com",- initial_url: "https://example.com",- regex_on_full_url: false,+ maxDepth: 10,+ baseUrl: "https://example.com",+ initialUrl: "https://example.com",+ regexOnFullUrl: false,excludes: [],includes: [],- allow_backward_crawling: true,- ignore_robots_txt: false,- robots_txt: nonUtf8Content,+ allowBackwardCrawling: true,+ ignoreRobotsTxt: false,+ robotsTxt: nonUtf8Content,});expect(result.links).toHaveLength(1);@@ -477,15 +477,15 @@ describe("Robots.txt FFI Integration tests", () => {const result = await filterLinks({links: ["https://example.com/safe"],limit: 10,- max_depth: 10,- base_url: "https://example.com",- initial_url: "https://example.com",- regex_on_full_url: false,+ maxDepth: 10,+ baseUrl: "https://example.com",+ initialUrl: "https://example.com",+ regexOnFullUrl: false,excludes: [],includes: [],- allow_backward_crawling: true,- ignore_robots_txt: false,- robots_txt: problematicContent,+ allowBackwardCrawling: true,+ ignoreRobotsTxt: false,+ robotsTxt: problematicContent,});expect(result.links).toHaveLength(1);diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 8eef9302b..70c077abc 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -9,7 +9,7 @@ import {scrapeTimeout,} from "./lib";import { describe, it, expect } from "@jest/globals";-import { filterLinks } from "../../../lib/crawler";+import { filterLinks } from "@mendable/firecrawl-rs";let identity: Identity;@@ -400,23 +400,23 @@ describe("Robots.txt FFI Integration tests", () => {"https://example.com/disallowed",],limit: 10,- max_depth: 10,- base_url: "https://example.com",- initial_url: "https://example.com",- regex_on_full_url: false,+ maxDepth: 10,+ baseUrl: "https://example.com",+ initialUrl: "https://example.com",+ regexOnFullUrl: false,excludes: [],includes: [],- allow_backward_crawling: true,- ignore_robots_txt: false,- robots_txt: "User-agent: *\nDisallow: /disallowed",+ allowBackwardCrawling: true,+ ignoreRobotsTxt: false,+ robotsTxt: "User-agent: *\nDisallow: /disallowed",});expect(result.links).toHaveLength(1);expect(result.links[0]).toBe("https://example.com/allowed");- expect(result.denial_reasons.has("https://example.com/disallowed")).toBe(+ expect("https://example.com/disallowed" in result.denialReasons).toBe(true,);- expect(result.denial_reasons.get("https://example.com/disallowed")).toBe(+ expect(result.denialReasons["https://example.com/disallowed"]).toBe("ROBOTS_TXT",);},@@ -429,15 +429,15 @@ describe("Robots.txt FFI Integration tests", () => {const result = await filterLinks({links: ["https://example.com/test"],limit: 10,- max_depth: 10,- base_url: "https://example.com",- initial_url: "https://example.com",- regex_on_full_url: false,+ maxDepth: 10,+ baseUrl: "https://example.com",+ initialUrl: "https://example.com",+ regexOnFullUrl: false,excludes: [],includes: [],- allow_backward_crawling: true,- ignore_robots_txt: false,- robots_txt:+ allowBackwardCrawling: true,+ ignoreRobotsTxt: false,+ robotsTxt:"Invalid robots.txt content with \x00 null bytes and malformed syntax",});@@ -455,15 +455,15 @@ describe("Robots.txt FFI Integration tests", () => {const result = await filterLinks({links: ["https://example.com/allowed"],limit: 10,- max_depth: 10,- base_url: "https://example.com",- initial_url: "https://example.com",- regex_on_full_url: false,+ maxDepth: 10,+ baseUrl: "https://example.com",+ initialUrl: "https://example.com",+ regexOnFullUrl: false,excludes: [],includes: [],- allow_backward_crawling: true,- ignore_robots_txt: false,- robots_txt: nonUtf8Content,+ allowBackwardCrawling: true,+ ignoreRobotsTxt: false,+ robotsTxt: nonUtf8Content,});expect(result.links).toHaveLength(1);@@ -479,15 +479,15 @@ describe("Robots.txt FFI Integration tests", () => {const result = await filterLinks({links: ["https://example.com/safe"],limit: 10,- max_depth: 10,- base_url: "https://example.com",- initial_url: "https://example.com",- regex_on_full_url: false,+ maxDepth: 10,+ baseUrl: "https://example.com",+ initialUrl: "https://example.com",+ regexOnFullUrl: false,excludes: [],includes: [],- allow_backward_crawling: true,- ignore_robots_txt: false,- robots_txt: problematicContent,+ allowBackwardCrawling: true,+ ignoreRobotsTxt: false,+ robotsTxt: problematicContent,});expect(result.links).toHaveLength(1);diff --git a/apps/api/src/lib/__tests__/html-transformer.test.ts b/apps/api/src/lib/__tests__/html-transformer.test.tsindex e3ddd76d7..8e93e3f82 100644--- a/apps/api/src/lib/__tests__/html-transformer.test.ts+++ b/apps/api/src/lib/__tests__/html-transformer.test.ts@@ -2,7 +2,8 @@ import {extractLinks,extractMetadata,transformHtml,-} from "../html-transformer";+ TransformHtmlOptions,+} from "@mendable/firecrawl-rs";describe("HTML Transformer", () => {describe("extractLinks", () => {@@ -190,9 +191,9 @@ describe("HTML Transformer", () => {});describe("extractMetadata", () => {- it("should return empty array for null or undefined input", async () => {- expect(await extractMetadata(null)).toEqual([]);- expect(await extractMetadata(undefined)).toEqual([]);+ it("should return empty object for null or undefined input", async () => {+ expect(await extractMetadata(null)).toEqual({});+ expect(await extractMetadata(undefined)).toEqual({});});it("should extract comprehensive metadata from HTML content", async () => {@@ -264,12 +265,12 @@ describe("HTML Transformer", () => {describe("transformHtml", () => {it("should transform HTML content according to options", async () => {- const options = {+ const options: TransformHtmlOptions = {html: "<div><p>Test</p><span>Remove me</span></div>",url: "https://example.com",- include_tags: ["p"],- exclude_tags: ["span"],- only_main_content: true,+ includeTags: ["p"],+ excludeTags: ["span"],+ onlyMainContent: true,};const result = await transformHtml(options);@@ -278,7 +279,7 @@ describe("HTML Transformer", () => {});it("should handle complex content filtering", async () => {- const options = {+ const options: TransformHtmlOptions = {html: `<div class="wrapper"><header>@@ -297,9 +298,9 @@ describe("HTML Transformer", () => {</div>`,url: "https://example.com",- include_tags: ["article", "h1", "p"],- exclude_tags: ["nav", "aside", "footer", ".ads", ".social-share"],- only_main_content: true,+ includeTags: ["article", "h1", "p"],+ excludeTags: ["nav", "aside", "footer", ".ads", ".social-share"],+ onlyMainContent: true,};const result = await transformHtml(options);@@ -312,7 +313,7 @@ describe("HTML Transformer", () => {});it("should handle nested content preservation and absolute links", async () => {- const options = {+ const options: TransformHtmlOptions = {html: `<article><div class="content">@@ -326,9 +327,9 @@ describe("HTML Transformer", () => {</article>`,url: "https://example.com",- include_tags: ["article", "p", "ul", "li"],- exclude_tags: [],- only_main_content: true,+ includeTags: ["article", "p", "ul", "li"],+ excludeTags: [],+ onlyMainContent: true,};const result = await transformHtml(options);@@ -338,12 +339,12 @@ describe("HTML Transformer", () => {});it("should handle empty HTML content", async () => {- const options = {+ const options: TransformHtmlOptions = {html: "",url: "https://example.com",- include_tags: [],- exclude_tags: [],- only_main_content: false,+ includeTags: [],+ excludeTags: [],+ onlyMainContent: false,};const result = await transformHtml(options);@@ -351,12 +352,12 @@ describe("HTML Transformer", () => {});it("should handle malformed HTML", async () => {- const options = {+ const options: TransformHtmlOptions = {html: "<div>Unclosed div",url: "https://example.com",- include_tags: [],- exclude_tags: [],- only_main_content: false,+ includeTags: [],+ excludeTags: [],+ onlyMainContent: false,};const result = await transformHtml(options);@@ -364,7 +365,7 @@ describe("HTML Transformer", () => {});it("should handle HTML with comments and scripts", async () => {- const options = {+ const options: TransformHtmlOptions = {html: `<div><!-- Comment -->@@ -375,9 +376,9 @@ describe("HTML Transformer", () => {</div>`,url: "https://example.com",- include_tags: ["p"],- exclude_tags: ["script", "style", "noscript"],- only_main_content: true,+ includeTags: ["p"],+ excludeTags: ["script", "style", "noscript"],+ onlyMainContent: true,};const result = await transformHtml(options);@@ -388,7 +389,7 @@ describe("HTML Transformer", () => {});it("should handle special characters and encoding", async () => {- const options = {+ const options: TransformHtmlOptions = {html: `<div><p>© 2024</p>@@ -398,9 +399,9 @@ describe("HTML Transformer", () => {</div>`,url: "https://example.com",- include_tags: ["p"],- exclude_tags: [],- only_main_content: true,+ includeTags: ["p"],+ excludeTags: [],+ onlyMainContent: true,};const result = await transformHtml(options);@@ -410,7 +411,7 @@ describe("HTML Transformer", () => {});it("should make all URLs absolute", async () => {- const options = {+ const options: TransformHtmlOptions = {html: `<div><a href="https://example.com/fullurl">hi</a>@@ -424,9 +425,9 @@ describe("HTML Transformer", () => {</div>`,url: "https://example.com",- include_tags: [],- exclude_tags: [],- only_main_content: true,+ includeTags: [],+ excludeTags: [],+ onlyMainContent: true,};const result = await transformHtml(options);diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.tsdeleted file mode 100644index cf4c311d5..000000000--- a/apps/api/src/lib/crawler.ts+++ /dev/null@@ -1,164 +0,0 @@-import koffi, { KoffiFunction } from "koffi";-import { stat } from "fs/promises";-import type { DenialReason } from "../scraper/WebScraper/crawler";-import { CRAWLER_PATH } from "../natives";--// TODO: add a timeout to the Rust transformer--export type FilterLinksCall = {- links: string[];- limit: number | undefined;- max_depth: number;- base_url: string;- initial_url: string;- regex_on_full_url: boolean;- excludes: string[];- includes: string[];- allow_backward_crawling: boolean;- ignore_robots_txt: boolean;- robots_txt: string;-};--export type FilterLinksResult = {- links: string[];- denial_reasons: Map<string, keyof typeof DenialReason>;-};--class RustCrawler {- private static instance: RustCrawler;- private _freeString: KoffiFunction;- private _filterLinks: KoffiFunction;- private _parseSitemapXml: KoffiFunction;- private _processSitemap: KoffiFunction;-- private constructor() {- const lib = koffi.load(CRAWLER_PATH);- this._freeString = lib.func("free_string", "void", ["string"]);- const cstn = "CString:" + crypto.randomUUID();- const freedResultString = koffi.disposable(- cstn,- "string",- this._freeString,- );- this._filterLinks = lib.func("filter_links", freedResultString, ["string"]);- this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, [- "string",- ]);- this._processSitemap = lib.func("process_sitemap", freedResultString, [- "string",- ]);- }-- public static async getInstance(): Promise<RustCrawler> {- if (!RustCrawler.instance) {- try {- await stat(CRAWLER_PATH);- } catch (_) {- throw Error("Rust crawler shared library not found");- }- RustCrawler.instance = new RustCrawler();- }- return RustCrawler.instance;- }-- public async filterLinks(call: FilterLinksCall): Promise<FilterLinksResult> {- return new Promise<FilterLinksResult>((resolve, reject) => {- this._filterLinks.async(- JSON.stringify(call),- (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- return reject(new Error(res.split("RUSTFC:ERROR:")[1]));- }-- if (res === "RUSTFC:ERROR") {- return reject(- new Error("Something went wrong on the Rust side."),- );- }-- try {- const raw = JSON.parse(res);- const result: FilterLinksResult = {- links: raw.links,- denial_reasons: new Map(Object.entries(raw.denial_reasons)),- };- resolve(result);- } catch (e) {- reject(e);- }- }- },- );- });- }-- public async parseSitemapXml(xmlContent: string): Promise<any> {- return new Promise<any>((resolve, reject) => {- this._parseSitemapXml.async(xmlContent, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- return reject(new Error(res.split("RUSTFC:ERROR:")[1]));- }-- if (res === "RUSTFC:ERROR") {- return reject(new Error("Something went wrong on the Rust side."));- }-- try {- const result = JSON.parse(res);- resolve(result);- } catch (e) {- reject(e);- }- }- });- });- }-- public async processSitemap(xmlContent: string): Promise<any> {- return new Promise<any>((resolve, reject) => {- this._processSitemap.async(xmlContent, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- return reject(new Error(res.split("RUSTFC:ERROR:")[1]));- }-- if (res === "RUSTFC:ERROR") {- return reject(new Error("Something went wrong on the Rust side."));- }-- try {- const result = JSON.parse(res);- resolve(result);- } catch (e) {- reject(e);- }- }- });- });- }-}--export async function filterLinks(- call: FilterLinksCall,-): Promise<FilterLinksResult> {- const converter = await RustCrawler.getInstance();- return await converter.filterLinks(call);-}--export async function parseSitemapXml(xmlContent: string): Promise<any> {- const converter = await RustCrawler.getInstance();- return await converter.parseSitemapXml(xmlContent);-}--export async function processSitemap(xmlContent: string): Promise<any> {- const converter = await RustCrawler.getInstance();- return await converter.processSitemap(xmlContent);-}diff --git a/apps/api/src/lib/html-transformer.ts b/apps/api/src/lib/html-transformer.tsdeleted file mode 100644index 247837e25..000000000--- a/apps/api/src/lib/html-transformer.ts+++ /dev/null@@ -1,284 +0,0 @@-import koffi, { KoffiFunction } from "koffi";-import { stat } from "fs/promises";-import { HTML_TRANSFORMER_PATH } from "../natives";--type TransformHtmlOptions = {- html: string;- url: string;- include_tags: string[];- exclude_tags: string[];- only_main_content: boolean;- omce_signatures?: string[];-};--class RustHTMLTransformer {- private static instance: RustHTMLTransformer;- private _extractLinks: KoffiFunction;- private _extractImages: KoffiFunction;- private _extractBaseHref: KoffiFunction;- private _extractMetadata: KoffiFunction;- private _transformHtml: KoffiFunction;- private _freeString: KoffiFunction;- private _getInnerJSON: KoffiFunction;- private _extractAttributes: KoffiFunction;-- private constructor() {- const lib = koffi.load(HTML_TRANSFORMER_PATH);- this._freeString = lib.func("free_string", "void", ["string"]);- const cstn = "CString:" + crypto.randomUUID();- const freedResultString = koffi.disposable(- cstn,- "string",- this._freeString,- );- this._extractLinks = lib.func("extract_links", freedResultString, [- "string",- ]);- this._extractImages = lib.func("extract_images", freedResultString, [- "string",- "string",- ]);- this._extractBaseHref = lib.func("extract_base_href", freedResultString, [- "string",- "string",- ]);- this._extractMetadata = lib.func("extract_metadata", freedResultString, [- "string",- ]);- this._transformHtml = lib.func("transform_html", freedResultString, [- "string",- ]);- this._getInnerJSON = lib.func("get_inner_json", freedResultString, [- "string",- ]);- this._extractAttributes = lib.func(- "extract_attributes",- freedResultString,- ["string", "string"],- );- }-- public static async getInstance(): Promise<RustHTMLTransformer> {- if (!RustHTMLTransformer.instance) {- try {- await stat(HTML_TRANSFORMER_PATH);- } catch (_) {- throw Error("Rust html-transformer shared library not found");- }- RustHTMLTransformer.instance = new RustHTMLTransformer();- }- return RustHTMLTransformer.instance;- }-- public async extractLinks(html: string): Promise<string[]> {- return new Promise<string[]>((resolve, reject) => {- this._extractLinks.async(html, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- resolve(JSON.parse(res));- }- });- });- }-- public async extractImages(html: string, baseUrl: string): Promise<string[]> {- return new Promise<string[]>((resolve, reject) => {- this._extractImages.async(html, baseUrl, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- reject(new Error(res.replace("RUSTFC:ERROR:", "")));- } else {- resolve(JSON.parse(res));- }- }- });- });- }-- public async extractBaseHref(html: string, url: string): Promise<string> {- return new Promise<string>((resolve, reject) => {- this._extractBaseHref.async(html, url, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- resolve(res);- }- });- });- }-- public async extractMetadata(html: string): Promise<any> {- return new Promise<string[]>((resolve, reject) => {- this._extractMetadata.async(html, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- resolve(JSON.parse(res));- }- });- });- }-- public async transformHtml(opts: TransformHtmlOptions): Promise<string> {- return new Promise<string>((resolve, reject) => {- this._transformHtml.async(- JSON.stringify(opts),- (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res === "RUSTFC:ERROR" || res.startsWith("RUSTFC:ERROR:")) {- reject(- new Error(- res.startsWith("RUSTFC:ERROR:")- ? "Something went wrong on the Rust side. " +- res.split("RUSTFC:ERROR:")[1]- : "Something went wrong on the Rust side.",- ),- );- } else {- resolve(res);- }- }- },- );- });- }-- public async getInnerJSON(html: string): Promise<string> {- return new Promise<string>((resolve, reject) => {- this._getInnerJSON.async(html, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res === "RUSTFC:ERROR") {- reject(new Error("Something went wrong on the Rust side."));- } else {- resolve(res);- }- }- });- });- }-- public async extractAttributes(- html: string,- options: string,- ): Promise<string> {- return new Promise<string>((resolve, reject) => {- this._extractAttributes.async(- html,- options,- (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- reject(- new Error(- "Rust attribute extraction failed: " +- res.split("RUSTFC:ERROR:")[1],- ),- );- } else {- resolve(res);- }- }- },- );- });- }-}--export async function extractLinks(- html: string | null | undefined,-): Promise<string[]> {- if (!html) {- return [];- }-- const converter = await RustHTMLTransformer.getInstance();- return await converter.extractLinks(html);-}--export async function extractImages(- html: string | null | undefined,- baseUrl: string = "",-): Promise<string[]> {- if (!html) {- return [];- }-- const converter = await RustHTMLTransformer.getInstance();- return await converter.extractImages(html, baseUrl);-}--export async function extractBaseHref(- html: string | null | undefined,- url: string,-): Promise<string> {- if (!html) {- return url;- }-- const converter = await RustHTMLTransformer.getInstance();- return await converter.extractBaseHref(html, url);-}--export async function extractMetadata(- html: string | null | undefined,-): Promise<any> {- if (!html) {- return [];- }-- const converter = await RustHTMLTransformer.getInstance();- return await converter.extractMetadata(html);-}--export async function transformHtml(- opts: TransformHtmlOptions,-): Promise<string> {- const converter = await RustHTMLTransformer.getInstance();- return await converter.transformHtml(opts);-}--export async function getInnerJSON(html: string): Promise<string> {- const converter = await RustHTMLTransformer.getInstance();- return await converter.getInnerJSON(html);-}--export type AttributeSelector = {- selector: string;- attribute: string;-};--export type AttributeResult = {- selector: string;- attribute: string;- values: string[];-};--export async function extractAttributesRust(- html: string,- selectors: AttributeSelector[],-): Promise<AttributeResult[]> {- if (!html || selectors.length === 0) {- return [];- }-- const converter = await RustHTMLTransformer.getInstance();- const options = JSON.stringify({ selectors });- const resultJson = await converter.extractAttributes(html, options);-- try {- return JSON.parse(resultJson);- } catch (error) {- throw new Error(- `Failed to parse Rust attribute extraction result: ${error}`,- );- }-}diff --git a/apps/api/src/lib/pdf-parser.ts b/apps/api/src/lib/pdf-parser.tsdeleted file mode 100644index b930b78a7..000000000--- a/apps/api/src/lib/pdf-parser.ts+++ /dev/null@@ -1,74 +0,0 @@-import koffi, { KoffiFunction } from "koffi";-import { join } from "path";-import { stat } from "fs/promises";-import { platform } from "os";-import { PDF_PARSER_PATH } from "../natives";--// TODO: add a timeout to the Rust parser--export type PDFMetadata = {- numPages: number;- title?: string;-};--class RustPDFParser {- private static instance: RustPDFParser;- private _getPDFMetadata: KoffiFunction;-- private constructor() {- const lib = koffi.load(PDF_PARSER_PATH);- this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]);- }-- public static async isParserAvailable(): Promise<boolean> {- if (RustPDFParser.instance) {- return true;- }-- try {- await stat(PDF_PARSER_PATH);- RustPDFParser.instance = new RustPDFParser();- return true;- } catch (_) {- return false;- }- }-- public static async getInstance(): Promise<RustPDFParser> {- if (!RustPDFParser.instance) {- try {- await stat(PDF_PARSER_PATH);- } catch (_) {- throw Error("Rust pdf-parser shared library not found");- }- RustPDFParser.instance = new RustPDFParser();- }- return RustPDFParser.instance;- }-- public async getPDFMetadata(path: string): Promise<PDFMetadata> {- return new Promise<PDFMetadata>((resolve, reject) => {- this._getPDFMetadata.async(path, (err: Error, res: string) => {- if (err) {- reject(err);- } else {- if (res.startsWith("RUSTFC:ERROR:")) {- reject(new Error(res.replace("RUSTFC:ERROR:", "")));- } else {- try {- const metadata = JSON.parse(res) as PDFMetadata;- resolve(metadata);- } catch (e) {- reject(new Error("Failed to parse PDF metadata."));- }- }- }- });- });- }-}--export async function getPDFMetadata(path: string): Promise<PDFMetadata> {- const converter = await RustPDFParser.getInstance();- return await converter.getPDFMetadata(path);-}diff --git a/apps/api/src/natives.ts b/apps/api/src/natives.tsindex 560a14cb7..c12e3b8af 100644--- a/apps/api/src/natives.ts+++ b/apps/api/src/natives.ts@@ -18,21 +18,6 @@ function createNativePath(subPath: string, filename: string): string {return join(process.cwd(), "sharedLibs", subPath, fullFilename);}-export const HTML_TRANSFORMER_PATH = createNativePath(- "html-transformer/target/release",- "html_transformer",-);--export const PDF_PARSER_PATH = createNativePath(- "pdf-parser/target/release",- "pdf_parser",-);--export const CRAWLER_PATH = createNativePath(- "crawler/target/release",- "crawler",-);-export const HTML_TO_MARKDOWN_PATH = createNativePath("go-html-to-md","html-to-markdown",diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 0f060e5ca..be91be585 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -7,8 +7,7 @@ import psl from "psl";import { getURLDepth } from "./utils/maxDepthUtils";import { logger as _logger } from "../../lib/logger";import { redisEvictConnection } from "../../services/redis";-import { extractLinks } from "../../lib/html-transformer";-import { filterLinks } from "../../lib/crawler";+import { extractLinks } from "@mendable/firecrawl-rs";import {fetchRobotsTxt,createRobotsChecker,@@ -17,6 +16,7 @@ import {import { ScrapeJobTimeoutError } from "../../lib/error";import { Logger } from "winston";import { ScrapeOptions } from "../../controllers/v2/types";+import { filterLinks } from "@mendable/firecrawl-rs";export const SITEMAP_LIMIT = 100;@@ -167,19 +167,19 @@ export class WebCrawler {const res = await filterLinks({links: sitemapLinks,limit: isFinite(limit) ? limit : undefined,- max_depth: maxDepth,- base_url: this.baseUrl,- initial_url: this.initialUrl,- regex_on_full_url: this.regexOnFullURL,+ maxDepth: maxDepth,+ baseUrl: this.baseUrl,+ initialUrl: this.initialUrl,+ regexOnFullUrl: this.regexOnFullURL,excludes: this.excludes,includes: this.includes,- allow_backward_crawling: this.allowBackwardCrawling,- ignore_robots_txt: this.ignoreRobotsTxt,- robots_txt: this.robotsTxt,+ allowBackwardCrawling: this.allowBackwardCrawling,+ ignoreRobotsTxt: this.ignoreRobotsTxt,+ robotsTxt: this.robotsTxt,});const fancyDenialReasons = new Map<string, string>();- res.denial_reasons.forEach((value, key) => {+ Object.entries(res.denialReasons).forEach(([key, value]) => {fancyDenialReasons.set(key, DenialReason[value]);});diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 4616df85b..27a02a609 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -4,10 +4,10 @@ import { scrapeURL } from "../scrapeURL";import { scrapeOptions } from "../../controllers/v2/types";import type { Logger } from "winston";import { CostTracking } from "../../lib/cost-tracking";-import { parseSitemapXml, processSitemap } from "../../lib/crawler";import { ScrapeJobTimeoutError } from "../../lib/error";import type { ScrapeOptions } from "../../controllers/v2/types";import { Engine } from "../scrapeURL/engines";+import { parseSitemapXml, processSitemap } from "@mendable/firecrawl-rs";const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&process.env.FIRE_ENGINE_BETA_URL !== undefined;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 899ee2c6a..b764a5b50 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -27,7 +27,7 @@ import * as Sentry from "@sentry/node";import { specialtyScrapeCheck } from "../utils/specialtyHandler";import { fireEngineDelete } from "./delete";import { MockState } from "../../lib/mock";-import { getInnerJSON } from "../../../../lib/html-transformer";+import { getInnerJson } from "@mendable/firecrawl-rs";import { hasFormatOfType } from "../../../../lib/format-utils";import { Action } from "../../../../controllers/v1/types";import { AbortManagerThrownError } from "../../lib/abortManager";@@ -161,7 +161,7 @@ async function performFireEngineScrape<) ?? [])[1] ?? "";if (contentType.includes("application/json")) {- status.content = await getInnerJSON(status.content);+ status.content = await getInnerJson(status.content);}if (status.file) {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 6c7d4c9d3..24028270d 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -16,8 +16,8 @@ import {import { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";import { useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";-import { PDFMetadata } from "../../../lib/pdf-parser";import { getPDFMaxPages } from "../../../controllers/v2/types";+import { PdfMetadata } from "@mendable/firecrawl-rs";export type Engine =| "fire-engine;chrome-cdp"@@ -119,7 +119,7 @@ export type EngineScrapeResult = {pdfs: string[];};- pdfMetadata?: PDFMetadata;+ pdfMetadata?: PdfMetadata;cacheInfo?: {created_at: Date;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 4399ba3c2..c34b5d6fa 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -16,7 +16,6 @@ import {import { readFile, unlink } from "node:fs/promises";import path from "node:path";import type { Response } from "undici";-import { getPDFMetadata } from "../../../../lib/pdf-parser";import {getPdfResultFromCache,savePdfResultToCache,@@ -26,6 +25,7 @@ import {shouldParsePDF,getPDFMaxPages,} from "../../../../controllers/v2/types";+import { getPdfMetadata } from "@mendable/firecrawl-rs";type PDFProcessorResult = { html: string; markdown?: string };@@ -240,7 +240,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}}- const pdfMetadata = await getPDFMetadata(tempFilePath);+ const pdfMetadata = await getPdfMetadata(tempFilePath);const effectivePageCount = maxPages? Math.min(pdfMetadata.numPages, maxPages): pdfMetadata.numPages;diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.tsindex 92187307b..ae2bfec97 100644--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts@@ -2,7 +2,7 @@ import { z } from "zod";import { EngineScrapeResult } from "..";import { Meta } from "../..";import { robustFetch } from "../../lib/fetch";-import { getInnerJSON } from "../../../../lib/html-transformer";+import { getInnerJson } from "@mendable/firecrawl-rs";export async function scrapeURLWithPlaywright(meta: Meta,@@ -32,7 +32,7 @@ export async function scrapeURLWithPlaywright(});if (response.contentType?.includes("application/json")) {- response.content = await getInnerJSON(response.content);+ response.content = await getInnerJson(response.content);}return {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 688708a06..2be357363 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -64,7 +64,6 @@ import {AbortManagerThrownError,} from "./lib/abortManager";import { ScrapeJobTimeoutError } from "../../lib/error";-import { transformHtml } from "../../lib/html-transformer";import { htmlTransform } from "./lib/removeUnwantedElements";export type ScrapeUrlResponse =diff --git a/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts b/apps/api/src/scraper/scrapeURL/lib/extractAttributes.tsindex 51fbe00ec..3b6d75e89 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractAttributes.ts@@ -1,6 +1,6 @@import { load } from "cheerio";import { logger } from "../../../lib/logger";-import { extractAttributesRust } from "../../../lib/html-transformer";+import { extractAttributes as _extractAttributes } from "@mendable/firecrawl-rs";export type AttributeResult = {selector: string;@@ -29,7 +29,7 @@ export async function extractAttributes(// Try Rust implementation first (faster, non-blocking)try {- const results = await extractAttributesRust(html, selectors);+ const results = await _extractAttributes(html, { selectors });logger.debug("Attribute extraction via Rust", {selectorsCount: selectors.length,diff --git a/apps/api/src/scraper/scrapeURL/lib/extractImages.ts b/apps/api/src/scraper/scrapeURL/lib/extractImages.tsindex df87bfd28..494a989f7 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractImages.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractImages.ts@@ -1,6 +1,6 @@import { load } from "cheerio";import { logger } from "../../../lib/logger";-import { extractImages as _extractImages } from "../../../lib/html-transformer";+import { extractImages as _extractImages } from "@mendable/firecrawl-rs";function resolveImageUrl(src: string,diff --git a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts b/apps/api/src/scraper/scrapeURL/lib/extractLinks.tsindex e55731840..7aecd12de 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractLinks.ts@@ -4,7 +4,7 @@ import { logger } from "../../../lib/logger";import {extractLinks as _extractLinks,extractBaseHref as _extractBaseHref,-} from "../../../lib/html-transformer";+} from "@mendable/firecrawl-rs";function resolveUrlWithBaseHref(href: string,diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.tsindex 515af3c76..9a891122e 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts@@ -1,7 +1,7 @@import { load } from "cheerio"; // rustifiedimport { Document } from "../../../controllers/v1/types";import { Meta } from "..";-import { extractMetadata as _extractMetadata } from "../../../lib/html-transformer";+import { extractMetadata as _extractMetadata } from "@mendable/firecrawl-rs";export async function extractMetadataRust(meta: Meta,diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex d966228ea..68dcfc6cd 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -2,7 +2,7 @@import { AnyNode, Cheerio, load } from "cheerio"; // rustifiedimport { ScrapeOptions } from "../../../controllers/v2/types";-import { transformHtml } from "../../../lib/html-transformer";+import { transformHtml } from "@mendable/firecrawl-rs";import { logger } from "../../../lib/logger";import { queryOMCESignatures } from "../../../services/index";@@ -91,14 +91,14 @@ export const htmlTransform = async (return await transformHtml({html,url,- include_tags: (scrapeOptions.includeTags ?? [])+ includeTags: (scrapeOptions.includeTags ?? []).map(x => x.trim()).filter(x => x.length !== 0),- exclude_tags: (scrapeOptions.excludeTags ?? [])+ excludeTags: (scrapeOptions.excludeTags ?? []).map(x => x.trim()).filter(x => x.length !== 0),- only_main_content: scrapeOptions.onlyMainContent,- omce_signatures: omce_signatures,+ onlyMainContent: scrapeOptions.onlyMainContent,+ omceSignatures: omce_signatures,});} catch (error) {logger.warn("Failed to call html-transformer! Falling back to cheerio...", {diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 9e49d6558..ddaf50392 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -7,7 +7,7 @@ import { redisEvictConnection } from "./redis";import type { Logger } from "winston";import psl from "psl";import { MapDocument } from "../controllers/v2/types";-import { PDFMetadata } from "../lib/pdf-parser";+import { PdfMetadata } from "@mendable/firecrawl-rs";import { storage } from "../lib/gcs-jobs";configDotenv();@@ -148,7 +148,7 @@ export async function saveIndexToGCS(statusCode: number;error?: string;screenshot?: string;- pdfMetadata?: PDFMetadata;+ pdfMetadata?: PdfMetadata;contentType?: string;},): Promise<void> {