Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(html-transformer, scrapeURL): omce support (#1764)
Gergő Móricz9cc32c282178
diff --git a/apps/api/sharedLibs/html-transformer/Cargo.lock b/apps/api/sharedLibs/html-transformer/Cargo.lockindex 5e5e8b53e..65be5cdde 100644--- a/apps/api/sharedLibs/html-transformer/Cargo.lock+++ b/apps/api/sharedLibs/html-transformer/Cargo.lock@@ -14,6 +14,12 @@ version = "1.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"+[[package]]+name = "base64"+version = "0.22.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]name = "bitflags"version = "1.3.2"@@ -26,6 +32,15 @@ version = "2.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"+[[package]]+name = "block-buffer"+version = "0.10.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"+dependencies = [+ "generic-array",+]+[[package]]name = "byteorder"version = "1.5.0"@@ -44,6 +59,25 @@ version = "0.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"+[[package]]+name = "cpufeatures"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"+dependencies = [+ "libc",+]++[[package]]+name = "crypto-common"+version = "0.1.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"+dependencies = [+ "generic-array",+ "typenum",+]+[[package]]name = "cssparser"version = "0.27.2"@@ -101,6 +135,16 @@ dependencies = ["syn 2.0.96",]+[[package]]+name = "digest"+version = "0.10.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"+dependencies = [+ "block-buffer",+ "crypto-common",+]+[[package]]name = "displaydoc"version = "0.2.5"@@ -176,6 +220,16 @@ dependencies = ["byteorder",]+[[package]]+name = "generic-array"+version = "0.14.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"+dependencies = [+ "typenum",+ "version_check",+]+[[package]]name = "getrandom"version = "0.1.16"@@ -198,6 +252,18 @@ dependencies = ["wasi 0.11.0+wasi-snapshot-preview1",]+[[package]]+name = "getrandom"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"+dependencies = [+ "cfg-if",+ "libc",+ "r-efi",+ "wasi 0.14.2+wasi-0.2.4",+]+[[package]]name = "hashbrown"version = "0.12.3"@@ -215,6 +281,12 @@ dependencies = ["foldhash",]+[[package]]+name = "hex"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"+[[package]]name = "html-transformer"version = "0.1.0"@@ -222,6 +294,7 @@ dependencies = ["kuchikiki","libc","lol_html",+ "nodesig","serde","serde_json","url",@@ -504,6 +577,18 @@ version = "1.0.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"+[[package]]+name = "nodesig"+version = "1.0.0"+source = "git+https://github.com/mendableai/nodesig#b8ebc4a710810dbc36e189d9a0df87b877702b3c"+dependencies = [+ "base64",+ "hex",+ "kuchikiki",+ "rand 0.9.1",+ "sha2",+]+[[package]]name = "nodrop"version = "0.1.14"@@ -692,6 +777,12 @@ dependencies = ["proc-macro2",]+[[package]]+name = "r-efi"+version = "5.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"+[[package]]name = "rand"version = "0.7.3"@@ -717,6 +808,16 @@ dependencies = ["rand_core 0.6.4",]+[[package]]+name = "rand"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"+dependencies = [+ "rand_chacha 0.9.0",+ "rand_core 0.9.3",+]+[[package]]name = "rand_chacha"version = "0.2.2"@@ -737,6 +838,16 @@ dependencies = ["rand_core 0.6.4",]+[[package]]+name = "rand_chacha"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"+dependencies = [+ "ppv-lite86",+ "rand_core 0.9.3",+]+[[package]]name = "rand_core"version = "0.5.1"@@ -755,6 +866,15 @@ dependencies = ["getrandom 0.2.15",]+[[package]]+name = "rand_core"+version = "0.9.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"+dependencies = [+ "getrandom 0.3.3",+]+[[package]]name = "rand_hc"version = "0.2.0"@@ -899,6 +1019,17 @@ dependencies = ["stable_deref_trait",]+[[package]]+name = "sha2"+version = "0.10.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"+dependencies = [+ "cfg-if",+ "cpufeatures",+ "digest",+]+[[package]]name = "siphasher"version = "0.3.11"@@ -1023,6 +1154,12 @@ dependencies = ["zerovec",]+[[package]]+name = "typenum"+version = "1.18.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"+[[package]]name = "unicode-ident"version = "1.0.15"@@ -1058,6 +1195,12 @@ version = "1.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"+[[package]]+name = "version_check"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"+[[package]]name = "wasi"version = "0.9.0+wasi-snapshot-preview1"@@ -1070,6 +1213,15 @@ version = "0.11.0+wasi-snapshot-preview1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"+[[package]]+name = "wasi"+version = "0.14.2+wasi-0.2.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"+dependencies = [+ "wit-bindgen-rt",+]+[[package]]name = "windows-targets"version = "0.52.6"@@ -1134,6 +1286,15 @@ version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"+[[package]]+name = "wit-bindgen-rt"+version = "0.39.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"+dependencies = [+ "bitflags 2.8.0",+]+[[package]]name = "write16"version = "1.0.0"diff --git a/apps/api/sharedLibs/html-transformer/Cargo.toml b/apps/api/sharedLibs/html-transformer/Cargo.tomlindex 0cd74dc60..f12de603c 100644--- a/apps/api/sharedLibs/html-transformer/Cargo.toml+++ b/apps/api/sharedLibs/html-transformer/Cargo.toml@@ -10,6 +10,7 @@ kuchikiki = "0.8.2"serde = { version = "1.0", features = ["derive"] }serde_json = "1.0"url = "2.5.4"+nodesig = { git = "https://github.com/mendableai/nodesig" }[lib]crate-type = ["cdylib"]diff --git a/apps/api/sharedLibs/html-transformer/src/lib.rs b/apps/api/sharedLibs/html-transformer/src/lib.rsindex c7826e333..411688d0b 100644--- a/apps/api/sharedLibs/html-transformer/src/lib.rs+++ b/apps/api/sharedLibs/html-transformer/src/lib.rs@@ -1,6 +1,7 @@-use std::{collections::HashMap, ffi::{CStr, CString}};+use std::{collections::{HashMap, HashSet}, ffi::{CStr, CString}};-use kuchikiki::{parse_html, traits::TendrilSink, NodeRef};+use nodesig::{get_node_signature, SignatureMode};+use kuchikiki::{iter::NodeEdge, parse_html, traits::TendrilSink, NodeRef};use serde::Deserialize;use serde_json::Value;use url::Url;@@ -256,6 +257,7 @@ struct TranformHTMLOptions {include_tags: Vec<String>,exclude_tags: Vec<String>,only_main_content: bool,+ omce_signatures: Option<Vec<String>>,}struct ImageSource {@@ -267,7 +269,7 @@ struct ImageSource {fn _transform_html_inner(opts: TranformHTMLOptions) -> Result<String, ()> {let mut document = parse_html().one(opts.html.as_ref());let url = Url::parse(&_extract_base_href_from_document(&document).unwrap_or(opts.url)).map_err(|_| ())?;-+if !opts.include_tags.is_empty() {let new_document = parse_html().one("<div></div>");let root = new_document.select_first("div")?;@@ -302,6 +304,43 @@ fn _transform_html_inner(opts: TranformHTMLOptions) -> Result<String, ()> {x.as_node().detach();}+ // The first operation applied must be OMCE.+ if opts.only_main_content {+ if let Some(signatures) = opts.omce_signatures.as_ref() {+ let mut nodes_to_drop: Vec<NodeRef> = Vec::new();++ let modes = signatures.iter().map(|x| Into::<SignatureMode>::into(x.split(':').nth(1).unwrap().to_string())).collect::<HashSet<_>>();+ for mode in modes {+ let matcher = format!(":{}:", Into::<String>::into(mode));+ let signatures = signatures.iter().filter(|x| x.contains(&matcher)).cloned().collect::<HashSet<_>>();++ for edge in document.traverse() {+ match edge {+ NodeEdge::Start(_) => {},+ NodeEdge::End(node) => {+ if node.as_element().is_none() {+ continue;+ }++ if node.text_contents().trim().is_empty() {+ continue;+ }++ let signature = get_node_signature(&node, mode);+ if signatures.contains(&signature) {+ nodes_to_drop.push(node);+ }+ }+ }+ }+ }++ for node in nodes_to_drop {+ node.detach();+ }+ }+ }+for x in opts.exclude_tags.iter() {// TODO: implement weird versionwhile let Ok(x) = document.select_first(x) {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ec5660dde..2c7989270 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -343,6 +343,7 @@ const baseScrapeOptions = z// @deprecated__experimental_cache: z.boolean().default(false).optional(),__searchPreviewToken: z.string().optional(),+ __experimental_omce: z.boolean().default(false).optional(),}).strict(strictMessage);diff --git a/apps/api/src/lib/html-transformer.ts b/apps/api/src/lib/html-transformer.tsindex b1f58f826..70c01550e 100644--- a/apps/api/src/lib/html-transformer.ts+++ b/apps/api/src/lib/html-transformer.ts@@ -16,6 +16,7 @@ type TransformHtmlOptions = {include_tags: string[],exclude_tags: string[],only_main_content: boolean,+ omce_signatures?: string[],};class RustHTMLTransformer {diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex 62a0a7260..25dfb6ef4 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -4,6 +4,7 @@ import { AnyNode, Cheerio, load } from "cheerio"; // rustifiedimport { ScrapeOptions } from "../../../controllers/v1/types";import { transformHtml } from "../../../lib/html-transformer";import { logger } from "../../../lib/logger";+import { queryOMCESignatures } from "../../../services/index";const excludeNonMainTags = ["header",@@ -70,6 +71,21 @@ export const htmlTransform = async (url: string,scrapeOptions: ScrapeOptions,) => {+ let omce_signatures: string[] | undefined = undefined;++ if (scrapeOptions.__experimental_omce) {+ try {+ const hostname = new URL(url).hostname;+ omce_signatures = await queryOMCESignatures(hostname);+ } catch (error) {+ logger.warn("Failed to get omce signatures.", {+ error,+ scrapeURL: url,+ module: "scrapeURL", method: "htmlTransform",+ })+ };+ }+try {return await transformHtml({html,@@ -77,12 +93,13 @@ export const htmlTransform = async (include_tags: (scrapeOptions.includeTags ?? []).map(x => x.trim()).filter((x) => x.length !== 0),exclude_tags: (scrapeOptions.excludeTags ?? []).map(x => x.trim()).filter((x) => x.length !== 0),only_main_content: scrapeOptions.onlyMainContent,+ omce_signatures: omce_signatures,})} catch (error) {logger.warn("Failed to call html-transformer! Falling back to cheerio...", {- error,- module: "scrapeURL", method: "extractLinks"- });+ error,+ module: "scrapeURL", method: "htmlTransform"+ });}let soup = load(html);diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 882fed151..281b3476f 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -334,3 +334,29 @@ export async function queryIndexAtDomainSplitLevel(hostname: string, limit: numbiteration++;}}++export async function queryOMCESignatures(hostname: string, maxAge = 2 * 24 * 60 * 60 * 1000): Promise<string[]> {+ if (!useIndex || process.env.FIRECRAWL_INDEX_WRITE_ONLY === "true") {+ return [];+ }++ const domainSplitsHash = generateDomainSplits(hostname).map(x => hashURL(x));++ const level = domainSplitsHash.length - 1;+ if (domainSplitsHash.length === 0) {+ return [];+ }++ const { data, error } = await index_supabase_service+ .rpc("query_omce_signatures", {+ i_domain_hash: domainSplitsHash[level],+ i_newer_than: new Date(Date.now() - maxAge).toISOString(),+ });++ if (error) {+ _logger.warn("Error querying index (omce)", { error, hostname });+ return [];+ }++ return data?.[0]?.signatures ?? [];+}