Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add swoogo classes to force include main tags
Gergő Móricz2e2c3d52ced4
diff --git a/apps/api/sharedLibs/html-transformer/src/lib.rs b/apps/api/sharedLibs/html-transformer/src/lib.rsindex 8643208dc..8d944f22b 100644--- a/apps/api/sharedLibs/html-transformer/src/lib.rs+++ b/apps/api/sharedLibs/html-transformer/src/lib.rs@@ -197,8 +197,22 @@ const EXCLUDE_NON_MAIN_TAGS: [&str; 41] = ["#cookie",];-const FORCE_INCLUDE_MAIN_TAGS: [&str; 1] = [- "#main"+const FORCE_INCLUDE_MAIN_TAGS: [&str; 13] = [+ "#main",++ // swoogo event software as .widget in all of their content+ ".swoogo-cols",+ ".swoogo-text",+ ".swoogo-table-div",+ ".swoogo-space",+ ".swoogo-alert",+ ".swoogo-sponsors",+ ".swoogo-title",+ ".swoogo-tabs",+ ".swoogo-logo",+ ".swoogo-image",+ ".swoogo-button",+ ".swoogo-agenda",];#[derive(Deserialize)]diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex 693b02bba..62a0a7260 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -49,7 +49,21 @@ const excludeNonMainTags = ["#cookie",];-const forceIncludeMainTags = ["#main"];+const forceIncludeMainTags = [+ "#main",+ ".swoogo-cols",+ ".swoogo-text",+ ".swoogo-table-div",+ ".swoogo-space",+ ".swoogo-alert",+ ".swoogo-sponsors",+ ".swoogo-title",+ ".swoogo-tabs",+ ".swoogo-logo",+ ".swoogo-image",+ ".swoogo-button",+ ".swoogo-agenda"+];export const htmlTransform = async (html: string,