Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL): includeTags/excludeTags
Gergő Móricz552d55c8fcf5
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex cfed6a646..e06ee3378 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -56,7 +56,7 @@ export const removeUnwantedElements = () => {const soup = load(html);- if (scrapeOptions.includeTags && scrapeOptions.includeTags.length > 0) {+ if (scrapeOptions.includeTags && scrapeOptions.includeTags.filter(x => x.trim().length !== 0).length > 0) {// Create a new root element to hold the tags to keepconst newRoot = load("<div></div>")("div");scrapeOptions.includeTags.forEach((tag) => {@@ -69,7 +69,7 @@ export const removeUnwantedElements = (soup("script, style, noscript, meta, head").remove();- if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.length > 0) {+ if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.filter(x => x.trim().length !== 0).length > 0) {scrapeOptions.excludeTags.forEach((tag) => {let elementsToRemove: Cheerio<AnyNode>;if (tag.startsWith("*") && tag.endsWith("*")) {