Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(html,markdown): always get absolute links
Gergő Móricz6c94db7ed04f
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.tsindex 2c23c2f16..92df68f47 100644--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts@@ -49,12 +49,14 @@ const excludeNonMainTags = [const forceIncludeMainTags = ["#main"];-export const removeUnwantedElements = (+export const htmlTransform = (html: string,+ url: string,scrapeOptions: ScrapeOptions,) => {const soup = load(html);+ // remove unwanted elementsif (scrapeOptions.includeTags &&scrapeOptions.includeTags.filter((x) => x.trim().length !== 0).length > 0@@ -138,6 +140,18 @@ export const removeUnwantedElements = (el.attribs.src = sizes[0]?.url;});+ // absolute links+ soup("img[src]").each((_, el) => {+ try {+ el.attribs.src = new URL(el.attribs.src, url).href;+ } catch (_) {}+ });+ soup("a[href]").each((_, el) => {+ try {+ el.attribs.href = new URL(el.attribs.href, url).href;+ } catch (_) {}+ });+const cleanedHtml = soup.html();return cleanedHtml;};diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex e14896eff..54bf0d46c 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -1,7 +1,7 @@import { parseMarkdown } from "../../../lib/html-to-markdown";import { Meta } from "..";import { Document } from "../../../controllers/v1/types";-import { removeUnwantedElements } from "../lib/removeUnwantedElements";+import { htmlTransform } from "../lib/removeUnwantedElements";import { extractLinks } from "../lib/extractLinks";import { extractMetadata } from "../lib/extractMetadata";import { performLLMExtract } from "./llmExtract";@@ -41,7 +41,7 @@ export function deriveHTMLFromRawHTML();}- document.html = removeUnwantedElements(document.rawHtml, meta.options);+ document.html = htmlTransform(document.rawHtml, document.metadata.url ?? document.metadata.sourceURL ?? meta.url, meta.options);return document;}