Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(html,markdown): always get absolute links

Gergő Móricz6c94db7ed04f
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
index 2c23c2f16..92df68f47 100644
--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -49,12 +49,14 @@ const excludeNonMainTags = [
const forceIncludeMainTags = ["#main"];
-export const removeUnwantedElements = (
+export const htmlTransform = (
html: string,
+ url: string,
scrapeOptions: ScrapeOptions,
) => {
const soup = load(html);
+ // remove unwanted elements
if (
scrapeOptions.includeTags &&
scrapeOptions.includeTags.filter((x) => x.trim().length !== 0).length > 0
@@ -138,6 +140,18 @@ export const removeUnwantedElements = (
el.attribs.src = sizes[0]?.url;
});
+ // absolute links
+ soup("img[src]").each((_, el) => {
+ try {
+ el.attribs.src = new URL(el.attribs.src, url).href;
+ } catch (_) {}
+ });
+ soup("a[href]").each((_, el) => {
+ try {
+ el.attribs.href = new URL(el.attribs.href, url).href;
+ } catch (_) {}
+ });
+
const cleanedHtml = soup.html();
return cleanedHtml;
};
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index e14896eff..54bf0d46c 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -1,7 +1,7 @@
import { parseMarkdown } from "../../../lib/html-to-markdown";
import { Meta } from "..";
import { Document } from "../../../controllers/v1/types";
-import { removeUnwantedElements } from "../lib/removeUnwantedElements";
+import { htmlTransform } from "../lib/removeUnwantedElements";
import { extractLinks } from "../lib/extractLinks";
import { extractMetadata } from "../lib/extractMetadata";
import { performLLMExtract } from "./llmExtract";
@@ -41,7 +41,7 @@ export function deriveHTMLFromRawHTML(
);
}
- document.html = removeUnwantedElements(document.rawHtml, meta.options);
+ document.html = htmlTransform(document.rawHtml, document.metadata.url ?? document.metadata.sourceURL ?? meta.url, meta.options);
return document;
}