Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL): fix HTML transforming logic (#1999)

Gergő Móricza4a2fcf5f669
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 86203afe0..7917a4291 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -10,7 +10,7 @@ import {
WebSearchResult,
} from "../../lib/entities";
import { agentOptionsExtract, ScrapeOptions as V1ScrapeOptions } from "../v1/types";
-import { InternalOptions } from "../../scraper/scrapeURL";
+import type { InternalOptions } from "../../scraper/scrapeURL";
import { ErrorCodes } from "../../lib/error";
import Ajv from "ajv";
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 4e7f35209..86cd7bee1 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1,7 +1,7 @@
import { Logger } from "winston";
import * as Sentry from "@sentry/node";
-import { Document, ScrapeOptions, TeamFlags } from "../../controllers/v2/types";
+import { type Document, scrapeOptions, type ScrapeOptions, type TeamFlags } from "../../controllers/v2/types";
import { ScrapeOptions as ScrapeOptionsV1 } from "../../controllers/v1/types";
import { logger as _logger } from "../../lib/logger";
import {
@@ -44,6 +44,7 @@ import { checkRobotsTxt } from "../../lib/robots-txt";
import { AbortInstance, AbortManager, AbortManagerThrownError } from "./lib/abortManager";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { transformHtml } from "../../lib/html-transformer";
+import { htmlTransform } from "./lib/removeUnwantedElements";
export type ScrapeUrlResponse = (
| {
@@ -257,22 +258,10 @@ async function scrapeURLLoopIter(meta: Meta, engine: Engine, snipeAbort): Promis
abort: meta.abort.child(snipeAbort),
}, engine);
- let checkMarkdown = await parseMarkdown(await transformHtml({
- html: engineResult.html,
- url: meta.url,
- include_tags: [],
- exclude_tags: [],
- only_main_content: true,
- }));
+ let checkMarkdown = await parseMarkdown(await htmlTransform(engineResult.html, meta.url, scrapeOptions.parse({ onlyMainContent: true })));
if (checkMarkdown.trim().length === 0) {
- checkMarkdown = await parseMarkdown(await transformHtml({
- html: engineResult.html,
- url: meta.url,
- include_tags: [],
- exclude_tags: [],
- only_main_content: false,
- }));
+ checkMarkdown = await parseMarkdown(await htmlTransform(engineResult.html, meta.url, scrapeOptions.parse({ onlyMainContent: false })));
}
// Success factors