Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL): fix HTML transforming logic (#1999)
Gergő Móricza4a2fcf5f669
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 86203afe0..7917a4291 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -10,7 +10,7 @@ import {WebSearchResult,} from "../../lib/entities";import { agentOptionsExtract, ScrapeOptions as V1ScrapeOptions } from "../v1/types";-import { InternalOptions } from "../../scraper/scrapeURL";+import type { InternalOptions } from "../../scraper/scrapeURL";import { ErrorCodes } from "../../lib/error";import Ajv from "ajv";diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 4e7f35209..86cd7bee1 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1,7 +1,7 @@import { Logger } from "winston";import * as Sentry from "@sentry/node";-import { Document, ScrapeOptions, TeamFlags } from "../../controllers/v2/types";+import { type Document, scrapeOptions, type ScrapeOptions, type TeamFlags } from "../../controllers/v2/types";import { ScrapeOptions as ScrapeOptionsV1 } from "../../controllers/v1/types";import { logger as _logger } from "../../lib/logger";import {@@ -44,6 +44,7 @@ import { checkRobotsTxt } from "../../lib/robots-txt";import { AbortInstance, AbortManager, AbortManagerThrownError } from "./lib/abortManager";import { ScrapeJobTimeoutError } from "../../lib/error";import { transformHtml } from "../../lib/html-transformer";+import { htmlTransform } from "./lib/removeUnwantedElements";export type ScrapeUrlResponse = (| {@@ -257,22 +258,10 @@ async function scrapeURLLoopIter(meta: Meta, engine: Engine, snipeAbort): Promisabort: meta.abort.child(snipeAbort),}, engine);- let checkMarkdown = await parseMarkdown(await transformHtml({- html: engineResult.html,- url: meta.url,- include_tags: [],- exclude_tags: [],- only_main_content: true,- }));+ let checkMarkdown = await parseMarkdown(await htmlTransform(engineResult.html, meta.url, scrapeOptions.parse({ onlyMainContent: true })));if (checkMarkdown.trim().length === 0) {- checkMarkdown = await parseMarkdown(await transformHtml({- html: engineResult.html,- url: meta.url,- include_tags: [],- exclude_tags: [],- only_main_content: false,- }));+ checkMarkdown = await parseMarkdown(await htmlTransform(engineResult.html, meta.url, scrapeOptions.parse({ onlyMainContent: false })));}// Success factors