Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL): odd timeout on large HTML (#1994)

Gergő Móricz6af184d6f7ee
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 560160143..4e7f35209 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -39,11 +39,11 @@ import { LLMRefusalError } from "./transformers/llmExtract";
import { urlSpecificParams } from "./lib/urlSpecificParams";
import { loadMock, MockState } from "./lib/mock";
import { CostTracking } from "../../lib/extract/extraction-service";
-import { robustFetch } from "./lib/fetch";
import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";
import { checkRobotsTxt } from "../../lib/robots-txt";
import { AbortInstance, AbortManager, AbortManagerThrownError } from "./lib/abortManager";
import { ScrapeJobTimeoutError } from "../../lib/error";
+import { transformHtml } from "../../lib/html-transformer";
export type ScrapeUrlResponse = (
| {
@@ -248,35 +248,35 @@ export type InternalOptions = {
export type EngineScrapeResultWithContext = {
engine: Engine;
unsupportedFeatures: Set<FeatureFlag>;
- result: EngineScrapeResult & { markdown: string };
+ result: EngineScrapeResult;
};
-function safeguardCircularError<T>(error: T): T {
- if (typeof error === "object" && error !== null && (error as any).results) {
- const newError = structuredClone(error);
- delete (newError as any).results;
- return newError;
- } else {
- return error;
- }
-}
-
-async function scrapeURLLoopIter(meta: Meta, engine: Engine, snipeAbort): Promise<EngineScrapeResult & { markdown: string }> {
- const _engineResult = await scrapeURLWithEngine({
+async function scrapeURLLoopIter(meta: Meta, engine: Engine, snipeAbort): Promise<EngineScrapeResult> {
+ const engineResult = await scrapeURLWithEngine({
...meta,
abort: meta.abort.child(snipeAbort),
}, engine);
- if (_engineResult.markdown === undefined) {
- _engineResult.markdown = await parseMarkdown(_engineResult.html);
+ let checkMarkdown = await parseMarkdown(await transformHtml({
+ html: engineResult.html,
+ url: meta.url,
+ include_tags: [],
+ exclude_tags: [],
+ only_main_content: true,
+ }));
+
+ if (checkMarkdown.trim().length === 0) {
+ checkMarkdown = await parseMarkdown(await transformHtml({
+ html: engineResult.html,
+ url: meta.url,
+ include_tags: [],
+ exclude_tags: [],
+ only_main_content: false,
+ }));
}
- const engineResult = _engineResult as EngineScrapeResult & {
- markdown: string;
- };
-
// Success factors
- const isLongEnough = engineResult.markdown.length > 0;
+ const isLongEnough = checkMarkdown.trim().length > 0;
const isGoodStatusCode =
(engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||
engineResult.statusCode === 304;