Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/fire-engine): fast fail on chrome error
Gergő Móricz42980c899d8b
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 2c67e1961..53c19f3c2 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -3,7 +3,7 @@ import * as Sentry from "@sentry/node";import { z } from "zod";import { robustFetch } from "../../lib/fetch";-import { EngineError } from "../../error";+import { EngineError, SiteError } from "../../error";const successSchema = z.object({jobId: z.string(),@@ -90,11 +90,15 @@ export async function fireEngineCheckStatus(logger: Logger, jobId: string): Promthrow new StillProcessingError(jobId);} else if (failedParse.success) {logger.debug("Scrape job failed", { status, jobId });- throw new EngineError("Scrape job failed", {- cause: {- status, jobId- }- });+ if (typeof status.error === "string" && status.error.includes("Chrome error: ")) {+ throw new SiteError(status.error.split("Chrome error: ")[1]);+ } else {+ throw new EngineError("Scrape job failed", {+ cause: {+ status, jobId+ }+ });+ }} else {logger.debug("Check status returned response not matched by any schema", { status, jobId });throw new Error("Check status returned response not matched by any schema", {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 17bb40f20..106803ac4 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -3,7 +3,7 @@ import { Meta } from "../..";import { fireEngineScrape, FireEngineScrapeRequestChromeCDP, FireEngineScrapeRequestCommon, FireEngineScrapeRequestPlaywright, FireEngineScrapeRequestTLSClient } from "./scrape";import { EngineScrapeResult } from "..";import { fireEngineCheckStatus, FireEngineCheckStatusSuccess, StillProcessingError } from "./checkStatus";-import { EngineError, TimeoutError } from "../../error";+import { EngineError, SiteError, TimeoutError } from "../../error";import * as Sentry from "@sentry/node";import { Action } from "../../../../lib/entities";import { specialtyScrapeCheck } from "../utils/specialtyHandler";@@ -41,7 +41,7 @@ async function performFireEngineScrape<Engine extends FireEngineScrapeRequestChr} catch (error) {if (error instanceof StillProcessingError) {// nop- } else if (error instanceof EngineError) {+ } else if (error instanceof EngineError || error instanceof SiteError) {logger.debug("Fire-engine scrape job failed.", { error, jobId: scrape.jobId });throw error;} else {diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex c78c812d2..3eb46033b 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -31,4 +31,12 @@ export class AddFeatureError extends Error {super("New feature flags have been discovered: " + featureFlags.join(", "));this.featureFlags = featureFlags;}-}\ No newline at end of file+}++export class SiteError extends Error {+ public code: string;+ constructor(code: string) {+ super("Specified URL is failing to load in the browser. Error code: " + code)+ this.code = code;+ }+}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 7be8b67aa..eaa84bf77 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -5,7 +5,7 @@ import { Document, ScrapeOptions } from "../../controllers/v1/types";import { logger } from "../../lib/logger";import { buildFallbackList, Engine, EngineScrapeResult, FeatureFlag, scrapeURLWithEngine } from "./engines";import { parseMarkdown } from "../../lib/html-to-markdown";-import { AddFeatureError, EngineError, NoEnginesLeftError, TimeoutError } from "./error";+import { AddFeatureError, EngineError, NoEnginesLeftError, SiteError, TimeoutError } from "./error";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";import { urlSpecificParams } from "./lib/urlSpecificParams";@@ -227,6 +227,8 @@ async function scrapeURLLoop(error.results = results;meta.logger.warn("LLM refusal encountered", { error });throw error;+ } else if (error instanceof SiteError) {+ throw error;} else {Sentry.captureException(error);meta.logger.info("An unexpected error happened while scraping with " + engine + ".", { error });@@ -306,6 +308,8 @@ export async function scrapeURL(} else if (error instanceof Error && error.message.includes("Invalid schema for response_format")) { // TODO: seperate into custom errormeta.logger.warn("scrapeURL: LLM schema error", { error });// TODO: results?+ } else if (error instanceof SiteError) {+ meta.logger.warn("scrapeURL: Site failed to load in browser", { error });} else {Sentry.captureException(error);meta.logger.error("scrapeURL: Unexpected error happened", { error });