Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fix actions errors
Nicolas1214d219e12c
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 630635769..83e899bb1 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -96,6 +96,7 @@ export async function runWebScraper({...internalOptions,});if (!response.success) {+ error = response.error;if (response.error instanceof Error) {throw response.error;} else {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 328931ba5..6f65db98e 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -3,7 +3,7 @@ import * as Sentry from "@sentry/node";import { z } from "zod";import { robustFetch } from "../../lib/fetch";-import { EngineError, SiteError } from "../../error";+import { ActionError, EngineError, SiteError } from "../../error";const successSchema = z.object({jobId: z.string(),@@ -111,6 +111,12 @@ export async function fireEngineCheckStatus(status.error.includes("Chrome error: ")) {throw new SiteError(status.error.split("Chrome error: ")[1]);+ } else if (+ typeof status.error === "string" &&+ // TODO: improve this later+ status.error.includes("Element")+ ) {+ throw new ActionError(status.error.split("Error: ")[1]);} else {throw new EngineError("Scrape job failed", {cause: {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex ef0b41fc0..a2deeed2d 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -13,7 +13,7 @@ import {FireEngineCheckStatusSuccess,StillProcessingError,} from "./checkStatus";-import { EngineError, SiteError, TimeoutError } from "../../error";+import { ActionError, EngineError, SiteError, TimeoutError } from "../../error";import * as Sentry from "@sentry/node";import { Action } from "../../../../lib/entities";import { specialtyScrapeCheck } from "../utils/specialtyHandler";@@ -68,7 +68,11 @@ async function performFireEngineScrape<} catch (error) {if (error instanceof StillProcessingError) {// nop- } else if (error instanceof EngineError || error instanceof SiteError) {+ } else if (+ error instanceof EngineError ||+ error instanceof SiteError ||+ error instanceof ActionError+ ) {logger.debug("Fire-engine scrape job failed.", {error,jobId: scrape.jobId,diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex ec0447459..0a4f6e5b6 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -56,3 +56,13 @@ export class SiteError extends Error {this.code = code;}}++export class ActionError extends Error {+ public code: string;+ constructor(code: string) {+ super(+ "Action(s) failed to complete. Error code: " + code,+ );+ this.code = code;+ }+}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex c0b6d4e52..800457a86 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -12,6 +12,7 @@ import {} from "./engines";import { parseMarkdown } from "../../lib/html-to-markdown";import {+ ActionError,AddFeatureError,EngineError,NoEnginesLeftError,@@ -288,6 +289,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {throw error;} else if (error instanceof SiteError) {throw error;+ } else if (error instanceof ActionError) {+ throw error;} else {Sentry.captureException(error);meta.logger.info(@@ -408,6 +411,8 @@ export async function scrapeURL(// TODO: results?} else if (error instanceof SiteError) {meta.logger.warn("scrapeURL: Site failed to load in browser", { error });+ } else if (error instanceof ActionError) {+ meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });} else {Sentry.captureException(error);meta.logger.error("scrapeURL: Unexpected error happened", { error });