Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/fire-engine): explicitly delete job after scrape
Gergő Móricz477295131372
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex aa869836e..aeafebeab 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -17,6 +17,7 @@ import { ActionError, EngineError, SiteError, TimeoutError, UnsupportedFileErrorimport * as Sentry from "@sentry/node";import { Action } from "../../../../lib/entities";import { specialtyScrapeCheck } from "../utils/specialtyHandler";+import { fireEngineDelete } from "./delete";// This function does not take `Meta` on purpose. It may not access any// meta values to construct the request -- that must be done by the@@ -44,6 +45,13 @@ async function performFireEngineScrape<while (status === undefined) {if (errors.length >= errorLimit) {logger.error("Error limit hit.", { errors });+ fireEngineDelete(+ logger.child({+ method: "performFireEngineScrape/fireEngineDelete",+ afterErrors: errors,+ }),+ scrape.jobId,+ );throw new Error("Error limit hit. See e.cause.errors for errors.", {cause: { errors },});@@ -74,6 +82,13 @@ async function performFireEngineScrape<error instanceof ActionError ||error instanceof UnsupportedFileError) {+ fireEngineDelete(+ logger.child({+ method: "performFireEngineScrape/fireEngineDelete",+ afterError: error,+ }),+ scrape.jobId,+ );logger.debug("Fire-engine scrape job failed.", {error,jobId: scrape.jobId,@@ -105,6 +120,13 @@ async function performFireEngineScrape<status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag}+ fireEngineDelete(+ logger.child({+ method: "performFireEngineScrape/fireEngineDelete",+ }),+ scrape.jobId,+ );+return status;}