Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(smart-scrape): log failed costs
Gergő Móricz2b2e648d417a
diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.tsindex 97e5b666d..e40ebf4c8 100644--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts@@ -3,7 +3,7 @@ import { logger as _logger } from "../../../lib/logger";import { robustFetch } from "./fetch";import fs from "fs/promises";import { configDotenv } from "dotenv";-import { CostTracking } from "../../../lib/extract/extraction-service";+import { CostLimitExceededError, CostTracking } from "../../../lib/extract/extraction-service";configDotenv();// Define schemas outside the function scope@@ -119,12 +119,25 @@ export async function smartScrape({errorResponse.success === false &&errorResponse.error) {- if (errorResponse.error === "Cost limit exceeded") {- throw new Error("Cost limit exceeded", {- cause: { tokenUsage: (errorResponse as any).tokenUsage },+ if ((errorResponse as any).tokenUsage) {+ logger.info("Failed smart scrape cost $" + (errorResponse as any).tokenUsage);+ costTracking.addCall({+ type: "smartScrape",+ cost: (errorResponse as any).tokenUsage,+ model: "firecrawl/smart-scrape",+ metadata: {+ module: "smartScrape",+ method: "smartScrape",+ url,+ sessionId,+ },});}+ if (errorResponse.error === "Cost limit exceeded") {+ throw new CostLimitExceededError();+ }+logger.error("Smart scrape returned error response", {url,prompt,@@ -155,6 +168,36 @@ export async function smartScrape({return response; // The response type now matches SmartScrapeResult} catch (error) {+ if (error instanceof CostLimitExceededError) {+ throw error;+ }++ if (error instanceof Error && error.message === "Request sent failure status" && error.cause && (error.cause as any).response) {+ const response = (error.cause as any).response;+ try {+ const json = JSON.parse(response.body);++ if (json.tokenUsage) {+ logger.info("Failed smart scrape cost $" + json.tokenUsage);+ costTracking.addCall({+ type: "smartScrape",+ cost: json.tokenUsage,+ model: "firecrawl/smart-scrape",+ metadata: {+ module: "smartScrape",+ method: "smartScrape",+ url,+ sessionId,+ },+ });+ }++ if (json.error === "Cost limit exceeded") {+ throw new CostLimitExceededError();+ }+ } catch (e) {}+ }+// Safely extract error information without circular referencesconst errorInfo = {message: error instanceof Error ? error.message : String(error),@@ -185,7 +228,7 @@ export async function smartScrape({};logger.error("Smart scrape request failed", {- error: JSON.stringify(errorInfo),+ error: errorInfo});// Rethrowing the error to be handled by the caller