Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(smart-scrape): log failed costs

Gergő Móricz2b2e648d417a
diff --git a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
index 97e5b666d..e40ebf4c8 100644
--- a/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/smartScrape.ts
@@ -3,7 +3,7 @@ import { logger as _logger } from "../../../lib/logger";
import { robustFetch } from "./fetch";
import fs from "fs/promises";
import { configDotenv } from "dotenv";
-import { CostTracking } from "../../../lib/extract/extraction-service";
+import { CostLimitExceededError, CostTracking } from "../../../lib/extract/extraction-service";
configDotenv();
// Define schemas outside the function scope
@@ -119,12 +119,25 @@ export async function smartScrape({
errorResponse.success === false &&
errorResponse.error
) {
- if (errorResponse.error === "Cost limit exceeded") {
- throw new Error("Cost limit exceeded", {
- cause: { tokenUsage: (errorResponse as any).tokenUsage },
+ if ((errorResponse as any).tokenUsage) {
+ logger.info("Failed smart scrape cost $" + (errorResponse as any).tokenUsage);
+ costTracking.addCall({
+ type: "smartScrape",
+ cost: (errorResponse as any).tokenUsage,
+ model: "firecrawl/smart-scrape",
+ metadata: {
+ module: "smartScrape",
+ method: "smartScrape",
+ url,
+ sessionId,
+ },
});
}
+ if (errorResponse.error === "Cost limit exceeded") {
+ throw new CostLimitExceededError();
+ }
+
logger.error("Smart scrape returned error response", {
url,
prompt,
@@ -155,6 +168,36 @@ export async function smartScrape({
return response; // The response type now matches SmartScrapeResult
} catch (error) {
+ if (error instanceof CostLimitExceededError) {
+ throw error;
+ }
+
+ if (error instanceof Error && error.message === "Request sent failure status" && error.cause && (error.cause as any).response) {
+ const response = (error.cause as any).response;
+ try {
+ const json = JSON.parse(response.body);
+
+ if (json.tokenUsage) {
+ logger.info("Failed smart scrape cost $" + json.tokenUsage);
+ costTracking.addCall({
+ type: "smartScrape",
+ cost: json.tokenUsage,
+ model: "firecrawl/smart-scrape",
+ metadata: {
+ module: "smartScrape",
+ method: "smartScrape",
+ url,
+ sessionId,
+ },
+ });
+ }
+
+ if (json.error === "Cost limit exceeded") {
+ throw new CostLimitExceededError();
+ }
+ } catch (e) {}
+ }
+
// Safely extract error information without circular references
const errorInfo = {
message: error instanceof Error ? error.message : String(error),
@@ -185,7 +228,7 @@ export async function smartScrape({
};
logger.error("Smart scrape request failed", {
- error: JSON.stringify(errorInfo),
+ error: errorInfo
});
// Rethrowing the error to be handled by the caller