Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: improve billing logging
Móricz Gergő780442d73b2f
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 96e6ea4f4..2d8acf5f8 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -265,9 +265,10 @@ export async function scrapeController(req: Request, res: Response) {}if (creditsToBeBilled > 0) {// billing for doc done on queue end, bill only for llm extraction- billTeam(team_id, chunk?.sub_id, creditsToBeBilled).catch((error) => {+ billTeam(team_id, chunk?.sub_id, creditsToBeBilled, logger).catch((error) => {logger.error(- `Failed to bill team ${team_id} for ${creditsToBeBilled} credits: ${error}`,+ `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,+ { error });// Optionally, you could notify an admin or add to a retry queue here});@@ -312,7 +313,7 @@ export async function scrapeController(req: Request, res: Response) {return res.status(result.returnCode).json(result);} catch (error) {Sentry.captureException(error);- logger.error(error);+ logger.error("Scrape error occcurred", { error });return res.status(500).json({error:error instanceof ZodErrordiff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 5fb574d4a..c50ab9c9b 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -7,7 +7,7 @@ import {import { billTeam } from "../services/billing/credit_billing";import { Document } from "../controllers/v1/types";import { supabase_service } from "../services/supabase";-import { logger } from "../lib/logger";+import { logger as _logger } from "../lib/logger";import { ScrapeEvents } from "../lib/scrape-events";import { configDotenv } from "dotenv";import {@@ -66,6 +66,12 @@ export async function runWebScraper({is_scrape = false,is_crawl = false,}: RunWebScraperParams): Promise<ScrapeUrlResponse> {+ const logger = _logger.child({+ method: "runWebScraper",+ module: "runWebscraper",+ scrapeId: bull_job_id,+ jobId: bull_job_id,+ })const tries = is_crawl ? 3 : 1;let response: ScrapeUrlResponse | undefined = undefined;@@ -75,10 +81,6 @@ export async function runWebScraper({for (let i = 0; i < tries; i++) {if (i > 0) {logger.debug("Retrying scrape...", {- scrapeId: bull_job_id,- jobId: bull_job_id,- method: "runWebScraper",- module: "runWebScraper",tries,i,previousStatusCode: (response as any)?.document?.metadata?.statusCode,@@ -171,9 +173,10 @@ export async function runWebScraper({creditsToBeBilled = 5;}- billTeam(team_id, undefined, creditsToBeBilled).catch((error) => {+ billTeam(team_id, undefined, creditsToBeBilled, logger).catch((error) => {logger.error(- `Failed to bill team ${team_id} for ${creditsToBeBilled} credits: ${error}`,+ `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,+ { error });// Optionally, you could notify an admin or add to a retry queue here});@@ -232,6 +235,11 @@ const saveJob = async (}ScrapeEvents.logJobEvent(job, "completed");} catch (error) {- logger.error(`🐂 Failed to update job status: ${error}`);+ _logger.error(`🐂 Failed to update job status`, {+ module: "runWebScraper",+ method: "saveJob",+ jobId: job.id,+ scrapeId: job.id,+ });}};diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex bbd04cc0c..c2671034b 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -10,6 +10,7 @@ import { issueCredits } from "./issue_credits";import { redlock } from "../redlock";import { autoCharge } from "./auto_charge";import { getValue, setValue } from "../redis";+import type { Logger } from "winston";const FREE_CREDITS = 500;@@ -20,22 +21,30 @@ export async function billTeam(team_id: string,subscription_id: string | null | undefined,credits: number,+ logger?: Logger,) {return withAuth(supaBillTeam, { success: true, message: "No DB, bypassed." })(team_id,subscription_id,credits,+ logger,);}export async function supaBillTeam(team_id: string,subscription_id: string | null | undefined,credits: number,+ __logger?: Logger,) {+ const _logger = (__logger ?? logger).child({+ module: "credit_billing",+ method: "supaBillTeam",+ });+if (team_id === "preview") {return { success: true, message: "Preview team, no credits used" };}- logger.info(`Billing team ${team_id} for ${credits} credits`);+ _logger.info(`Billing team ${team_id} for ${credits} credits`, { team_id, credits });const { data, error } = await supabase_service.rpc("bill_team", {_team_id: team_id,@@ -46,7 +55,7 @@ export async function supaBillTeam(if (error) {Sentry.captureException(error);- logger.error("Failed to bill team: " + JSON.stringify(error));+ _logger.error("Failed to bill team.", { error });return;}