Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore: improve llm extract logging (#2348)
tom7ee88590c371
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 27cb616ef..00aa07c9a 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -705,7 +705,7 @@ export async function generateCompletions({...(!schema && {onError: (error: Error) => {lastError = error;- console.error(error);+ logger.error("LLM extraction failed without schema", { error });},}),experimental_telemetry: {@@ -827,7 +827,7 @@ export async function generateCompletions({throw lastError;}} else if (NoObjectGeneratedError.isInstance(error)) {- console.log("No object generated", error);+ logger.warn("No object generated", { error });if (error.text &&error.text.startsWith("```json") &&diff --git a/apps/api/src/services/billing/auto_charge.ts b/apps/api/src/services/billing/auto_charge.tsindex a229d92de..13a4c5cc4 100644--- a/apps/api/src/services/billing/auto_charge.ts+++ b/apps/api/src/services/billing/auto_charge.ts@@ -26,7 +26,7 @@ const MAX_CHARGES_PER_HOUR = 5; // Maximum number of auto-charges per hourconst HOURLY_COUNTER_EXPIRY = 3600; // 1 hour in seconds// Type definition for auto-charge response-export type AutoChargeResponse = {+type AutoChargeResponse = {success: boolean;message: string;remainingCredits: number;@@ -72,290 +72,299 @@ async function _autoChargeScale(const resource = `auto-recharge:${chunk.team_id}`;try {- return await redlock.using([resource], 15000, async (signal: unknown): Promise<AutoChargeResponse> => {- logger.info("Lock acquired");-- const cooldownCheck = await redisEvictConnection.set(- "auto-recharge-cooldown:" + chunk.team_id,- "cooling",- "EX",- AUTO_RECHARGE_COOLDOWN,- "NX",- );-- if (cooldownCheck === null) {- logger.warn("Auto-recharge is on cooldown, aborting.");- return {- success: false,- message: "Auto-recharge is on cooldown",- remainingCredits: chunk.remaining_credits,- chunk,- };- }-- const updatedChunk = await getACUC(chunk.api_key, false, false);-- if (- updatedChunk &&- updatedChunk.remaining_credits < autoRechargeThreshold- ) {- // Check for recharges this month-- const currentMonth = new Date();- currentMonth.setUTCDate(1);- currentMonth.setUTCHours(0, 0, 0, 0);-- const { data: rechargesThisMonth, error: rechargesThisMonthError } =- await supabase_service- .from("subscriptions")- .select("*")- .eq("team_id", chunk.team_id)- .eq("metadata->>auto_recharge", "true")- .gte("current_period_start", currentMonth.toISOString());-- if (rechargesThisMonthError || !rechargesThisMonth) {- logger.error("Error fetching recharges this month", {- error: rechargesThisMonthError,- });- return {- success: false,- message: "Error fetching recharges this month",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- } else if (rechargesThisMonth.length >= 4) {- logger.warn("Auto-recharge failed: too many recharges this month");+ return await redlock.using(+ [resource],+ 15000,+ async (signal: unknown): Promise<AutoChargeResponse> => {+ logger.info("Lock acquired");++ const cooldownCheck = await redisEvictConnection.set(+ "auto-recharge-cooldown:" + chunk.team_id,+ "cooling",+ "EX",+ AUTO_RECHARGE_COOLDOWN,+ "NX",+ );++ if (cooldownCheck === null) {+ logger.warn("Auto-recharge is on cooldown, aborting.");return {success: false,- message: "Auto-recharge failed: too many recharges this month",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,+ message: "Auto-recharge is on cooldown",+ remainingCredits: chunk.remaining_credits,+ chunk,};- } else {- // Actually re-charge-- const { data: price, error: priceError } = await supabase_service- .from("prices")- .select("*")- .eq("id", chunk.price_associated_auto_recharge_price_id)- .single();- if (priceError || !price) {- logger.error("Error fetching price", {- error: priceError,- priceId:- chunk.price_associated_auto_recharge_price_id === undefined- ? "undefined"- : JSON.stringify(- chunk.price_associated_auto_recharge_price_id,- ),+ }++ const updatedChunk = await getACUC(chunk.api_key, false, false);++ if (+ updatedChunk &&+ updatedChunk.remaining_credits < autoRechargeThreshold+ ) {+ // Check for recharges this month++ const currentMonth = new Date();+ currentMonth.setUTCDate(1);+ currentMonth.setUTCHours(0, 0, 0, 0);++ const { data: rechargesThisMonth, error: rechargesThisMonthError } =+ await supabase_service+ .from("subscriptions")+ .select("*")+ .eq("team_id", chunk.team_id)+ .eq("metadata->>auto_recharge", "true")+ .gte("current_period_start", currentMonth.toISOString());++ if (rechargesThisMonthError || !rechargesThisMonth) {+ logger.error("Error fetching recharges this month", {+ error: rechargesThisMonthError,});return {success: false,- message: "Error fetching price",+ message: "Error fetching recharges this month",remainingCredits:updatedChunk?.remaining_credits ?? chunk.remaining_credits,chunk: updatedChunk ?? chunk,};- }-- if (!chunk.sub_user_id) {- logger.error("No sub_user_id found in chunk");+ } else if (rechargesThisMonth.length >= 4) {+ logger.warn("Auto-recharge failed: too many recharges this month");return {success: false,- message: "No sub_user_id found in chunk",+ message: "Auto-recharge failed: too many recharges this month",remainingCredits:updatedChunk?.remaining_credits ?? chunk.remaining_credits,chunk: updatedChunk ?? chunk,};- }+ } else {+ // Actually re-charge- const { data: customer, error: customersError } =- await supabase_rr_service- .from("customers")- .select("id, stripe_customer_id")- .eq("id", chunk.sub_user_id)+ const { data: price, error: priceError } = await supabase_service+ .from("prices")+ .select("*")+ .eq("id", chunk.price_associated_auto_recharge_price_id).single();+ if (priceError || !price) {+ logger.error("Error fetching price", {+ error: priceError,+ priceId:+ chunk.price_associated_auto_recharge_price_id === undefined+ ? "undefined"+ : JSON.stringify(+ chunk.price_associated_auto_recharge_price_id,+ ),+ });+ return {+ success: false,+ message: "Error fetching price",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }- if (customersError || !customer) {- logger.error("Error fetching customer data", {- error: customersError,- });- return {- success: false,- message: "Error fetching customer data",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }-- if (!customer.stripe_customer_id) {- logger.error("No stripe_customer_id found in customer");- return {- success: false,- message: "No stripe_customer_id found in customer",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }+ if (!chunk.sub_user_id) {+ logger.error("No sub_user_id found in chunk");+ return {+ success: false,+ message: "No sub_user_id found in chunk",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }- if (!chunk.sub_id) {- logger.error("No sub_id found in chunk");- return {- success: false,- message: "No sub_id found in chunk",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }+ const { data: customer, error: customersError } =+ await supabase_rr_service+ .from("customers")+ .select("id, stripe_customer_id")+ .eq("id", chunk.sub_user_id)+ .single();- const subscription = await createSubscription(- chunk.team_id,- customer.stripe_customer_id,- chunk.price_associated_auto_recharge_price_id,- chunk.sub_id,- );- if (!subscription) {- logger.error("Failed to create subscription");- return {- success: false,- message: "Failed to create subscription",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }+ if (customersError || !customer) {+ logger.error("Error fetching customer data", {+ error: customersError,+ });+ return {+ success: false,+ message: "Error fetching customer data",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }- const userId = await customerToUserId(customer.stripe_customer_id);- if (!userId) {- logger.error("Failed to get user id from customer");- return {- success: false,- message: "Failed to get user id from customer",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }+ if (!customer.stripe_customer_id) {+ logger.error("No stripe_customer_id found in customer");+ return {+ success: false,+ message: "No stripe_customer_id found in customer",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }- // Try to insert it into subscriptions ourselves in case webhook is slow- const { error: subscriptionError } = await supabase_service- .from("subscriptions")- .insert({- id: subscription.id,- user_id: userId,- metadata: subscription.metadata,- status: subscription.status,- price_id: chunk.price_associated_auto_recharge_price_id,- quantity: 1,- cancel_at_period_end: false,- cancel_at: null,- canceled_at: null,- current_period_start: subscription.current_period_start- ? new Date(- subscription.current_period_start * 1000,- ).toISOString()- : null,- current_period_end: subscription.current_period_end- ? new Date(subscription.current_period_end * 1000).toISOString()- : null,- created: subscription.created- ? new Date(subscription.created * 1000).toISOString()- : null,- ended_at: null,- trial_start: null,- trial_end: null,- team_id: chunk.team_id,- is_extract: false,- });+ if (!chunk.sub_id) {+ logger.error("No sub_id found in chunk");+ return {+ success: false,+ message: "No sub_id found in chunk",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }- if (subscriptionError) {- logger.warn(- "Failed to add subscription to supabase -- maybe we got sniped by the webhook?",- { error: subscriptionError },+ const subscription = await createSubscription(+ chunk.team_id,+ customer.stripe_customer_id,+ chunk.price_associated_auto_recharge_price_id,+ chunk.sub_id,);- }+ if (!subscription) {+ logger.error("Failed to create subscription");+ return {+ success: false,+ message: "Failed to create subscription",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }- // Reset ACUC cache to reflect the new credit balance- await clearACUC(chunk.api_key);- await clearACUCTeam(chunk.team_id);+ const userId = await customerToUserId(customer.stripe_customer_id);+ if (!userId) {+ logger.error("Failed to get user id from customer");+ return {+ success: false,+ message: "Failed to get user id from customer",+ remainingCredits:+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,+ };+ }- try {- // Check for frequent auto-recharges in the past week- const weeklyAutoRechargeKey = `auto-recharge-weekly:${chunk.team_id}`;- const weeklyRecharges = await redisRateLimitClient.incr(- weeklyAutoRechargeKey,- );- // Set expiry for 7 days if not already set- await redisRateLimitClient.expire(- weeklyAutoRechargeKey,- 7 * 24 * 60 * 60,+ // Try to insert it into subscriptions ourselves in case webhook is slow+ const { error: subscriptionError } = await supabase_service+ .from("subscriptions")+ .insert({+ id: subscription.id,+ user_id: userId,+ metadata: subscription.metadata,+ status: subscription.status,+ price_id: chunk.price_associated_auto_recharge_price_id,+ quantity: 1,+ cancel_at_period_end: false,+ cancel_at: null,+ canceled_at: null,+ current_period_start: subscription.current_period_start+ ? new Date(+ subscription.current_period_start * 1000,+ ).toISOString()+ : null,+ current_period_end: subscription.current_period_end+ ? new Date(+ subscription.current_period_end * 1000,+ ).toISOString()+ : null,+ created: subscription.created+ ? new Date(subscription.created * 1000).toISOString()+ : null,+ ended_at: null,+ trial_start: null,+ trial_end: null,+ team_id: chunk.team_id,+ is_extract: false,+ });++ if (subscriptionError) {+ logger.warn(+ "Failed to add subscription to supabase -- maybe we got sniped by the webhook?",+ { error: subscriptionError },+ );+ }++ // Reset ACUC cache to reflect the new credit balance+ await clearACUC(chunk.api_key);+ await clearACUCTeam(chunk.team_id);++ try {+ // Check for frequent auto-recharges in the past week+ const weeklyAutoRechargeKey = `auto-recharge-weekly:${chunk.team_id}`;+ const weeklyRecharges = await redisRateLimitClient.incr(+ weeklyAutoRechargeKey,+ );+ // Set expiry for 7 days if not already set+ await redisRateLimitClient.expire(+ weeklyAutoRechargeKey,+ 7 * 24 * 60 * 60,+ );++ // If this is the second auto-recharge in a week, send notification+ if (weeklyRecharges >= 2) {+ await sendNotificationWithCustomDays(+ chunk.team_id,+ NotificationType.AUTO_RECHARGE_FREQUENT,+ 7, // Send at most once per week+ false,+ );+ }+ } catch (error) {+ logger.error(+ `Error sending frequent auto-recharge notification`,+ {+ error,+ },+ );+ }++ await sendNotification(+ chunk.team_id,+ NotificationType.AUTO_RECHARGE_SUCCESS,+ chunk.sub_current_period_start,+ chunk.sub_current_period_end,+ chunk,+ true,);- // If this is the second auto-recharge in a week, send notification- if (weeklyRecharges >= 2) {- await sendNotificationWithCustomDays(- chunk.team_id,- NotificationType.AUTO_RECHARGE_FREQUENT,- 7, // Send at most once per week+ logger.info("Scale auto-recharge successful");++ if (process.env.SLACK_ADMIN_WEBHOOK_URL) {+ sendSlackWebhook(+ `💰 Auto-recharge successful on team ${chunk.team_id} for ${price.credits} credits (total auto-recharges this month: ${rechargesThisMonth.length + 1}).`,false,- );+ process.env.SLACK_ADMIN_WEBHOOK_URL,+ ).catch(error => {+ logger.debug(`Error sending slack notification: ${error}`);+ });}- } catch (error) {- logger.error(`Error sending frequent auto-recharge notification`, {- error,- });- }- await sendNotification(- chunk.team_id,- NotificationType.AUTO_RECHARGE_SUCCESS,- chunk.sub_current_period_start,- chunk.sub_current_period_end,- chunk,- true,- );-- logger.info("Scale auto-recharge successful");-- if (process.env.SLACK_ADMIN_WEBHOOK_URL) {- sendSlackWebhook(- `💰 Auto-recharge successful on team ${chunk.team_id} for ${price.credits} credits (total auto-recharges this month: ${rechargesThisMonth.length + 1}).`,- false,- process.env.SLACK_ADMIN_WEBHOOK_URL,- ).catch(error => {- logger.debug(`Error sending slack notification: ${error}`);- });+ return {+ success: true,+ message: "Auto-recharge successful",+ remainingCredits:+ (updatedChunk?.remaining_credits ?? chunk.remaining_credits) ++ price.credits,+ chunk: {+ ...(updatedChunk ?? chunk),+ remaining_credits:+ (updatedChunk?.remaining_credits ?? chunk.remaining_credits) ++ price.credits,+ },+ };}-+ } else {return {- success: true,- message: "Auto-recharge successful",+ success: false,+ message: "No need to auto-recharge",remainingCredits:- (updatedChunk?.remaining_credits ?? chunk.remaining_credits) +- price.credits,- chunk: {- ...(updatedChunk ?? chunk),- remaining_credits:- (updatedChunk?.remaining_credits ?? chunk.remaining_credits) +- price.credits,- },+ updatedChunk?.remaining_credits ?? chunk.remaining_credits,+ chunk: updatedChunk ?? chunk,};}- } else {- return {- success: false,- message: "No need to auto-recharge",- remainingCredits:- updatedChunk?.remaining_credits ?? chunk.remaining_credits,- chunk: updatedChunk ?? chunk,- };- }- });+ },+ );} catch (error) {logger.error("Auto-recharge failed", { error });return {