Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: log su usage
Nicolas6567ef81f6ca
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 7149709b8..cb386554c 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -5,6 +5,7 @@ import "dotenv/config";import { logger } from "../../lib/logger";import { configDotenv } from "dotenv";import { saveJobToGCS } from "../../lib/gcs-jobs";+import { logStealthUsage } from "./temp-stealth-log";configDotenv();function cleanOfNull<T>(x: T): T {@@ -109,6 +110,10 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) {if (process.env.FIRE_INDEX_SERVER_URL) {indexJob(job);}++ if (job.scrapeOptions?.proxy === "stealth" && job.team_id) {+ logStealthUsage(job.team_id);+ }if (process.env.GCS_BUCKET_NAME) {await saveJobToGCS(job);diff --git a/apps/api/src/services/logging/temp-stealth-log.ts b/apps/api/src/services/logging/temp-stealth-log.tsnew file mode 100644index 000000000..614c6ccdc--- /dev/null+++ b/apps/api/src/services/logging/temp-stealth-log.ts@@ -0,0 +1,43 @@+import { cacheRedis } from "../../lib/cache";+import { logger } from "../../lib/logger";+import { supabase_service } from "../supabase";++export async function logStealthUsage(team_id: string) {+ try {+ let useCache = true;+ if (!cacheRedis) {+ useCache = false;+ }+ // Check Redis first+ const redisKey = `stealth_usage:${team_id}`;+ const exists = useCache ? await cacheRedis?.get(redisKey) : false;++ if (exists) {+ return;+ }++ // Check DB if not in Redis+ const { data } = await supabase_service+ .from("stealth_usage")+ .select("team_id")+ .eq("team_id", team_id);++ if (!data?.length) {+ // Insert into DB+ const { error } = await supabase_service.from("stealth_usage").insert([+ {+ team_id,+ },+ ]);++ if (error) {+ logger.error("Failed to log stealth usage", { error, team_id });+ return;+ }+ }+ // Cache in Redis for future lookups+ await cacheRedis?.set(redisKey, "1", "EX", 60 * 60 * 24 * 10); // Cache for 10 days+ } catch (err) {+ logger.error("Error logging stealth usage", { error: err, team_id });+ }+}