Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
track scrape (#3303)
Thomas Kosmas43806499aee5
diff --git a/apps/api/src/lib/search-tracking.ts b/apps/api/src/lib/tracking.tssimilarity index 64%rename from apps/api/src/lib/search-tracking.tsrename to apps/api/src/lib/tracking.tsindex 092cd082a..b0a2d8928 100644--- a/apps/api/src/lib/search-tracking.ts+++ b/apps/api/src/lib/tracking.ts@@ -9,6 +9,47 @@ function extractDomain(url: string): string {}}+// =========================================+// Scrape tracking+// =========================================++interface TrackScrapeParams {+ scrapeId: string;+ requestId: string;+ teamId: string;+ url: string;+ origin: string;+ kind: string;+ isSuccessful: boolean;+ creditsCost: number;+ timeTaken: number;+ zeroDataRetention: boolean;+}++export async function trackScrape(opts: TrackScrapeParams): Promise<void> {+ if (opts.zeroDataRetention) return;++ await chInsert("scrape_results", [+ {+ scrape_id: opts.scrapeId,+ request_id: opts.requestId,+ team_id: opts.teamId,+ url: opts.url,+ url_domain: extractDomain(opts.url),+ origin: opts.origin,+ kind: opts.kind,+ is_successful: opts.isSuccessful,+ credits_cost: opts.creditsCost,+ time_taken: opts.timeTaken,+ created_at: new Date().toISOString(),+ },+ ]);+}++// =========================================+// Search tracking+// =========================================+interface SearchResultUrl {url: string;type: "web" | "news" | "image";diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex d0b4629cc..61d05e980 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -13,7 +13,7 @@ import {mergeScrapedContent,calculateScrapeCredits,} from "./scrape";-import { trackSearchResults } from "../lib/search-tracking";+import { trackSearchResults } from "../lib/tracking";import type { BillingMetadata } from "../services/billing/types";interface SearchOptions {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex e9dbcf241..6a8470c0e 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -66,6 +66,7 @@ import {UnknownError,} from "../../lib/error";import { serializeTransportableError } from "../../lib/error-serde";+import { trackScrape } from "../../lib/tracking";import type { NuQJob } from "./nuq";import {ScrapeJobData,@@ -522,6 +523,19 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {true,);+ trackScrape({+ scrapeId: job.id,+ requestId: job.data.requestId ?? job.data.crawl_id ?? job.id,+ teamId: job.data.team_id,+ url: job.data.url,+ origin: job.data.origin,+ kind: job.data.billing?.endpoint ?? "unknown",+ isSuccessful: true,+ creditsCost: credits_billed ?? 0,+ timeTaken: timeTakenInSeconds,+ zeroDataRetention: job.data.zeroDataRetention,+ }).catch(err => logger.warn("Scrape tracking failed", { error: err }));+if (job.data.v1) {const sender = await createWebhookSender({teamId: job.data.team_id,@@ -592,6 +606,19 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {false,);+ trackScrape({+ scrapeId: job.id,+ requestId: job.data.requestId ?? job.data.crawl_id ?? job.id,+ teamId: job.data.team_id,+ url: job.data.url,+ origin: job.data.origin,+ kind: job.data.billing?.endpoint ?? "unknown",+ isSuccessful: true,+ creditsCost: credits_billed ?? 0,+ timeTaken: timeTakenInSeconds,+ zeroDataRetention: job.data.zeroDataRetention,+ }).catch(err => logger.warn("Scrape tracking failed", { error: err }));+if (job.data.skipNuq) {// doesn't use GCS for result retrieval, safe to not awaitlogScrapePromise.catch(err =>@@ -753,6 +780,20 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {},true,);++ trackScrape({+ scrapeId: job.id,+ requestId: job.data.requestId ?? job.data.crawl_id ?? job.id,+ teamId: job.data.team_id,+ url: job.data.url,+ origin: job.data.origin,+ kind: job.data.billing?.endpoint ?? "unknown",+ isSuccessful: false,+ creditsCost: credits_billed ?? 0,+ timeTaken: timeTakenInSeconds,+ zeroDataRetention: job.data.zeroDataRetention,+ }).catch(err => logger.warn("Scrape tracking failed", { error: err }));+return data;} finally {if (abortTimeoutHandle) clearTimeout(abortTimeoutHandle);