Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

track scrape (#3303)

Thomas Kosmas43806499aee5
diff --git a/apps/api/src/lib/search-tracking.ts b/apps/api/src/lib/tracking.ts
similarity index 64%
rename from apps/api/src/lib/search-tracking.ts
rename to apps/api/src/lib/tracking.ts
index 092cd082a..b0a2d8928 100644
--- a/apps/api/src/lib/search-tracking.ts
+++ b/apps/api/src/lib/tracking.ts
@@ -9,6 +9,47 @@ function extractDomain(url: string): string {
}
}
+// =========================================
+// Scrape tracking
+// =========================================
+
+interface TrackScrapeParams {
+ scrapeId: string;
+ requestId: string;
+ teamId: string;
+ url: string;
+ origin: string;
+ kind: string;
+ isSuccessful: boolean;
+ creditsCost: number;
+ timeTaken: number;
+ zeroDataRetention: boolean;
+}
+
+export async function trackScrape(opts: TrackScrapeParams): Promise<void> {
+ if (opts.zeroDataRetention) return;
+
+ await chInsert("scrape_results", [
+ {
+ scrape_id: opts.scrapeId,
+ request_id: opts.requestId,
+ team_id: opts.teamId,
+ url: opts.url,
+ url_domain: extractDomain(opts.url),
+ origin: opts.origin,
+ kind: opts.kind,
+ is_successful: opts.isSuccessful,
+ credits_cost: opts.creditsCost,
+ time_taken: opts.timeTaken,
+ created_at: new Date().toISOString(),
+ },
+ ]);
+}
+
+// =========================================
+// Search tracking
+// =========================================
+
interface SearchResultUrl {
url: string;
type: "web" | "news" | "image";
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index d0b4629cc..61d05e980 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -13,7 +13,7 @@ import {
mergeScrapedContent,
calculateScrapeCredits,
} from "./scrape";
-import { trackSearchResults } from "../lib/search-tracking";
+import { trackSearchResults } from "../lib/tracking";
import type { BillingMetadata } from "../services/billing/types";
interface SearchOptions {
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index e9dbcf241..6a8470c0e 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -66,6 +66,7 @@ import {
UnknownError,
} from "../../lib/error";
import { serializeTransportableError } from "../../lib/error-serde";
+import { trackScrape } from "../../lib/tracking";
import type { NuQJob } from "./nuq";
import {
ScrapeJobData,
@@ -522,6 +523,19 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
true,
);
+ trackScrape({
+ scrapeId: job.id,
+ requestId: job.data.requestId ?? job.data.crawl_id ?? job.id,
+ teamId: job.data.team_id,
+ url: job.data.url,
+ origin: job.data.origin,
+ kind: job.data.billing?.endpoint ?? "unknown",
+ isSuccessful: true,
+ creditsCost: credits_billed ?? 0,
+ timeTaken: timeTakenInSeconds,
+ zeroDataRetention: job.data.zeroDataRetention,
+ }).catch(err => logger.warn("Scrape tracking failed", { error: err }));
+
if (job.data.v1) {
const sender = await createWebhookSender({
teamId: job.data.team_id,
@@ -592,6 +606,19 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
false,
);
+ trackScrape({
+ scrapeId: job.id,
+ requestId: job.data.requestId ?? job.data.crawl_id ?? job.id,
+ teamId: job.data.team_id,
+ url: job.data.url,
+ origin: job.data.origin,
+ kind: job.data.billing?.endpoint ?? "unknown",
+ isSuccessful: true,
+ creditsCost: credits_billed ?? 0,
+ timeTaken: timeTakenInSeconds,
+ zeroDataRetention: job.data.zeroDataRetention,
+ }).catch(err => logger.warn("Scrape tracking failed", { error: err }));
+
if (job.data.skipNuq) {
// doesn't use GCS for result retrieval, safe to not await
logScrapePromise.catch(err =>
@@ -753,6 +780,20 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
},
true,
);
+
+ trackScrape({
+ scrapeId: job.id,
+ requestId: job.data.requestId ?? job.data.crawl_id ?? job.id,
+ teamId: job.data.team_id,
+ url: job.data.url,
+ origin: job.data.origin,
+ kind: job.data.billing?.endpoint ?? "unknown",
+ isSuccessful: false,
+ creditsCost: credits_billed ?? 0,
+ timeTaken: timeTakenInSeconds,
+ zeroDataRetention: job.data.zeroDataRetention,
+ }).catch(err => logger.warn("Scrape tracking failed", { error: err }));
+
return data;
} finally {
if (abortTimeoutHandle) clearTimeout(abortTimeoutHandle);