Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
search requests (#3304)
Thomas Kosmas39883e56150b
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 13a432b70..aebc8949d 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -167,6 +167,7 @@ export async function searchController(flags: req.acuc?.flags ?? null,requestId: jobId,jobId,+ apiVersion: "v1",bypassBilling: false,zeroDataRetention: false,agentIndexOnly: (req as any).agentIndexOnly ?? false,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex f7e0db0dc..74b87b2c8 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -138,6 +138,7 @@ export async function searchController(flags: req.acuc?.flags ?? null,requestId: agentRequestId ?? jobId,jobId,+ apiVersion: "v2",bypassBilling: !shouldBill,zeroDataRetention: isZDROrAnon,billing,diff --git a/apps/api/src/lib/tracking.ts b/apps/api/src/lib/tracking.tsindex b0a2d8928..e37c751a6 100644--- a/apps/api/src/lib/tracking.ts+++ b/apps/api/src/lib/tracking.ts@@ -47,7 +47,63 @@ export async function trackScrape(opts: TrackScrapeParams): Promise<void> {}// =========================================-// Search tracking+// Search request tracking+// =========================================++interface TrackSearchRequestParams {+ searchId: string;+ requestId: string;+ teamId: string;+ query: string;+ origin: string;+ kind: string;+ apiVersion: string;+ lang?: string;+ country?: string;+ sources: string[];+ numResults: number;+ searchCredits: number;+ scrapeCredits: number;+ totalCredits: number;+ hasScrapeFormats: boolean;+ scrapeFormats: string[];+ isSuccessful: boolean;+ timeTaken: number;+ zeroDataRetention: boolean;+}++export async function trackSearchRequest(+ opts: TrackSearchRequestParams,+): Promise<void> {+ if (opts.zeroDataRetention) return;++ await chInsert("search_requests", [+ {+ search_id: opts.searchId,+ request_id: opts.requestId,+ team_id: opts.teamId,+ query: opts.query,+ origin: opts.origin,+ kind: opts.kind,+ api_version: opts.apiVersion,+ lang: opts.lang ?? "",+ country: opts.country ?? "",+ sources: opts.sources,+ num_results: opts.numResults,+ search_credits: opts.searchCredits,+ scrape_credits: opts.scrapeCredits,+ total_credits: opts.totalCredits,+ has_scrape_formats: opts.hasScrapeFormats,+ scrape_formats: opts.scrapeFormats,+ is_successful: opts.isSuccessful,+ time_taken: opts.timeTaken,+ created_at: new Date().toISOString(),+ },+ ]);+}++// =========================================+// Search result tracking// =========================================interface SearchResultUrl {diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 61d05e980..3c94c904c 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -13,7 +13,7 @@ import {mergeScrapedContent,calculateScrapeCredits,} from "./scrape";-import { trackSearchResults } from "../lib/tracking";+import { trackSearchResults, trackSearchRequest } from "../lib/tracking";import type { BillingMetadata } from "../services/billing/types";interface SearchOptions {@@ -38,6 +38,7 @@ interface SearchContext {flags: TeamFlags;requestId: string;jobId: string;+ apiVersion: string;bypassBilling?: boolean;zeroDataRetention?: boolean;billing?: BillingMetadata;@@ -175,6 +176,36 @@ export async function executeSearch(}}+ const scrapeFormats = scrapeOptions?.formats+ ? scrapeOptions.formats.map((f: any) =>+ typeof f === "string" ? f : f.type,+ )+ : [];++ trackSearchRequest({+ searchId: context.jobId,+ requestId: context.requestId,+ teamId,+ query,+ origin,+ kind: billing?.endpoint ?? "search",+ apiVersion: context.apiVersion,+ lang: options.lang,+ country: options.country,+ sources: searchTypes,+ numResults: totalResultsCount,+ searchCredits,+ scrapeCredits,+ totalCredits: searchCredits + scrapeCredits,+ hasScrapeFormats: shouldScrape ?? false,+ scrapeFormats,+ isSuccessful: true,+ timeTaken: 0, // filled by caller if needed+ zeroDataRetention: zeroDataRetention ?? false,+ }).catch(err =>+ logger.warn("Search request tracking failed", { error: err }),+ );+trackSearchResults({searchId: context.jobId,teamId,