Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): store keyless team ids as UUIDs in blocklist hits and analytics (#5188)
Max31e468a5c619
diff --git a/apps/api/src/lib/job-access-store.ts b/apps/api/src/lib/job-access-store.tsindex f5c033cfc..614ec72c5 100644--- a/apps/api/src/lib/job-access-store.ts+++ b/apps/api/src/lib/job-access-store.ts@@ -1,7 +1,7 @@import { config } from "../config";import { getBigtableTable } from "./bigtable-client";import { saltedUuidV7RowKey } from "./bigtable-row-key";-import { keylessTeamUuid } from "./keyless";+import { persistedTeamId } from "./keyless-team";import { setSpanAttributes, withSpan } from "./otel-tracer";/**@@ -14,15 +14,8 @@ export const JOB_ACCESS_TTL_MS = 24 * 60 * 60 * 1000;const FAMILY = "j";const QUALIFIER = "v";-const PREVIEW_TEAM_ID = "3adefd26-77ec-5968-8dcf-c94b5630d1de";-export function normalizeJobAccessTeamId(teamId: string): string {- return (- keylessTeamUuid(teamId) ??- (teamId === "preview" || teamId.startsWith("preview_")- ? PREVIEW_TEAM_ID- : teamId)- );+ return persistedTeamId(teamId);}export const API_JOB_KINDS = [diff --git a/apps/api/src/lib/keyless-team.ts b/apps/api/src/lib/keyless-team.tsnew file mode 100644index 000000000..9c162f6f1--- /dev/null+++ b/apps/api/src/lib/keyless-team.ts@@ -0,0 +1,47 @@+import { v5 as uuidv5 } from "uuid";++// Keyless team ids live here, apart from lib/keyless, so modules that only+// need them do not load its Redis and database clients.++// Keyless teams reuse the `preview_` prefix so billing (autumn `isPreviewTeam`)+// and GCS persistence are skipped automatically, with a dedicated infix so the+// IP can be recovered when charging credits after a request completes.+export const KEYLESS_TEAM_PREFIX = "preview_keyless_";++export function keylessTeamId(ip: string): string {+ return `${KEYLESS_TEAM_PREFIX}${ip}`;+}++// Fixed namespace for deriving a stable per-keyless-team UUID. Tables like+// `scrapes` require a UUID team_id, but keyless teams are `preview_keyless_<ip>`+// strings. Mapping each to a deterministic UUIDv5 keeps rows per-IP-distinct+// (so ownership checks such as interact still isolate keyless users) while+// satisfying the UUID column, unlike the shared preview placeholder.+const KEYLESS_TEAM_UUID_NAMESPACE = "9e6c8f2a-3b1d-4c7e-8a5f-2d4b6e8c0a1f";++/**+ * Deterministic UUID for a keyless team's persisted rows, or null for+ * non-keyless teams (callers then fall back to the raw/placeholder team_id).+ */+export function keylessTeamUuid(+ teamId: string | null | undefined,+): string | null {+ if (!teamId || !teamId.startsWith(KEYLESS_TEAM_PREFIX)) return null;+ return uuidv5(teamId, KEYLESS_TEAM_UUID_NAMESPACE);+}++// Shared team UUID that other preview callers are stored under.+const PREVIEW_TEAM_ID = "3adefd26-77ec-5968-8dcf-c94b5630d1de";++/**+ * The team id to store in a UUID column: a keyless caller's UUID, the shared+ * preview team for other preview callers, or the team id unchanged.+ */+export function persistedTeamId(teamId: string): string {+ return (+ keylessTeamUuid(teamId) ??+ (teamId === "preview" || teamId.startsWith("preview_")+ ? PREVIEW_TEAM_ID+ : teamId)+ );+}diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.tsindex b034b1052..da3aa9dff 100644--- a/apps/api/src/lib/keyless.ts+++ b/apps/api/src/lib/keyless.ts@@ -1,7 +1,11 @@import { createHmac } from "node:crypto";import { isIPv4 } from "node:net";-import { v5 as uuidv5 } from "uuid";import { config } from "../config";+import {+ KEYLESS_TEAM_PREFIX,+ keylessTeamId,+ keylessTeamUuid,+} from "./keyless-team";import { db } from "../db/connection";import * as schema from "../db/schema";import { redisRateLimitClient } from "../services/rate-limiter";@@ -91,14 +95,7 @@ export function keylessExhaustionTelemetry(ip: string): Record<string, string> {return conversionCohort ? { conversionCohort } : {};}-// Keyless teams reuse the `preview_` prefix so billing (autumn `isPreviewTeam`)-// and GCS persistence are skipped automatically, with a dedicated infix so the-// IP can be recovered when charging credits after a request completes.-const KEYLESS_TEAM_PREFIX = "preview_keyless_";--export function keylessTeamId(ip: string): string {- return `${KEYLESS_TEAM_PREFIX}${ip}`;-}+export { keylessTeamId, keylessTeamUuid };function keylessIpFromTeamId(teamId: string): string | null {return teamId.startsWith(KEYLESS_TEAM_PREFIX)@@ -106,24 +103,6 @@ function keylessIpFromTeamId(teamId: string): string | null {: null;}-// Fixed namespace for deriving a stable per-keyless-team UUID. Tables like-// `scrapes` require a UUID team_id, but keyless teams are `preview_keyless_<ip>`-// strings. Mapping each to a deterministic UUIDv5 keeps rows per-IP-distinct-// (so ownership checks such as interact still isolate keyless users) while-// satisfying the UUID column — unlike the shared preview placeholder.-const KEYLESS_TEAM_UUID_NAMESPACE = "9e6c8f2a-3b1d-4c7e-8a5f-2d4b6e8c0a1f";--/**- * Deterministic UUID for a keyless team's persisted rows, or null for- * non-keyless teams (callers then fall back to the raw/placeholder team_id).- */-export function keylessTeamUuid(- teamId: string | null | undefined,-): string | null {- if (!teamId || !teamId.startsWith(KEYLESS_TEAM_PREFIX)) return null;- return uuidv5(teamId, KEYLESS_TEAM_UUID_NAMESPACE);-}-/*** Keyless is allowed only for a *valid IPv4* client identity. This both:* - denies IPv6 (a single client controls a huge block — a /64 is ~18diff --git a/apps/api/src/lib/tracking.test.ts b/apps/api/src/lib/tracking.test.tsindex 4b02034c8..b9787fa75 100644--- a/apps/api/src/lib/tracking.test.ts+++ b/apps/api/src/lib/tracking.test.ts@@ -1,7 +1,11 @@import { chInsert } from "./clickhouse-client";+import { keylessTeamId, keylessTeamUuid } from "./keyless-team";import {buildMonitorTargetInterestRows,trackMonitorTargetInterest,+ trackScrape,+ trackSearchRequest,+ trackSearchResults,} from "./tracking";import type { MonitorTarget } from "../services/monitoring/types";@@ -169,3 +173,71 @@ describe("monitor target interest tracking", () => {expect(chInsert).not.toHaveBeenCalled();});});++describe("analytics team ids", () => {+ const keyless = keylessTeamId("203.0.113.7");+ const previewTeam = "3adefd26-77ec-5968-8dcf-c94b5630d1de";++ beforeEach(() => {+ vi.clearAllMocks();+ });++ // The analytics team_id columns are UUIDs, so keyless and preview callers+ // are stored under their UUIDs rather than their raw team ids.+ it.each([+ [keyless, keylessTeamUuid(keyless)],+ ["preview", previewTeam],+ [+ "5f0c2b1e-8d4a-4c3b-9e2f-1a2b3c4d5e6f",+ "5f0c2b1e-8d4a-4c3b-9e2f-1a2b3c4d5e6f",+ ],+ ])("stores team %s as %s", async (teamId, stored) => {+ await trackScrape({+ scrapeId: "scrape-1",+ requestId: "request-1",+ teamId,+ url: "https://example.com/",+ origin: "api",+ kind: "scrape",+ isSuccessful: true,+ creditsCost: 1,+ timeTaken: 1,+ zeroDataRetention: false,+ });+ await trackSearchRequest({+ searchId: "search-1",+ requestId: "request-1",+ teamId,+ query: "example",+ origin: "api",+ kind: "search",+ apiVersion: "v2",+ sources: ["web"],+ numResults: 1,+ searchCredits: 1,+ scrapeCredits: 0,+ totalCredits: 1,+ hasScrapeFormats: false,+ scrapeFormats: [],+ isSuccessful: true,+ timeTaken: 1,+ zeroDataRetention: false,+ });+ await trackSearchResults({+ searchId: "search-1",+ teamId,+ response: { web: [{ url: "https://example.com/" }] } as any,+ zeroDataRetention: false,+ hasScrapeFormats: false,+ });++ const tables = vi.mocked(chInsert).mock.calls.map(([table]) => table);+ expect(tables).toEqual([+ "scrape_results",+ "search_requests",+ "search_results",+ ]);+ for (const [, rows] of vi.mocked(chInsert).mock.calls)+ expect(rows.map(row => row.team_id)).toEqual([stored]);+ });+});diff --git a/apps/api/src/lib/tracking.ts b/apps/api/src/lib/tracking.tsindex 07b907115..5a7228f7c 100644--- a/apps/api/src/lib/tracking.ts+++ b/apps/api/src/lib/tracking.ts@@ -1,5 +1,6 @@import { createHash } from "crypto";import { chInsert } from "./clickhouse-client";+import { persistedTeamId } from "./keyless-team";import type { SearchV2Response } from "./entities";import type { MonitorTarget } from "../services/monitoring/types";@@ -54,7 +55,7 @@ export async function trackScrape(opts: TrackScrapeParams): Promise<void> {{scrape_id: opts.scrapeId,request_id: opts.requestId,- team_id: opts.teamId,+ team_id: persistedTeamId(opts.teamId),url: opts.url,url_domain: extractDomain(opts.url),origin: opts.origin,@@ -259,7 +260,7 @@ export async function trackSearchRequest({search_id: opts.searchId,request_id: opts.requestId,- team_id: opts.teamId,+ team_id: persistedTeamId(opts.teamId),query: opts.query,origin: opts.origin,kind: opts.kind,@@ -338,7 +339,7 @@ export async function trackSearchResults("search_results",urls.map(({ url, type, index }) => ({search_id: opts.searchId,- team_id: opts.teamId,+ team_id: persistedTeamId(opts.teamId),url,url_domain: extractDomain(url),result_type: type,diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist-hits.test.ts b/apps/api/src/scraper/WebScraper/utils/blocklist-hits.test.tsnew file mode 100644index 000000000..0ac827893--- /dev/null+++ b/apps/api/src/scraper/WebScraper/utils/blocklist-hits.test.ts@@ -0,0 +1,83 @@+import { keylessTeamId, keylessTeamUuid } from "../../../lib/keyless-team";+import { logger } from "../../../lib/logger";+import { initializeBlocklist, isUrlBlocked } from "./blocklist";++const insertedHits = vi.hoisted(() =>+ vi.fn(async (_hits: { team_id: string | null }[]) => {}),+);++vi.mock("../../../config", () => ({+ config: {+ USE_DB_AUTHENTICATION: true,+ DISABLE_BLOCKLIST: false,+ },+}));++vi.mock("../../../db/connection", () => ({+ db: {+ insert: vi.fn(() => ({ values: insertedHits })),+ },+ dbRr: {+ select: vi.fn(() => ({+ from: vi.fn(async () => [+ {+ org_id: null,+ data: { blocklist: ["facebook.com"], allowedKeywords: [] },+ },+ ]),+ })),+ },+}));++beforeEach(async () => {+ vi.useFakeTimers();+ insertedHits.mockClear();+ await initializeBlocklist();+});++afterEach(() => {+ vi.useRealTimers();+ vi.restoreAllMocks();+});++const recordHits = async (teamIds: (string | null)[]) => {+ for (const team_id of teamIds)+ expect(+ isUrlBlocked("https://facebook.com/page", null, {+ team_id,+ origin: "test",+ }),+ ).toBe(true);+ await vi.advanceTimersByTimeAsync(5000);+};++it("stores every hit in a batch under a UUID team id or null", async () => {+ const team = "5f0c2b1e-8d4a-4c3b-9e2f-1a2b3c4d5e6f";+ const keyless = keylessTeamId("203.0.113.7");++ await recordHits([team, keyless, "preview", "not-a-uuid", null]);++ // One batch carries every hit, so each team id must be a UUID or null.+ expect(insertedHits).toHaveBeenCalledOnce();+ expect(insertedHits.mock.calls[0][0].map(hit => hit.team_id)).toEqual([+ team,+ keylessTeamUuid(keyless),+ "3adefd26-77ec-5968-8dcf-c94b5630d1de",+ null,+ null,+ ]);+});++it("logs a failed batch with only its SQLSTATE", async () => {+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);+ insertedHits.mockRejectedValueOnce(+ Object.assign(new Error("https://facebook.com/page"), { code: "22P02" }),+ );++ await recordHits([null, null]);++ expect(warn).toHaveBeenCalledWith("Failed to record blocklist hits", {+ hits: 2,+ errorCode: "22P02",+ });+});diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex a8b7f8ed7..c0be26e70 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -1,4 +1,5 @@import { configDotenv } from "dotenv";+import { DrizzleQueryError } from "drizzle-orm/errors";import { v7 as uuidv7 } from "uuid";import { config } from "../../../config";import { parse } from "tldts";@@ -6,6 +7,9 @@ import { TeamFlags } from "../../../controllers/v1/types";import { db, dbRr } from "../../../db/connection";import * as schema from "../../../db/schema";import { exchangeClaimsUrl } from "../../../lib/exchange";+import { persistedTeamId } from "../../../lib/keyless-team";+import { logger } from "../../../lib/logger";+import { isValidUuid } from "../../../lib/owner-id";configDotenv();@@ -42,7 +46,19 @@ async function flushHits(): Promise<void> {if (config.USE_DB_AUTHENTICATION !== true) return;try {await db.insert(schema.blocklist_hits).values(batch);- } catch {}+ } catch (error) {+ // Database errors can include the inserted URLs; log only the SQLSTATE.+ const cause = error instanceof DrizzleQueryError ? error.cause : error;+ const code =+ cause !== null && typeof cause === "object" && "code" in cause+ ? cause.code+ : undefined;+ logger.warn("Failed to record blocklist hits", {+ hits: batch.length,+ errorCode:+ typeof code === "string" && /^[0-9A-Z]{5}$/.test(code) ? code : null,+ });+ }}function scheduleHitFlush(): void {@@ -54,6 +70,14 @@ function scheduleHitFlush(): void {if (typeof hitFlushTimer.unref === "function") hitFlushTimer.unref();}+// blocklist_hits.team_id is a uuid column, and one invalid value fails the whole+// batched insert, so a team id that is still not a UUID is stored as null.+function hitTeamId(teamId: string | null | undefined): string | null {+ if (!teamId) return null;+ const stored = persistedTeamId(teamId);+ return isValidUuid(stored) ? stored : null;+}+function recordHit(url: string,domain: string,@@ -65,7 +89,7 @@ function recordHit(id: uuidv7(),domain,url: url.length > 2048 ? url.slice(0, 2048) : url,- team_id: context.team_id ?? null,+ team_id: hitTeamId(context.team_id),origin: context.origin ?? null,});if (hitBuffer.length >= HIT_BUFFER_MAX_SIZE) {