Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): store keyless team ids as UUIDs in blocklist hits and analytics (#5188)

Max31e468a5c619
diff --git a/apps/api/src/lib/job-access-store.ts b/apps/api/src/lib/job-access-store.ts
index f5c033cfc..614ec72c5 100644
--- a/apps/api/src/lib/job-access-store.ts
+++ b/apps/api/src/lib/job-access-store.ts
@@ -1,7 +1,7 @@
import { config } from "../config";
import { getBigtableTable } from "./bigtable-client";
import { saltedUuidV7RowKey } from "./bigtable-row-key";
-import { keylessTeamUuid } from "./keyless";
+import { persistedTeamId } from "./keyless-team";
import { setSpanAttributes, withSpan } from "./otel-tracer";
/**
@@ -14,15 +14,8 @@ export const JOB_ACCESS_TTL_MS = 24 * 60 * 60 * 1000;
const FAMILY = "j";
const QUALIFIER = "v";
-const PREVIEW_TEAM_ID = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
-
export function normalizeJobAccessTeamId(teamId: string): string {
- return (
- keylessTeamUuid(teamId) ??
- (teamId === "preview" || teamId.startsWith("preview_")
- ? PREVIEW_TEAM_ID
- : teamId)
- );
+ return persistedTeamId(teamId);
}
export const API_JOB_KINDS = [
diff --git a/apps/api/src/lib/keyless-team.ts b/apps/api/src/lib/keyless-team.ts
new file mode 100644
index 000000000..9c162f6f1
--- /dev/null
+++ b/apps/api/src/lib/keyless-team.ts
@@ -0,0 +1,47 @@
+import { v5 as uuidv5 } from "uuid";
+
+// Keyless team ids live here, apart from lib/keyless, so modules that only
+// need them do not load its Redis and database clients.
+
+// Keyless teams reuse the `preview_` prefix so billing (autumn `isPreviewTeam`)
+// and GCS persistence are skipped automatically, with a dedicated infix so the
+// IP can be recovered when charging credits after a request completes.
+export const KEYLESS_TEAM_PREFIX = "preview_keyless_";
+
+export function keylessTeamId(ip: string): string {
+ return `${KEYLESS_TEAM_PREFIX}${ip}`;
+}
+
+// Fixed namespace for deriving a stable per-keyless-team UUID. Tables like
+// `scrapes` require a UUID team_id, but keyless teams are `preview_keyless_<ip>`
+// strings. Mapping each to a deterministic UUIDv5 keeps rows per-IP-distinct
+// (so ownership checks such as interact still isolate keyless users) while
+// satisfying the UUID column, unlike the shared preview placeholder.
+const KEYLESS_TEAM_UUID_NAMESPACE = "9e6c8f2a-3b1d-4c7e-8a5f-2d4b6e8c0a1f";
+
+/**
+ * Deterministic UUID for a keyless team's persisted rows, or null for
+ * non-keyless teams (callers then fall back to the raw/placeholder team_id).
+ */
+export function keylessTeamUuid(
+ teamId: string | null | undefined,
+): string | null {
+ if (!teamId || !teamId.startsWith(KEYLESS_TEAM_PREFIX)) return null;
+ return uuidv5(teamId, KEYLESS_TEAM_UUID_NAMESPACE);
+}
+
+// Shared team UUID that other preview callers are stored under.
+const PREVIEW_TEAM_ID = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
+
+/**
+ * The team id to store in a UUID column: a keyless caller's UUID, the shared
+ * preview team for other preview callers, or the team id unchanged.
+ */
+export function persistedTeamId(teamId: string): string {
+ return (
+ keylessTeamUuid(teamId) ??
+ (teamId === "preview" || teamId.startsWith("preview_")
+ ? PREVIEW_TEAM_ID
+ : teamId)
+ );
+}
diff --git a/apps/api/src/lib/keyless.ts b/apps/api/src/lib/keyless.ts
index b034b1052..da3aa9dff 100644
--- a/apps/api/src/lib/keyless.ts
+++ b/apps/api/src/lib/keyless.ts
@@ -1,7 +1,11 @@
import { createHmac } from "node:crypto";
import { isIPv4 } from "node:net";
-import { v5 as uuidv5 } from "uuid";
import { config } from "../config";
+import {
+ KEYLESS_TEAM_PREFIX,
+ keylessTeamId,
+ keylessTeamUuid,
+} from "./keyless-team";
import { db } from "../db/connection";
import * as schema from "../db/schema";
import { redisRateLimitClient } from "../services/rate-limiter";
@@ -91,14 +95,7 @@ export function keylessExhaustionTelemetry(ip: string): Record<string, string> {
return conversionCohort ? { conversionCohort } : {};
}
-// Keyless teams reuse the `preview_` prefix so billing (autumn `isPreviewTeam`)
-// and GCS persistence are skipped automatically, with a dedicated infix so the
-// IP can be recovered when charging credits after a request completes.
-const KEYLESS_TEAM_PREFIX = "preview_keyless_";
-
-export function keylessTeamId(ip: string): string {
- return `${KEYLESS_TEAM_PREFIX}${ip}`;
-}
+export { keylessTeamId, keylessTeamUuid };
function keylessIpFromTeamId(teamId: string): string | null {
return teamId.startsWith(KEYLESS_TEAM_PREFIX)
@@ -106,24 +103,6 @@ function keylessIpFromTeamId(teamId: string): string | null {
: null;
}
-// Fixed namespace for deriving a stable per-keyless-team UUID. Tables like
-// `scrapes` require a UUID team_id, but keyless teams are `preview_keyless_<ip>`
-// strings. Mapping each to a deterministic UUIDv5 keeps rows per-IP-distinct
-// (so ownership checks such as interact still isolate keyless users) while
-// satisfying the UUID column — unlike the shared preview placeholder.
-const KEYLESS_TEAM_UUID_NAMESPACE = "9e6c8f2a-3b1d-4c7e-8a5f-2d4b6e8c0a1f";
-
-/**
- * Deterministic UUID for a keyless team's persisted rows, or null for
- * non-keyless teams (callers then fall back to the raw/placeholder team_id).
- */
-export function keylessTeamUuid(
- teamId: string | null | undefined,
-): string | null {
- if (!teamId || !teamId.startsWith(KEYLESS_TEAM_PREFIX)) return null;
- return uuidv5(teamId, KEYLESS_TEAM_UUID_NAMESPACE);
-}
-
/**
* Keyless is allowed only for a *valid IPv4* client identity. This both:
* - denies IPv6 (a single client controls a huge block — a /64 is ~18
diff --git a/apps/api/src/lib/tracking.test.ts b/apps/api/src/lib/tracking.test.ts
index 4b02034c8..b9787fa75 100644
--- a/apps/api/src/lib/tracking.test.ts
+++ b/apps/api/src/lib/tracking.test.ts
@@ -1,7 +1,11 @@
import { chInsert } from "./clickhouse-client";
+import { keylessTeamId, keylessTeamUuid } from "./keyless-team";
import {
buildMonitorTargetInterestRows,
trackMonitorTargetInterest,
+ trackScrape,
+ trackSearchRequest,
+ trackSearchResults,
} from "./tracking";
import type { MonitorTarget } from "../services/monitoring/types";
@@ -169,3 +173,71 @@ describe("monitor target interest tracking", () => {
expect(chInsert).not.toHaveBeenCalled();
});
});
+
+describe("analytics team ids", () => {
+ const keyless = keylessTeamId("203.0.113.7");
+ const previewTeam = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ // The analytics team_id columns are UUIDs, so keyless and preview callers
+ // are stored under their UUIDs rather than their raw team ids.
+ it.each([
+ [keyless, keylessTeamUuid(keyless)],
+ ["preview", previewTeam],
+ [
+ "5f0c2b1e-8d4a-4c3b-9e2f-1a2b3c4d5e6f",
+ "5f0c2b1e-8d4a-4c3b-9e2f-1a2b3c4d5e6f",
+ ],
+ ])("stores team %s as %s", async (teamId, stored) => {
+ await trackScrape({
+ scrapeId: "scrape-1",
+ requestId: "request-1",
+ teamId,
+ url: "https://example.com/",
+ origin: "api",
+ kind: "scrape",
+ isSuccessful: true,
+ creditsCost: 1,
+ timeTaken: 1,
+ zeroDataRetention: false,
+ });
+ await trackSearchRequest({
+ searchId: "search-1",
+ requestId: "request-1",
+ teamId,
+ query: "example",
+ origin: "api",
+ kind: "search",
+ apiVersion: "v2",
+ sources: ["web"],
+ numResults: 1,
+ searchCredits: 1,
+ scrapeCredits: 0,
+ totalCredits: 1,
+ hasScrapeFormats: false,
+ scrapeFormats: [],
+ isSuccessful: true,
+ timeTaken: 1,
+ zeroDataRetention: false,
+ });
+ await trackSearchResults({
+ searchId: "search-1",
+ teamId,
+ response: { web: [{ url: "https://example.com/" }] } as any,
+ zeroDataRetention: false,
+ hasScrapeFormats: false,
+ });
+
+ const tables = vi.mocked(chInsert).mock.calls.map(([table]) => table);
+ expect(tables).toEqual([
+ "scrape_results",
+ "search_requests",
+ "search_results",
+ ]);
+ for (const [, rows] of vi.mocked(chInsert).mock.calls)
+ expect(rows.map(row => row.team_id)).toEqual([stored]);
+ });
+});
diff --git a/apps/api/src/lib/tracking.ts b/apps/api/src/lib/tracking.ts
index 07b907115..5a7228f7c 100644
--- a/apps/api/src/lib/tracking.ts
+++ b/apps/api/src/lib/tracking.ts
@@ -1,5 +1,6 @@
import { createHash } from "crypto";
import { chInsert } from "./clickhouse-client";
+import { persistedTeamId } from "./keyless-team";
import type { SearchV2Response } from "./entities";
import type { MonitorTarget } from "../services/monitoring/types";
@@ -54,7 +55,7 @@ export async function trackScrape(opts: TrackScrapeParams): Promise<void> {
{
scrape_id: opts.scrapeId,
request_id: opts.requestId,
- team_id: opts.teamId,
+ team_id: persistedTeamId(opts.teamId),
url: opts.url,
url_domain: extractDomain(opts.url),
origin: opts.origin,
@@ -259,7 +260,7 @@ export async function trackSearchRequest(
{
search_id: opts.searchId,
request_id: opts.requestId,
- team_id: opts.teamId,
+ team_id: persistedTeamId(opts.teamId),
query: opts.query,
origin: opts.origin,
kind: opts.kind,
@@ -338,7 +339,7 @@ export async function trackSearchResults(
"search_results",
urls.map(({ url, type, index }) => ({
search_id: opts.searchId,
- team_id: opts.teamId,
+ team_id: persistedTeamId(opts.teamId),
url,
url_domain: extractDomain(url),
result_type: type,
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist-hits.test.ts b/apps/api/src/scraper/WebScraper/utils/blocklist-hits.test.ts
new file mode 100644
index 000000000..0ac827893
--- /dev/null
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist-hits.test.ts
@@ -0,0 +1,83 @@
+import { keylessTeamId, keylessTeamUuid } from "../../../lib/keyless-team";
+import { logger } from "../../../lib/logger";
+import { initializeBlocklist, isUrlBlocked } from "./blocklist";
+
+const insertedHits = vi.hoisted(() =>
+ vi.fn(async (_hits: { team_id: string | null }[]) => {}),
+);
+
+vi.mock("../../../config", () => ({
+ config: {
+ USE_DB_AUTHENTICATION: true,
+ DISABLE_BLOCKLIST: false,
+ },
+}));
+
+vi.mock("../../../db/connection", () => ({
+ db: {
+ insert: vi.fn(() => ({ values: insertedHits })),
+ },
+ dbRr: {
+ select: vi.fn(() => ({
+ from: vi.fn(async () => [
+ {
+ org_id: null,
+ data: { blocklist: ["facebook.com"], allowedKeywords: [] },
+ },
+ ]),
+ })),
+ },
+}));
+
+beforeEach(async () => {
+ vi.useFakeTimers();
+ insertedHits.mockClear();
+ await initializeBlocklist();
+});
+
+afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+});
+
+const recordHits = async (teamIds: (string | null)[]) => {
+ for (const team_id of teamIds)
+ expect(
+ isUrlBlocked("https://facebook.com/page", null, {
+ team_id,
+ origin: "test",
+ }),
+ ).toBe(true);
+ await vi.advanceTimersByTimeAsync(5000);
+};
+
+it("stores every hit in a batch under a UUID team id or null", async () => {
+ const team = "5f0c2b1e-8d4a-4c3b-9e2f-1a2b3c4d5e6f";
+ const keyless = keylessTeamId("203.0.113.7");
+
+ await recordHits([team, keyless, "preview", "not-a-uuid", null]);
+
+ // One batch carries every hit, so each team id must be a UUID or null.
+ expect(insertedHits).toHaveBeenCalledOnce();
+ expect(insertedHits.mock.calls[0][0].map(hit => hit.team_id)).toEqual([
+ team,
+ keylessTeamUuid(keyless),
+ "3adefd26-77ec-5968-8dcf-c94b5630d1de",
+ null,
+ null,
+ ]);
+});
+
+it("logs a failed batch with only its SQLSTATE", async () => {
+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
+ insertedHits.mockRejectedValueOnce(
+ Object.assign(new Error("https://facebook.com/page"), { code: "22P02" }),
+ );
+
+ await recordHits([null, null]);
+
+ expect(warn).toHaveBeenCalledWith("Failed to record blocklist hits", {
+ hits: 2,
+ errorCode: "22P02",
+ });
+});
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index a8b7f8ed7..c0be26e70 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -1,4 +1,5 @@
import { configDotenv } from "dotenv";
+import { DrizzleQueryError } from "drizzle-orm/errors";
import { v7 as uuidv7 } from "uuid";
import { config } from "../../../config";
import { parse } from "tldts";
@@ -6,6 +7,9 @@ import { TeamFlags } from "../../../controllers/v1/types";
import { db, dbRr } from "../../../db/connection";
import * as schema from "../../../db/schema";
import { exchangeClaimsUrl } from "../../../lib/exchange";
+import { persistedTeamId } from "../../../lib/keyless-team";
+import { logger } from "../../../lib/logger";
+import { isValidUuid } from "../../../lib/owner-id";
configDotenv();
@@ -42,7 +46,19 @@ async function flushHits(): Promise<void> {
if (config.USE_DB_AUTHENTICATION !== true) return;
try {
await db.insert(schema.blocklist_hits).values(batch);
- } catch {}
+ } catch (error) {
+ // Database errors can include the inserted URLs; log only the SQLSTATE.
+ const cause = error instanceof DrizzleQueryError ? error.cause : error;
+ const code =
+ cause !== null && typeof cause === "object" && "code" in cause
+ ? cause.code
+ : undefined;
+ logger.warn("Failed to record blocklist hits", {
+ hits: batch.length,
+ errorCode:
+ typeof code === "string" && /^[0-9A-Z]{5}$/.test(code) ? code : null,
+ });
+ }
}
function scheduleHitFlush(): void {
@@ -54,6 +70,14 @@ function scheduleHitFlush(): void {
if (typeof hitFlushTimer.unref === "function") hitFlushTimer.unref();
}
+// blocklist_hits.team_id is a uuid column, and one invalid value fails the whole
+// batched insert, so a team id that is still not a UUID is stored as null.
+function hitTeamId(teamId: string | null | undefined): string | null {
+ if (!teamId) return null;
+ const stored = persistedTeamId(teamId);
+ return isValidUuid(stored) ? stored : null;
+}
+
function recordHit(
url: string,
domain: string,
@@ -65,7 +89,7 @@ function recordHit(
id: uuidv7(),
domain,
url: url.length > 2048 ? url.slice(0, 2048) : url,
- team_id: context.team_id ?? null,
+ team_id: hitTeamId(context.team_id),
origin: context.origin ?? null,
});
if (hitBuffer.length >= HIT_BUFFER_MAX_SIZE) {