Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): switch to uuidv7 (#2449)

Gergő Móriczb4a193adc62c
diff --git a/apps/api/jest.config.ts b/apps/api/jest.config.ts
index c25eb711f..027f247f4 100644
--- a/apps/api/jest.config.ts
+++ b/apps/api/jest.config.ts
@@ -1,9 +1,8 @@
-import type { Config } from "jest";
+import { createDefaultEsmPreset, type JestConfigWithTsJest } from "ts-jest";
-const config: Config = {
+const config: JestConfigWithTsJest = {
+ ...createDefaultEsmPreset(),
verbose: true,
- preset: "ts-jest",
- testEnvironment: "node",
testPathIgnorePatterns: ["<rootDir>/dist/"],
forceExit: true,
detectOpenHandles: true,
diff --git a/apps/api/package.json b/apps/api/package.json
index 8a9d8ae36..2c384f856 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -139,7 +139,7 @@
"tough-cookie": "^4.1.4",
"turndown": "^7.1.3",
"undici": "^7.10.0",
- "uuid": "^10.0.0",
+ "uuid": "^13.0.0",
"winston": "^3.14.2",
"ws": "^8.18.0",
"x402-express": "^0.6.5",
@@ -178,8 +178,7 @@
],
"overrides": {
"brace-expansion@>=2.0.0 <=2.0.1": "2.0.2",
- "debug": ">=4.4.3",
- "uuid": "<=10.0.0"
+ "debug": ">=4.4.3"
}
},
"lint-staged": {
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 8b6f3aa1a..fd704a66e 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -8,7 +8,6 @@ settings:
overrides:
brace-expansion@>=2.0.0 <=2.0.1: 2.0.2
debug: '>=4.4.3'
- uuid: <=10.0.0
importers:
@@ -231,8 +230,8 @@ importers:
specifier: ^7.10.0
version: 7.10.0
uuid:
- specifier: <=10.0.0
- version: 9.0.1
+ specifier: ^13.0.0
+ version: 13.0.0
winston:
specifier: ^3.14.2
version: 3.14.2
@@ -6847,6 +6846,14 @@ packages:
resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}
engines: {node: '>= 0.4.0'}
+ resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==}
+ hasBin: true
+
+ resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
+ hasBin: true
+
resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==}
hasBin: true
@@ -7565,7 +7572,7 @@ snapshots:
idb-keyval: 6.2.1
preact: 10.24.2
transitivePeerDependencies:
- '@types/react'
@@ -7639,7 +7646,7 @@ snapshots:
idb-keyval: 6.2.1
preact: 10.24.2
transitivePeerDependencies:
- '@types/react'
@@ -7842,11 +7849,11 @@ snapshots:
ethereum-cryptography: 2.2.1
micro-ftch: 0.3.1
dependencies:
'@metamask/rpc-errors': 7.0.2
eventemitter3: 5.0.1
transitivePeerDependencies:
- supports-color
@@ -7912,7 +7919,7 @@ snapshots:
p-limit: 3.1.0
retry-request: 7.0.2([email protected])
teeny-request: 9.0.0([email protected])
- uuid: 9.0.1
+ uuid: 8.3.2
transitivePeerDependencies:
- encoding
- supports-color
@@ -8379,7 +8386,7 @@ snapshots:
readable-stream: 3.6.2
socket.io-client: 4.8.1([email protected])([email protected])
utf-8-validate: 5.0.10
- uuid: 9.0.1
+ uuid: 8.3.2
transitivePeerDependencies:
- supports-color
@@ -8408,7 +8415,7 @@ snapshots:
socket.io-client: 4.8.1([email protected])([email protected])
tslib: 2.8.1
util: 0.12.5
- uuid: 9.0.1
+ uuid: 8.3.2
transitivePeerDependencies:
- bufferutil
- encoding
@@ -9915,7 +9922,7 @@ snapshots:
dependencies:
big.js: 6.2.2
dayjs: 1.11.13
transitivePeerDependencies:
- bufferutil
- typescript
@@ -9928,7 +9935,7 @@ snapshots:
'@reown/appkit-wallet': 1.7.8([email protected])([email protected])([email protected])
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10074,7 +10081,7 @@ snapshots:
'@walletconnect/logger': 2.1.2
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10127,7 +10134,7 @@ snapshots:
bs58: 6.0.0
valtio: 1.13.2([email protected])
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
@@ -10168,7 +10175,7 @@ snapshots:
dependencies:
'@safe-global/safe-gateway-typescript-sdk': 3.23.1
transitivePeerDependencies:
- bufferutil
- typescript
@@ -11195,19 +11202,19 @@ snapshots:
optional: true
dependencies:
'@safe-global/safe-apps-provider': 0.18.6([email protected])([email protected])([email protected])([email protected])
cbw-sdk: '@coinbase/[email protected]'
optionalDependencies:
typescript: 5.8.3
transitivePeerDependencies:
@@ -11241,11 +11248,11 @@ snapshots:
- wagmi
- zod
dependencies:
eventemitter3: 5.0.1
mipd: 0.0.7([email protected])
optionalDependencies:
'@tanstack/query-core': 5.83.1
@@ -11793,6 +11800,11 @@ snapshots:
typescript: 5.8.3
zod: 3.22.4
+ optionalDependencies:
+ typescript: 5.8.3
+ zod: 3.24.2
+
optionalDependencies:
typescript: 5.8.3
@@ -11803,6 +11815,11 @@ snapshots:
typescript: 5.8.3
zod: 3.22.4
+ optionalDependencies:
+ typescript: 5.8.3
+ zod: 3.24.2
+
optionalDependencies:
typescript: 5.8.3
@@ -13396,7 +13413,7 @@ snapshots:
json-stringify-safe: 5.0.1
stream-json: 1.9.1
- uuid: 9.0.1
+ uuid: 8.3.2
transitivePeerDependencies:
- bufferutil
@@ -14289,6 +14306,21 @@ snapshots:
transitivePeerDependencies:
- zod
+ dependencies:
+ '@adraffy/ens-normalize': 1.11.1
+ '@noble/ciphers': 1.3.0
+ '@noble/curves': 1.9.6
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ eventemitter3: 5.0.1
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - zod
+
dependencies:
'@adraffy/ens-normalize': 1.11.1
@@ -14516,21 +14548,21 @@ snapshots:
dependencies:
hono: 4.9.10
idb-keyval: 6.2.2
mipd: 0.0.7([email protected])
zod: 4.1.11
optionalDependencies:
'@tanstack/react-query': 5.84.1([email protected])
react: 18.3.1
typescript: 5.8.3
transitivePeerDependencies:
- '@types/react'
- immer
@@ -14813,7 +14845,7 @@ snapshots:
'@types/ws': 8.18.1
buffer: 6.0.3
eventemitter3: 5.0.1
- uuid: 9.0.1
+ uuid: 8.3.2
optionalDependencies:
bufferutil: 4.0.9
@@ -15438,6 +15470,10 @@ snapshots:
+
+
@@ -15492,6 +15528,23 @@ snapshots:
- utf-8-validate
- zod
+ dependencies:
+ '@noble/curves': 1.9.2
+ '@noble/hashes': 1.8.0
+ '@scure/bip32': 1.7.0
+ '@scure/bip39': 1.6.0
+ optionalDependencies:
+ typescript: 5.8.3
+ transitivePeerDependencies:
+ - bufferutil
+ - utf-8-validate
+ - zod
+
dependencies:
'@noble/curves': 1.9.2
@@ -15513,14 +15566,14 @@ snapshots:
dependencies:
xml-name-validator: 5.0.0
dependencies:
'@tanstack/react-query': 5.84.1([email protected])
react: 18.3.1
use-sync-external-store: 1.4.0([email protected])
optionalDependencies:
typescript: 5.8.3
transitivePeerDependencies:
@@ -15729,8 +15782,8 @@ snapshots:
zod: 3.25.76
transitivePeerDependencies:
- '@azure/app-configuration'
diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
index a7486fad1..532b1d222 100644
--- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
+++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
@@ -1,6 +1,6 @@
import request from "supertest";
import dotenv from "dotenv";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
dotenv.config();
@@ -482,7 +482,7 @@ describe("E2E Tests for API Routes", () => {
it.concurrent(
"should prevent duplicate requests using the same idempotency key",
async () => {
- const uniqueIdempotencyKey = uuidv4();
+ const uniqueIdempotencyKey = uuidv7();
// First request with the idempotency key
const firstResponse = await request(TEST_URL)
diff --git a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts b/apps/api/src/__tests__/snips/v2/robots-txt.test.ts
deleted file mode 100644
index 60582704f..000000000
--- a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts
+++ /dev/null
@@ -1,72 +0,0 @@
-import { createRobotsChecker, fetchRobotsTxt } from "../../../lib/robots-txt";
-import { scrapeTimeout } from "./lib";
-
-// TODO: test self-hosted
-describe("Robots.txt tests", () => {
- const robotsUrl = "https://www.google.com/robots.txt";
- const siteUrl = "https://www.google.com/";
- let robotsTxt: string;
- let checker: ReturnType<typeof createRobotsChecker>;
-
- beforeAll(async () => {
- const logger = {
- error: jest.fn(),
- } as any;
- robotsTxt = (
- await fetchRobotsTxt(
- { url: robotsUrl, zeroDataRetention: true },
- "test-scrape",
- logger,
- )
- ).content;
- checker = createRobotsChecker(siteUrl, robotsTxt);
- }, scrapeTimeout);
-
- it(
- "contains expected directives",
- () => {
- expect(robotsTxt).toContain("User-agent");
- expect(robotsTxt).toContain("Disallow");
- },
- scrapeTimeout,
- );
-
- it("allows and disallows URLs as per rules", () => {
- expect(
- checker.robots.isAllowed(
- "https://www.google.com/search?q=test",
- "FireCrawlAgent",
- ),
- ).toBe(false);
-
- expect(
- checker.robots.isAllowed(
- "https://www.google.com/search/about",
- "FireCrawlAgent",
- ),
- ).toBe(true);
-
- expect(
- checker.robots.isAllowed(
- "https://www.google.com/groups",
- "FireCrawlAgent",
- ),
- ).toBe(false);
-
- expect(
- checker.robots.isAllowed(
- "https://www.google.com/preferences",
- "FireCrawlAgent",
- ),
- ).toBe(true);
- });
-
- it("includes Google's sitemap URLs", () => {
- const sitemaps = checker.robots.getSitemaps();
-
- expect(Array.isArray(sitemaps)).toBe(true);
- expect(sitemaps.length).toBeGreaterThan(0);
-
- expect(sitemaps).toContain("https://www.google.com/sitemap.xml");
- });
-});
diff --git a/apps/api/src/controllers/__tests__/crawl.test.ts b/apps/api/src/controllers/__tests__/crawl.test.ts
index a004ee3c0..8d6b539d8 100644
--- a/apps/api/src/controllers/__tests__/crawl.test.ts
+++ b/apps/api/src/controllers/__tests__/crawl.test.ts
@@ -3,7 +3,7 @@ import { Request, Response } from "express";
import { authenticateUser } from "../auth"; // Ensure this import is correct
import { createIdempotencyKey } from "../../services/idempotency/create";
import { validateIdempotencyKey } from "../../services/idempotency/validate";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
jest.mock("../auth", () => ({
authenticateUser: jest.fn().mockResolvedValue({
@@ -20,7 +20,7 @@ describe("crawlController", () => {
it("should prevent duplicate requests using the same idempotency key", async () => {
const req = {
headers: {
- "x-idempotency-key": await uuidv4(),
+ "x-idempotency-key": await uuidv7(),
Authorization: `Bearer ${process.env.TEST_API_KEY}`,
},
body: {
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 8c795f06f..6e21c5e97 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -11,7 +11,7 @@ import {
defaultCrawlerOptions,
defaultOrigin,
} from "../../../src/lib/default-values";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { logger } from "../../../src/lib/logger";
import {
addCrawlJob,
@@ -51,7 +51,7 @@ export async function crawlController(req: Request, res: Response) {
});
}
- const id = uuidv4();
+ const id = uuidv7();
redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>
logger.error("Failed to add team to teams_using_v0", {
@@ -151,7 +151,7 @@ export async function crawlController(req: Request, res: Response) {
// try {
// const a = new WebScraperDataProvider();
// await a.setOptions({
- // jobId: uuidv4(),
+ // jobId: uuidv7(),
// mode: "single_urls",
// urls: [url],
// crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },
@@ -226,7 +226,7 @@ export async function crawlController(req: Request, res: Response) {
basePriority: 21,
});
const jobs = urls.map(url => {
- const uuid = uuidv4();
+ const uuid = uuidv7();
return {
jobId: uuid,
data: {
@@ -270,7 +270,7 @@ export async function crawlController(req: Request, res: Response) {
// Not needed, first one should be 15.
// const jobPriority = await getJobPriority({team_id, basePriority: 10})
- const jobId = uuidv4();
+ const jobId = uuidv7();
await addScrapeJob(
{
url,
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 2229295b9..267054bb1 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -13,7 +13,7 @@ import {
} from "../../lib/default-values";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { redisEvictConnection } from "../../../src/services/redis";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { logger } from "../../lib/logger";
import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
@@ -176,7 +176,7 @@ export async function scrapeController(req: Request, res: Response) {
});
}
- const jobId = uuidv4();
+ const jobId = uuidv7();
redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>
logger.error("Failed to add team to teams_using_v0", {
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 812fdc08b..9a9722748 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -9,7 +9,7 @@ import { logJob } from "../../services/logging/log_job";
import { PageOptions, SearchOptions } from "../../lib/entities";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { logger } from "../../lib/logger";
import { redisEvictConnection } from "../../../src/services/redis";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
@@ -106,7 +106,7 @@ async function searchHelper(
const jobDatas = res.map(x => {
const url = x.url;
- const uuid = uuidv4();
+ const uuid = uuidv7();
return {
jobId: uuid,
data: {
@@ -175,7 +175,7 @@ export async function searchController(req: Request, res: Response) {
});
}
- const jobId = uuidv4();
+ const jobId = uuidv7();
redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>
logger.error("Failed to add team to teams_using_v0", {
diff --git a/apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIP b/apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIP
index 621c74361..6db51ff27 100644
--- a/apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIP
+++ b/apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIP
@@ -3,7 +3,7 @@ import { Request, Response } from 'express';
import { authenticateUser } from '../auth'; // Ensure this import is correct
import { createIdempotencyKey } from '../../services/idempotency/create';
import { validateIdempotencyKey } from '../../services/idempotency/validate';
-import { v4 as uuidv4 } from 'uuid';
+import { v7 as uuidv7 } from 'uuid';
jest.mock('../auth', () => ({
authenticateUser: jest.fn().mockResolvedValue({
@@ -20,7 +20,7 @@ describe('crawlController', () => {
it('should prevent duplicate requests using the same idempotency key', async () => {
const req = {
headers: {
- 'x-idempotency-key': await uuidv4(),
+ 'x-idempotency-key': await uuidv7(),
'Authorization': `Bearer ${process.env.TEST_API_KEY}`
},
body: {
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index a43064e8b..cf31a4d6f 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -1,5 +1,5 @@
import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import {
BatchScrapeRequest,
batchScrapeRequestSchema,
@@ -49,7 +49,7 @@ export async function batchScrapeController(
const zeroDataRetention =
req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
- const id = req.body.appendToId ?? uuidv4();
+ const id = req.body.appendToId ?? uuidv7();
const logger = _logger.child({
crawlId: id,
batchScrapeId: id,
@@ -160,7 +160,7 @@ export async function batchScrapeController(
logger.debug("Using job priority " + jobPriority, { jobPriority });
const jobs = urls.map(x => ({
- jobId: uuidv4(),
+ jobId: uuidv7(),
data: {
url: x,
mode: "single_urls" as const,
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index 6c4c6155b..634e8ce1c 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -8,7 +8,7 @@ import {
RequestWithAuth,
} from "./types";
import { WebSocket } from "ws";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { logger } from "../../lib/logger";
import {
getCrawl,
@@ -190,7 +190,7 @@ export async function crawlStatusWSController(
} catch (err) {
Sentry.captureException(err);
- const id = uuidv4();
+ const id = uuidv7();
let verbose = JSON.stringify(err);
if (verbose === "{}") {
if (err instanceof Error) {
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index bdc3190e3..6ab3407d2 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -1,5 +1,5 @@
import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import {
CrawlRequest,
crawlRequestSchema,
@@ -37,7 +37,7 @@ export async function crawlController(
const zeroDataRetention =
req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
- const id = uuidv4();
+ const id = uuidv7();
const logger = _logger.child({
crawlId: id,
module: "api/v1",
diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.ts
index 682b95de3..086d6f655 100644
--- a/apps/api/src/controllers/v1/deep-research.ts
+++ b/apps/api/src/controllers/v1/deep-research.ts
@@ -1,3 +1,4 @@
+import { v7 as uuidv7 } from "uuid";
import { Request, Response } from "express";
import { ErrorResponse, extractOptions, RequestWithAuth } from "./types";
import { getDeepResearchQueue } from "../../services/queue-service";
@@ -89,7 +90,7 @@ export async function deepResearchController(
req.body = deepResearchRequestSchema.parse(req.body);
- const researchId = crypto.randomUUID();
+ const researchId = uuidv7();
const jobData = {
request: req.body,
teamId: req.auth.team_id,
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 09218c158..fb4ba20e1 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -1,3 +1,4 @@
+import { v7 as uuidv7 } from "uuid";
import { Request, Response } from "express";
import {
RequestWithAuth,
@@ -121,7 +122,7 @@ export async function extractController(
}
}
- const extractId = crypto.randomUUID();
+ const extractId = uuidv7();
_logger.info("Extract starting...", {
request: req.body,
diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.ts
index 2e6464e73..d9974b92d 100644
--- a/apps/api/src/controllers/v1/generate-llmstxt.ts
+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts
@@ -1,3 +1,4 @@
+import { v7 as uuidv7 } from "uuid";
import { Response } from "express";
import {
ErrorResponse,
@@ -36,7 +37,7 @@ export async function generateLLMsTextController(
req.body = generateLLMsTextRequestSchema.parse(req.body);
- const generationId = crypto.randomUUID();
+ const generationId = uuidv7();
const jobData = {
request: req.body,
teamId: req.auth.team_id,
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 3d3908884..9dbf0f93f 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -1,5 +1,5 @@
import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import {
MapDocument,
mapRequestSchema,
@@ -115,7 +115,7 @@ export async function getMapResults({
timeout?: number;
location?: ScrapeOptions["location"];
}): Promise<MapResult> {
- const id = uuidv4();
+ const id = uuidv7();
let links: string[] = [url];
let mapResults: MapDocument[] = [];
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 4bfc7cc17..ca3a18af2 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -7,7 +7,7 @@ import {
scrapeRequestSchema,
ScrapeResponse,
} from "./types";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { getJobPriority } from "../../lib/job-priority";
import { fromV1ScrapeOptions } from "../v2/types";
import { TransportableError } from "../../lib/error";
@@ -27,7 +27,7 @@ export async function scrapeController(
(req as any).requestTiming?.startTime || new Date().getTime();
const controllerStartTime = new Date().getTime();
- const jobId: string = uuidv4();
+ const jobId: string = uuidv7();
const preNormalizedBody = { ...req.body };
req.body = scrapeRequestSchema.parse(req.body);
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index d25cc6603..1ddf63799 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -9,7 +9,7 @@ import {
TeamFlags,
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
import { search } from "../../search";
@@ -86,7 +86,7 @@ async function scrapeSearchResult(
directToBullMQ: boolean = false,
isSearchPreview: boolean = false,
): Promise<DocumentWithCostTracking> {
- const jobId = uuidv4();
+ const jobId = uuidv7();
const costTracking = new CostTracking();
@@ -229,7 +229,7 @@ export async function searchController(
(req as any).requestTiming?.startTime || new Date().getTime();
const controllerStartTime = new Date().getTime();
- const jobId = uuidv4();
+ const jobId = uuidv7();
let logger = _logger.child({
jobId,
teamId: req.auth.team_id,
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index fc90e8f97..88b218fd0 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -8,7 +8,7 @@ import {
ScrapeOptions,
TeamFlags,
} from "./types";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
import { search } from "../../search";
@@ -43,7 +43,7 @@ async function scrapeX402SearchResult(
directToBullMQ: boolean = false,
isSearchPreview: boolean = false,
): Promise<DocumentWithCostTracking> {
- const jobId = uuidv4();
+ const jobId = uuidv7();
const costTracking = new CostTracking();
@@ -174,7 +174,7 @@ export async function x402SearchController(
req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
res: Response<SearchResponse & { request?: any }>,
) {
- const jobId = uuidv4();
+ const jobId = uuidv7();
let logger = _logger.child({
jobId,
teamId: req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 4e3dafb4f..6ad641ab2 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -1,5 +1,5 @@
import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import {
BatchScrapeRequest,
batchScrapeRequestSchema,
@@ -49,7 +49,7 @@ export async function batchScrapeController(
const zeroDataRetention =
req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
- const id = req.body.appendToId ?? uuidv4();
+ const id = req.body.appendToId ?? uuidv7();
const logger = _logger.child({
crawlId: id,
batchScrapeId: id,
@@ -157,7 +157,7 @@ export async function batchScrapeController(
delete (scrapeOptions as any).appendToId;
const jobs = urls.map(x => ({
- jobId: uuidv4(),
+ jobId: uuidv7(),
data: {
url: x,
mode: "single_urls" as const,
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index 91c1dd809..b139907ad 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -8,7 +8,7 @@ import {
RequestWithAuth,
} from "./types";
import { WebSocket } from "ws";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { logger } from "../../lib/logger";
import {
getCrawl,
@@ -193,7 +193,7 @@ export async function crawlStatusWSController(
} catch (err) {
Sentry.captureException(err);
- const id = uuidv4();
+ const id = uuidv7();
let verbose = JSON.stringify(err);
if (verbose === "{}") {
if (err instanceof Error) {
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 1652a0aa7..5beba6eea 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -1,5 +1,5 @@
import { Response } from "express";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import {
CrawlRequest,
crawlRequestSchema,
@@ -44,7 +44,7 @@ export async function crawlController(
const zeroDataRetention =
req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
- const id = uuidv4();
+ const id = uuidv7();
const logger = _logger.child({
crawlId: id,
module: "api/v2",
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
index 08dce3a62..740f952f6 100644
--- a/apps/api/src/controllers/v2/extract.ts
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -1,3 +1,4 @@
+import { v7 as uuidv7 } from "uuid";
import { Response } from "express";
import {
RequestWithAuth,
@@ -47,7 +48,7 @@ export async function extractController(
}
}
- const extractId = crypto.randomUUID();
+ const extractId = uuidv7();
const createdAt = Date.now();
_logger.info("Extract starting...", {
request: req.body,
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 652401cd3..be2f78424 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -13,7 +13,7 @@ import { logger as _logger } from "../../lib/logger";
import { MapTimeoutError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
import { getMapResults, MapResult } from "../../lib/map-utils";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
configDotenv();
@@ -23,7 +23,7 @@ export async function mapController(
res: Response<MapResponse>,
) {
const logger = _logger.child({
- jobId: uuidv4(),
+ jobId: uuidv7(),
teamId: req.auth.team_id,
module: "api/v2",
method: "mapController",
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index b009b59f2..de66d59b2 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -8,7 +8,7 @@ import {
scrapeRequestSchema,
ScrapeResponse,
} from "./types";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { hasFormatOfType } from "../../lib/format-utils";
import { TransportableError } from "../../lib/error";
import { NuQJob } from "../../services/worker/nuq";
@@ -31,7 +31,7 @@ export async function scrapeController(
(req as any).requestTiming?.startTime || new Date().getTime();
const controllerStartTime = new Date().getTime();
- const jobId = uuidv4();
+ const jobId = uuidv7();
const preNormalizedBody = { ...req.body };
// Set initial span attributes
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index c399427f2..c24c697fb 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -9,7 +9,7 @@ import {
TeamFlags,
} from "./types";
import { billTeam } from "../../services/billing/credit_billing";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
import { search } from "../../search/v2";
@@ -57,7 +57,7 @@ async function startScrapeJob(
directToBullMQ: boolean = false,
isSearchPreview: boolean = false,
): Promise<string> {
- const jobId = uuidv4();
+ const jobId = uuidv7();
const zeroDataRetention = flags?.forceZDR ?? false;
@@ -206,7 +206,7 @@ export async function searchController(
(req as any).requestTiming?.startTime || new Date().getTime();
const controllerStartTime = new Date().getTime();
- const jobId = uuidv4();
+ const jobId = uuidv7();
let logger = _logger.child({
jobId,
teamId: req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.ts
index 7dd76719b..64b773c05 100644
--- a/apps/api/src/controllers/v2/x402-search.ts
+++ b/apps/api/src/controllers/v2/x402-search.ts
@@ -8,7 +8,7 @@ import {
ScrapeOptions,
TeamFlags,
} from "./types";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
import { search } from "../../search/v2";
@@ -55,7 +55,7 @@ async function startX420ScrapeJob(
directToBullMQ: boolean = false,
isSearchPreview: boolean = false,
): Promise<string> {
- const jobId = uuidv4();
+ const jobId = uuidv7();
const zeroDataRetention = flags?.forceZDR ?? false;
@@ -199,7 +199,7 @@ export async function x402SearchController(
req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
res: Response<SearchResponse>,
) {
- const jobId = uuidv4();
+ const jobId = uuidv7();
let logger = _logger.child({
jobId,
teamId: req.auth.team_id,
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index c5217ceeb..c38d64e2e 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -26,7 +26,7 @@ import {
ResponseWithSentry,
} from "./controllers/v1/types";
import { ZodError } from "zod";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { attachWsProxy } from "./services/agentLivecastWS";
import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
import { v2Router } from "./routes/v2";
@@ -214,7 +214,7 @@ app.use(
});
}
- const id = res.sentry ?? uuidv4();
+ const id = res.sentry ?? uuidv7();
logger.error(
"Error occurred in request! (" + req.path + ") -- ID " + id + " -- ",
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index ffc99e567..72937a520 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -1,3 +1,4 @@
+import { v7 as uuidv7 } from "uuid";
import {
Document,
ScrapeOptions,
@@ -39,7 +40,7 @@ export async function scrapeDocument(
}
async function attemptScrape(timeout: number) {
- const jobId = crypto.randomUUID();
+ const jobId = uuidv7();
const jobPriority = await getJobPriority({
team_id: options.teamId,
basePriority: 10,
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index 7a23c5737..22f257e82 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -1,3 +1,4 @@
+import { v7 as uuidv7 } from "uuid";
import {
Document,
ScrapeOptions,
@@ -39,7 +40,7 @@ export async function scrapeDocument_F0(
}
async function attemptScrape(timeout: number) {
- const jobId = crypto.randomUUID();
+ const jobId = uuidv7();
const jobPriority = await getJobPriority({
team_id: options.teamId,
basePriority: 10,
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 4c24a80fa..5f479a0e3 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import {
TeamFlags,
MapDocument,
@@ -122,7 +122,7 @@ export async function getMapResults({
url = urlObj.toString();
}
- const id = uuidv4();
+ const id = uuidv7();
let mapResults: MapDocument[] = [];
const zeroDataRetention = flags?.forceZDR ?? false;
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index fa0b7989b..c27d76230 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -6,7 +6,7 @@ import { getExtractQueue, getRedisConnection } from "./queue-service";
import { Job, Queue, Worker } from "bullmq";
import { logger as _logger } from "../lib/logger";
import systemMonitor from "./system-monitor";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { configDotenv } from "dotenv";
import {
ExtractResult,
@@ -204,7 +204,7 @@ const workerFun = async (
_logger.info("No longer accepting new jobs. SIGINT");
break;
}
- const token = uuidv4();
+ const token = uuidv7();
const canAcceptConnection = await monitor.acceptConnection();
if (!canAcceptConnection) {
console.log("Can't accept connection due to RAM/CPU load");
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index fa5711778..f107bd444 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -15,7 +15,7 @@ import {
startBillingBatchProcessing,
} from "../billing/batch_billing";
import systemMonitor from "../system-monitor";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import {
index_supabase_service,
processIndexInsertJobs,
@@ -475,7 +475,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
zeroDataRetention: false,
};
- const crawlId = uuidv4();
+ const crawlId = uuidv7();
// robots disabled for now
// const crawler = crawlToCrawler(crawlId, sc, null);
@@ -597,7 +597,7 @@ const workerFun = async (
break;
}
- const token = uuidv4();
+ const token = uuidv7();
const canAcceptConnection = await monitor.acceptConnection();
if (!canAcceptConnection) {
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 6a009c15b..fe0c62ace 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { NotificationType, RateLimiterMode, ScrapeJobData } from "../types";
import {
cleanOldConcurrencyLimitEntries,
@@ -316,7 +316,7 @@ async function addScrapeJobRaw(
export async function addScrapeJob(
webScraperOptions: ScrapeJobData,
- jobId: string = uuidv4(),
+ jobId: string = uuidv7(),
priority: number = 0,
directToBullMQ: boolean = false,
listenable: boolean = false,
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index c1a7cb076..ef033b6a7 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -10,7 +10,7 @@ import {
import { Job, Queue, Worker } from "bullmq";
import { logger as _logger } from "../lib/logger";
import systemMonitor from "./system-monitor";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import { configDotenv } from "dotenv";
import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";
import { performDeepResearch } from "../lib/deep-research/deep-research-service";
@@ -263,7 +263,7 @@ const workerFun = async (
_logger.info("No longer accepting new jobs. SIGINT");
break;
}
- const token = uuidv4();
+ const token = uuidv7();
const canAcceptConnection = await monitor.acceptConnection();
if (!canAcceptConnection) {
console.log("Can't accept connection due to RAM/CPU load");
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 1e5af10e8..35bc26ddb 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -10,7 +10,7 @@ import {
} from "../../lib/concurrency-limit";
import { addJobPriority, deleteJobPriority } from "../../lib/job-priority";
import { cacheableLookup } from "../../scraper/scrapeURL/lib/cacheableLookup";
-import { v4 as uuidv4 } from "uuid";
+import { v7 as uuidv7 } from "uuid";
import {
addCrawlJob,
addCrawlJobs,
@@ -110,7 +110,7 @@ async function billScrapeJob(
process.env.USE_DB_AUTHENTICATION === "true"
) {
try {
- const billingJobId = uuidv4();
+ const billingJobId = uuidv7();
logger.debug(
`Adding billing job to queue for team ${job.data.team_id}`,
{
@@ -372,7 +372,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
team_id: sc.team_id,
basePriority: job.data.crawl_id ? 20 : 10,
});
- const jobId = uuidv4();
+ const jobId = uuidv7();
logger.debug(
"Determined job priority " +
@@ -779,7 +779,7 @@ async function addKickoffSitemapJob(
return;
}
- const jobId = uuidv4();
+ const jobId = uuidv7();
await _addScrapeJobToBullMQ(
{
mode: "kickoff_sitemap" as const,
@@ -828,7 +828,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
logger.debug("Locking URL...");
await lockURL(job.data.crawl_id, sc, job.data.url);
- const jobId = uuidv4();
+ const jobId = uuidv7();
logger.debug("Adding scrape job to Redis...", { jobId });
await addScrapeJob(
{
@@ -914,7 +914,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
logger.debug("Using job priority " + jobPriority, { jobPriority });
const jobs = indexLinks.map(url => {
- const uuid = uuidv4();
+ const uuid = uuidv7();
return {
jobId: uuid,
data: {
@@ -1040,7 +1040,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
job.data.zeroDataRetention || (sc.zeroDataRetention ?? false),
apiKeyId: job.data.apiKeyId,
} satisfies ScrapeJobSingleUrls,
- jobId: uuidv4(),
+ jobId: uuidv7(),
priority: jobPriority,
}));