Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): switch to uuidv7 (#2449)
Gergő Móriczb4a193adc62c
diff --git a/apps/api/jest.config.ts b/apps/api/jest.config.tsindex c25eb711f..027f247f4 100644--- a/apps/api/jest.config.ts+++ b/apps/api/jest.config.ts@@ -1,9 +1,8 @@-import type { Config } from "jest";+import { createDefaultEsmPreset, type JestConfigWithTsJest } from "ts-jest";-const config: Config = {+const config: JestConfigWithTsJest = {+ ...createDefaultEsmPreset(),verbose: true,- preset: "ts-jest",- testEnvironment: "node",testPathIgnorePatterns: ["<rootDir>/dist/"],forceExit: true,detectOpenHandles: true,diff --git a/apps/api/package.json b/apps/api/package.jsonindex 8a9d8ae36..2c384f856 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -139,7 +139,7 @@"tough-cookie": "^4.1.4","turndown": "^7.1.3","undici": "^7.10.0",- "uuid": "^10.0.0",+ "uuid": "^13.0.0","winston": "^3.14.2","ws": "^8.18.0","x402-express": "^0.6.5",@@ -178,8 +178,7 @@],"overrides": {"brace-expansion@>=2.0.0 <=2.0.1": "2.0.2",- "debug": ">=4.4.3",- "uuid": "<=10.0.0"+ "debug": ">=4.4.3"}},"lint-staged": {diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 8b6f3aa1a..fd704a66e 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -8,7 +8,6 @@ settings:overrides:brace-expansion@>=2.0.0 <=2.0.1: 2.0.2debug: '>=4.4.3'- uuid: <=10.0.0importers:@@ -231,8 +230,8 @@ importers:specifier: ^7.10.0version: 7.10.0uuid:- specifier: <=10.0.0- version: 9.0.1+ specifier: ^13.0.0+ version: 13.0.0winston:specifier: ^3.14.2version: 3.14.2@@ -6847,6 +6846,14 @@ packages:resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}engines: {node: '>= 0.4.0'}+ resolution: {integrity: sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w==}+ hasBin: true++ resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}+ hasBin: true+resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==}hasBin: true@@ -7565,7 +7572,7 @@ snapshots:idb-keyval: 6.2.1ox: 0.6.9([email protected])([email protected])preact: 10.24.2transitivePeerDependencies:- '@types/react'@@ -7639,7 +7646,7 @@ snapshots:idb-keyval: 6.2.1ox: 0.6.9([email protected])([email protected])preact: 10.24.2transitivePeerDependencies:- '@types/react'@@ -7842,11 +7849,11 @@ snapshots:ethereum-cryptography: 2.2.1micro-ftch: 0.3.1- '@gemini-wallet/[email protected]([email protected]([email protected])([email protected])([email protected])([email protected]))':+ '@gemini-wallet/[email protected]([email protected]([email protected])([email protected])([email protected])([email protected]))':dependencies:'@metamask/rpc-errors': 7.0.2eventemitter3: 5.0.1transitivePeerDependencies:- supports-color@@ -7912,7 +7919,7 @@ snapshots:p-limit: 3.1.0retry-request: 7.0.2([email protected])teeny-request: 9.0.0([email protected])- uuid: 9.0.1+ uuid: 8.3.2transitivePeerDependencies:- encoding- supports-color@@ -8379,7 +8386,7 @@ snapshots:readable-stream: 3.6.2socket.io-client: 4.8.1([email protected])([email protected])utf-8-validate: 5.0.10- uuid: 9.0.1+ uuid: 8.3.2transitivePeerDependencies:- supports-color@@ -8408,7 +8415,7 @@ snapshots:socket.io-client: 4.8.1([email protected])([email protected])tslib: 2.8.1util: 0.12.5- uuid: 9.0.1+ uuid: 8.3.2transitivePeerDependencies:- bufferutil- encoding@@ -9915,7 +9922,7 @@ snapshots:dependencies:big.js: 6.2.2dayjs: 1.11.13transitivePeerDependencies:- bufferutil- typescript@@ -9928,7 +9935,7 @@ snapshots:'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10074,7 +10081,7 @@ snapshots:'@walletconnect/logger': 2.1.2'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10127,7 +10134,7 @@ snapshots:'@walletconnect/universal-provider': 2.21.0([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])bs58: 6.0.0valtio: 1.13.2([email protected])transitivePeerDependencies:- '@azure/app-configuration'- '@azure/cosmos'@@ -10168,7 +10175,7 @@ snapshots:'@safe-global/[email protected]([email protected])([email protected])([email protected])([email protected])':dependencies:'@safe-global/safe-gateway-typescript-sdk': 3.23.1transitivePeerDependencies:- bufferutil- typescript@@ -11195,19 +11202,19 @@ snapshots:'@unrs/[email protected]':optional: true- '@wagmi/[email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])':+ '@wagmi/[email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])':dependencies:'@base-org/account': 1.1.1([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])'@coinbase/wallet-sdk': 4.3.6([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])- '@gemini-wallet/core': 0.2.0([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@gemini-wallet/core': 0.2.0([email protected]([email protected])([email protected])([email protected])([email protected]))'@safe-global/safe-apps-provider': 0.18.6([email protected])([email protected])([email protected])([email protected])'@safe-global/safe-apps-sdk': 9.1.0([email protected])([email protected])([email protected])([email protected])- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))'@walletconnect/ethereum-provider': 2.21.1([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])cbw-sdk: '@coinbase/[email protected]'- porto: 0.2.19(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))+ porto: 0.2.19(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))optionalDependencies:typescript: 5.8.3transitivePeerDependencies:@@ -11241,11 +11248,11 @@ snapshots:- wagmi- zod- '@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))':+ '@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))':dependencies:eventemitter3: 5.0.1mipd: 0.0.7([email protected])optionalDependencies:'@tanstack/query-core': 5.83.1@@ -11793,6 +11800,11 @@ snapshots:typescript: 5.8.3zod: 3.22.4+ optionalDependencies:+ typescript: 5.8.3+ zod: 3.24.2+optionalDependencies:typescript: 5.8.3@@ -11803,6 +11815,11 @@ snapshots:typescript: 5.8.3zod: 3.22.4+ optionalDependencies:+ typescript: 5.8.3+ zod: 3.24.2+optionalDependencies:typescript: 5.8.3@@ -13396,7 +13413,7 @@ snapshots:json-stringify-safe: 5.0.1stream-json: 1.9.1- uuid: 9.0.1+ uuid: 8.3.2ws: 7.5.10([email protected])([email protected])transitivePeerDependencies:- bufferutil@@ -14289,6 +14306,21 @@ snapshots:transitivePeerDependencies:- zod+ dependencies:+ '@adraffy/ens-normalize': 1.11.1+ '@noble/ciphers': 1.3.0+ '@noble/curves': 1.9.6+ '@noble/hashes': 1.8.0+ '@scure/bip32': 1.7.0+ '@scure/bip39': 1.6.0+ abitype: 1.1.1([email protected])([email protected])+ eventemitter3: 5.0.1+ optionalDependencies:+ typescript: 5.8.3+ transitivePeerDependencies:+ - zod+dependencies:'@adraffy/ens-normalize': 1.11.1@@ -14516,21 +14548,21 @@ snapshots:- [email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])):+ [email protected](@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])):dependencies:- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))hono: 4.9.10idb-keyval: 6.2.2mipd: 0.0.7([email protected])ox: 0.9.8([email protected])([email protected])zod: 4.1.11optionalDependencies:'@tanstack/react-query': 5.84.1([email protected])react: 18.3.1typescript: 5.8.3- wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])+ wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])transitivePeerDependencies:- '@types/react'- immer@@ -14813,7 +14845,7 @@ snapshots:'@types/ws': 8.18.1buffer: 6.0.3eventemitter3: 5.0.1- uuid: 9.0.1+ uuid: 8.3.2ws: 8.18.3([email protected])([email protected])optionalDependencies:bufferutil: 4.0.9@@ -15438,6 +15470,10 @@ snapshots:+ [email protected]: {}++ [email protected]: {}+@@ -15492,6 +15528,23 @@ snapshots:- utf-8-validate- zod+ dependencies:+ '@noble/curves': 1.9.2+ '@noble/hashes': 1.8.0+ '@scure/bip32': 1.7.0+ '@scure/bip39': 1.6.0+ abitype: 1.0.8([email protected])([email protected])+ ox: 0.8.6([email protected])([email protected])+ ws: 8.18.2([email protected])([email protected])+ optionalDependencies:+ typescript: 5.8.3+ transitivePeerDependencies:+ - bufferutil+ - utf-8-validate+ - zod+dependencies:'@noble/curves': 1.9.2@@ -15513,14 +15566,14 @@ snapshots:dependencies:xml-name-validator: 5.0.0- [email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]):+ [email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]):dependencies:'@tanstack/react-query': 5.84.1([email protected])- '@wagmi/connectors': 5.11.2(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])- '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))+ '@wagmi/connectors': 5.11.2(@tanstack/[email protected]([email protected]))(@wagmi/[email protected](@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected])))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected](@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected]))([email protected])+ '@wagmi/core': 2.21.2(@tanstack/[email protected])([email protected])([email protected])([email protected]([email protected]))([email protected]([email protected])([email protected])([email protected])([email protected]))react: 18.3.1use-sync-external-store: 1.4.0([email protected])optionalDependencies:typescript: 5.8.3transitivePeerDependencies:@@ -15729,8 +15782,8 @@ snapshots:'@solana-program/token-2022': 0.4.2(@solana/[email protected]([email protected])([email protected])([email protected]([email protected])([email protected])))(@solana/[email protected]([email protected])([email protected]))'@solana/kit': 2.3.0([email protected])([email protected])([email protected]([email protected])([email protected]))'@solana/transaction-confirmation': 2.3.0([email protected])([email protected])([email protected]([email protected])([email protected]))- wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])+ wagmi: 2.17.5(@tanstack/[email protected])(@tanstack/[email protected]([email protected]))([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]([email protected])([email protected])([email protected])([email protected]))([email protected])zod: 3.25.76transitivePeerDependencies:- '@azure/app-configuration'diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.tsindex a7486fad1..532b1d222 100644--- a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts+++ b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts@@ -1,6 +1,6 @@import request from "supertest";import dotenv from "dotenv";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";dotenv.config();@@ -482,7 +482,7 @@ describe("E2E Tests for API Routes", () => {it.concurrent("should prevent duplicate requests using the same idempotency key",async () => {- const uniqueIdempotencyKey = uuidv4();+ const uniqueIdempotencyKey = uuidv7();// First request with the idempotency keyconst firstResponse = await request(TEST_URL)diff --git a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts b/apps/api/src/__tests__/snips/v2/robots-txt.test.tsdeleted file mode 100644index 60582704f..000000000--- a/apps/api/src/__tests__/snips/v2/robots-txt.test.ts+++ /dev/null@@ -1,72 +0,0 @@-import { createRobotsChecker, fetchRobotsTxt } from "../../../lib/robots-txt";-import { scrapeTimeout } from "./lib";--// TODO: test self-hosted-describe("Robots.txt tests", () => {- const robotsUrl = "https://www.google.com/robots.txt";- const siteUrl = "https://www.google.com/";- let robotsTxt: string;- let checker: ReturnType<typeof createRobotsChecker>;-- beforeAll(async () => {- const logger = {- error: jest.fn(),- } as any;- robotsTxt = (- await fetchRobotsTxt(- { url: robotsUrl, zeroDataRetention: true },- "test-scrape",- logger,- )- ).content;- checker = createRobotsChecker(siteUrl, robotsTxt);- }, scrapeTimeout);-- it(- "contains expected directives",- () => {- expect(robotsTxt).toContain("User-agent");- expect(robotsTxt).toContain("Disallow");- },- scrapeTimeout,- );-- it("allows and disallows URLs as per rules", () => {- expect(- checker.robots.isAllowed(- "https://www.google.com/search?q=test",- "FireCrawlAgent",- ),- ).toBe(false);-- expect(- checker.robots.isAllowed(- "https://www.google.com/search/about",- "FireCrawlAgent",- ),- ).toBe(true);-- expect(- checker.robots.isAllowed(- "https://www.google.com/groups",- "FireCrawlAgent",- ),- ).toBe(false);-- expect(- checker.robots.isAllowed(- "https://www.google.com/preferences",- "FireCrawlAgent",- ),- ).toBe(true);- });-- it("includes Google's sitemap URLs", () => {- const sitemaps = checker.robots.getSitemaps();-- expect(Array.isArray(sitemaps)).toBe(true);- expect(sitemaps.length).toBeGreaterThan(0);-- expect(sitemaps).toContain("https://www.google.com/sitemap.xml");- });-});diff --git a/apps/api/src/controllers/__tests__/crawl.test.ts b/apps/api/src/controllers/__tests__/crawl.test.tsindex a004ee3c0..8d6b539d8 100644--- a/apps/api/src/controllers/__tests__/crawl.test.ts+++ b/apps/api/src/controllers/__tests__/crawl.test.ts@@ -3,7 +3,7 @@ import { Request, Response } from "express";import { authenticateUser } from "../auth"; // Ensure this import is correctimport { createIdempotencyKey } from "../../services/idempotency/create";import { validateIdempotencyKey } from "../../services/idempotency/validate";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";jest.mock("../auth", () => ({authenticateUser: jest.fn().mockResolvedValue({@@ -20,7 +20,7 @@ describe("crawlController", () => {it("should prevent duplicate requests using the same idempotency key", async () => {const req = {headers: {- "x-idempotency-key": await uuidv4(),+ "x-idempotency-key": await uuidv7(),Authorization: `Bearer ${process.env.TEST_API_KEY}`,},body: {diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 8c795f06f..6e21c5e97 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -11,7 +11,7 @@ import {defaultCrawlerOptions,defaultOrigin,} from "../../../src/lib/default-values";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { logger } from "../../../src/lib/logger";import {addCrawlJob,@@ -51,7 +51,7 @@ export async function crawlController(req: Request, res: Response) {});}- const id = uuidv4();+ const id = uuidv7();redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>logger.error("Failed to add team to teams_using_v0", {@@ -151,7 +151,7 @@ export async function crawlController(req: Request, res: Response) {// try {// const a = new WebScraperDataProvider();// await a.setOptions({- // jobId: uuidv4(),+ // jobId: uuidv7(),// mode: "single_urls",// urls: [url],// crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true },@@ -226,7 +226,7 @@ export async function crawlController(req: Request, res: Response) {basePriority: 21,});const jobs = urls.map(url => {- const uuid = uuidv4();+ const uuid = uuidv7();return {jobId: uuid,data: {@@ -270,7 +270,7 @@ export async function crawlController(req: Request, res: Response) {// Not needed, first one should be 15.// const jobPriority = await getJobPriority({team_id, basePriority: 10})- const jobId = uuidv4();+ const jobId = uuidv7();await addScrapeJob({url,diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 2229295b9..267054bb1 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -13,7 +13,7 @@ import {} from "../../lib/default-values";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { redisEvictConnection } from "../../../src/services/redis";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { logger } from "../../lib/logger";import * as Sentry from "@sentry/node";import { getJobPriority } from "../../lib/job-priority";@@ -176,7 +176,7 @@ export async function scrapeController(req: Request, res: Response) {});}- const jobId = uuidv4();+ const jobId = uuidv7();redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>logger.error("Failed to add team to teams_using_v0", {diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 812fdc08b..9a9722748 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -9,7 +9,7 @@ import { logJob } from "../../services/logging/log_job";import { PageOptions, SearchOptions } from "../../lib/entities";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { logger } from "../../lib/logger";import { redisEvictConnection } from "../../../src/services/redis";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";@@ -106,7 +106,7 @@ async function searchHelper(const jobDatas = res.map(x => {const url = x.url;- const uuid = uuidv4();+ const uuid = uuidv7();return {jobId: uuid,data: {@@ -175,7 +175,7 @@ export async function searchController(req: Request, res: Response) {});}- const jobId = uuidv4();+ const jobId = uuidv7();redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>logger.error("Failed to add team to teams_using_v0", {diff --git a/apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIP b/apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIPindex 621c74361..6db51ff27 100644--- a/apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIP+++ b/apps/api/src/controllers/v1/__tests__/crawl.test.ts.WIP@@ -3,7 +3,7 @@ import { Request, Response } from 'express';import { authenticateUser } from '../auth'; // Ensure this import is correctimport { createIdempotencyKey } from '../../services/idempotency/create';import { validateIdempotencyKey } from '../../services/idempotency/validate';-import { v4 as uuidv4 } from 'uuid';+import { v7 as uuidv7 } from 'uuid';jest.mock('../auth', () => ({authenticateUser: jest.fn().mockResolvedValue({@@ -20,7 +20,7 @@ describe('crawlController', () => {it('should prevent duplicate requests using the same idempotency key', async () => {const req = {headers: {- 'x-idempotency-key': await uuidv4(),+ 'x-idempotency-key': await uuidv7(),'Authorization': `Bearer ${process.env.TEST_API_KEY}`},body: {diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex a43064e8b..cf31a4d6f 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -1,5 +1,5 @@import { Response } from "express";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import {BatchScrapeRequest,batchScrapeRequestSchema,@@ -49,7 +49,7 @@ export async function batchScrapeController(const zeroDataRetention =req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;- const id = req.body.appendToId ?? uuidv4();+ const id = req.body.appendToId ?? uuidv7();const logger = _logger.child({crawlId: id,batchScrapeId: id,@@ -160,7 +160,7 @@ export async function batchScrapeController(logger.debug("Using job priority " + jobPriority, { jobPriority });const jobs = urls.map(x => ({- jobId: uuidv4(),+ jobId: uuidv7(),data: {url: x,mode: "single_urls" as const,diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 6c4c6155b..634e8ce1c 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -8,7 +8,7 @@ import {RequestWithAuth,} from "./types";import { WebSocket } from "ws";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { logger } from "../../lib/logger";import {getCrawl,@@ -190,7 +190,7 @@ export async function crawlStatusWSController(} catch (err) {Sentry.captureException(err);- const id = uuidv4();+ const id = uuidv7();let verbose = JSON.stringify(err);if (verbose === "{}") {if (err instanceof Error) {diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex bdc3190e3..6ab3407d2 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -1,5 +1,5 @@import { Response } from "express";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import {CrawlRequest,crawlRequestSchema,@@ -37,7 +37,7 @@ export async function crawlController(const zeroDataRetention =req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;- const id = uuidv4();+ const id = uuidv7();const logger = _logger.child({crawlId: id,module: "api/v1",diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex 682b95de3..086d6f655 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -1,3 +1,4 @@+import { v7 as uuidv7 } from "uuid";import { Request, Response } from "express";import { ErrorResponse, extractOptions, RequestWithAuth } from "./types";import { getDeepResearchQueue } from "../../services/queue-service";@@ -89,7 +90,7 @@ export async function deepResearchController(req.body = deepResearchRequestSchema.parse(req.body);- const researchId = crypto.randomUUID();+ const researchId = uuidv7();const jobData = {request: req.body,teamId: req.auth.team_id,diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 09218c158..fb4ba20e1 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -1,3 +1,4 @@+import { v7 as uuidv7 } from "uuid";import { Request, Response } from "express";import {RequestWithAuth,@@ -121,7 +122,7 @@ export async function extractController(}}- const extractId = crypto.randomUUID();+ const extractId = uuidv7();_logger.info("Extract starting...", {request: req.body,diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex 2e6464e73..d9974b92d 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -1,3 +1,4 @@+import { v7 as uuidv7 } from "uuid";import { Response } from "express";import {ErrorResponse,@@ -36,7 +37,7 @@ export async function generateLLMsTextController(req.body = generateLLMsTextRequestSchema.parse(req.body);- const generationId = crypto.randomUUID();+ const generationId = uuidv7();const jobData = {request: req.body,teamId: req.auth.team_id,diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 3d3908884..9dbf0f93f 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -1,5 +1,5 @@import { Response } from "express";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import {MapDocument,mapRequestSchema,@@ -115,7 +115,7 @@ export async function getMapResults({timeout?: number;location?: ScrapeOptions["location"];}): Promise<MapResult> {- const id = uuidv4();+ const id = uuidv7();let links: string[] = [url];let mapResults: MapDocument[] = [];diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 4bfc7cc17..ca3a18af2 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -7,7 +7,7 @@ import {scrapeRequestSchema,ScrapeResponse,} from "./types";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { getJobPriority } from "../../lib/job-priority";import { fromV1ScrapeOptions } from "../v2/types";import { TransportableError } from "../../lib/error";@@ -27,7 +27,7 @@ export async function scrapeController((req as any).requestTiming?.startTime || new Date().getTime();const controllerStartTime = new Date().getTime();- const jobId: string = uuidv4();+ const jobId: string = uuidv7();const preNormalizedBody = { ...req.body };req.body = scrapeRequestSchema.parse(req.body);diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex d25cc6603..1ddf63799 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -9,7 +9,7 @@ import {TeamFlags,} from "./types";import { billTeam } from "../../services/billing/credit_billing";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";import { search } from "../../search";@@ -86,7 +86,7 @@ async function scrapeSearchResult(directToBullMQ: boolean = false,isSearchPreview: boolean = false,): Promise<DocumentWithCostTracking> {- const jobId = uuidv4();+ const jobId = uuidv7();const costTracking = new CostTracking();@@ -229,7 +229,7 @@ export async function searchController((req as any).requestTiming?.startTime || new Date().getTime();const controllerStartTime = new Date().getTime();- const jobId = uuidv4();+ const jobId = uuidv7();let logger = _logger.child({jobId,teamId: req.auth.team_id,diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex fc90e8f97..88b218fd0 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -8,7 +8,7 @@ import {ScrapeOptions,TeamFlags,} from "./types";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";import { search } from "../../search";@@ -43,7 +43,7 @@ async function scrapeX402SearchResult(directToBullMQ: boolean = false,isSearchPreview: boolean = false,): Promise<DocumentWithCostTracking> {- const jobId = uuidv4();+ const jobId = uuidv7();const costTracking = new CostTracking();@@ -174,7 +174,7 @@ export async function x402SearchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,res: Response<SearchResponse & { request?: any }>,) {- const jobId = uuidv4();+ const jobId = uuidv7();let logger = _logger.child({jobId,teamId: req.auth.team_id,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 4e3dafb4f..6ad641ab2 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -1,5 +1,5 @@import { Response } from "express";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import {BatchScrapeRequest,batchScrapeRequestSchema,@@ -49,7 +49,7 @@ export async function batchScrapeController(const zeroDataRetention =req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;- const id = req.body.appendToId ?? uuidv4();+ const id = req.body.appendToId ?? uuidv7();const logger = _logger.child({crawlId: id,batchScrapeId: id,@@ -157,7 +157,7 @@ export async function batchScrapeController(delete (scrapeOptions as any).appendToId;const jobs = urls.map(x => ({- jobId: uuidv4(),+ jobId: uuidv7(),data: {url: x,mode: "single_urls" as const,diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex 91c1dd809..b139907ad 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -8,7 +8,7 @@ import {RequestWithAuth,} from "./types";import { WebSocket } from "ws";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { logger } from "../../lib/logger";import {getCrawl,@@ -193,7 +193,7 @@ export async function crawlStatusWSController(} catch (err) {Sentry.captureException(err);- const id = uuidv4();+ const id = uuidv7();let verbose = JSON.stringify(err);if (verbose === "{}") {if (err instanceof Error) {diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 1652a0aa7..5beba6eea 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -1,5 +1,5 @@import { Response } from "express";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import {CrawlRequest,crawlRequestSchema,@@ -44,7 +44,7 @@ export async function crawlController(const zeroDataRetention =req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;- const id = uuidv4();+ const id = uuidv7();const logger = _logger.child({crawlId: id,module: "api/v2",diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 08dce3a62..740f952f6 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -1,3 +1,4 @@+import { v7 as uuidv7 } from "uuid";import { Response } from "express";import {RequestWithAuth,@@ -47,7 +48,7 @@ export async function extractController(}}- const extractId = crypto.randomUUID();+ const extractId = uuidv7();const createdAt = Date.now();_logger.info("Extract starting...", {request: req.body,diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 652401cd3..be2f78424 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -13,7 +13,7 @@ import { logger as _logger } from "../../lib/logger";import { MapTimeoutError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";import { getMapResults, MapResult } from "../../lib/map-utils";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";configDotenv();@@ -23,7 +23,7 @@ export async function mapController(res: Response<MapResponse>,) {const logger = _logger.child({- jobId: uuidv4(),+ jobId: uuidv7(),teamId: req.auth.team_id,module: "api/v2",method: "mapController",diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex b009b59f2..de66d59b2 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -8,7 +8,7 @@ import {scrapeRequestSchema,ScrapeResponse,} from "./types";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { hasFormatOfType } from "../../lib/format-utils";import { TransportableError } from "../../lib/error";import { NuQJob } from "../../services/worker/nuq";@@ -31,7 +31,7 @@ export async function scrapeController((req as any).requestTiming?.startTime || new Date().getTime();const controllerStartTime = new Date().getTime();- const jobId = uuidv4();+ const jobId = uuidv7();const preNormalizedBody = { ...req.body };// Set initial span attributesdiff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex c399427f2..c24c697fb 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -9,7 +9,7 @@ import {TeamFlags,} from "./types";import { billTeam } from "../../services/billing/credit_billing";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";import { search } from "../../search/v2";@@ -57,7 +57,7 @@ async function startScrapeJob(directToBullMQ: boolean = false,isSearchPreview: boolean = false,): Promise<string> {- const jobId = uuidv4();+ const jobId = uuidv7();const zeroDataRetention = flags?.forceZDR ?? false;@@ -206,7 +206,7 @@ export async function searchController((req as any).requestTiming?.startTime || new Date().getTime();const controllerStartTime = new Date().getTime();- const jobId = uuidv4();+ const jobId = uuidv7();let logger = _logger.child({jobId,teamId: req.auth.team_id,diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex 7dd76719b..64b773c05 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -8,7 +8,7 @@ import {ScrapeOptions,TeamFlags,} from "./types";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";import { search } from "../../search/v2";@@ -55,7 +55,7 @@ async function startX420ScrapeJob(directToBullMQ: boolean = false,isSearchPreview: boolean = false,): Promise<string> {- const jobId = uuidv4();+ const jobId = uuidv7();const zeroDataRetention = flags?.forceZDR ?? false;@@ -199,7 +199,7 @@ export async function x402SearchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,res: Response<SearchResponse>,) {- const jobId = uuidv4();+ const jobId = uuidv7();let logger = _logger.child({jobId,teamId: req.auth.team_id,diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex c5217ceeb..c38d64e2e 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -26,7 +26,7 @@ import {ResponseWithSentry,} from "./controllers/v1/types";import { ZodError } from "zod";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { attachWsProxy } from "./services/agentLivecastWS";import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";import { v2Router } from "./routes/v2";@@ -214,7 +214,7 @@ app.use(});}- const id = res.sentry ?? uuidv4();+ const id = res.sentry ?? uuidv7();logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- ",diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex ffc99e567..72937a520 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -1,3 +1,4 @@+import { v7 as uuidv7 } from "uuid";import {Document,ScrapeOptions,@@ -39,7 +40,7 @@ export async function scrapeDocument(}async function attemptScrape(timeout: number) {- const jobId = crypto.randomUUID();+ const jobId = uuidv7();const jobPriority = await getJobPriority({team_id: options.teamId,basePriority: 10,diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex 7a23c5737..22f257e82 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -1,3 +1,4 @@+import { v7 as uuidv7 } from "uuid";import {Document,ScrapeOptions,@@ -39,7 +40,7 @@ export async function scrapeDocument_F0(}async function attemptScrape(timeout: number) {- const jobId = crypto.randomUUID();+ const jobId = uuidv7();const jobPriority = await getJobPriority({team_id: options.teamId,basePriority: 10,diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 4c24a80fa..5f479a0e3 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -1,4 +1,4 @@-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import {TeamFlags,MapDocument,@@ -122,7 +122,7 @@ export async function getMapResults({url = urlObj.toString();}- const id = uuidv4();+ const id = uuidv7();let mapResults: MapDocument[] = [];const zeroDataRetention = flags?.forceZDR ?? false;diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex fa0b7989b..c27d76230 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -6,7 +6,7 @@ import { getExtractQueue, getRedisConnection } from "./queue-service";import { Job, Queue, Worker } from "bullmq";import { logger as _logger } from "../lib/logger";import systemMonitor from "./system-monitor";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { configDotenv } from "dotenv";import {ExtractResult,@@ -204,7 +204,7 @@ const workerFun = async (_logger.info("No longer accepting new jobs. SIGINT");break;}- const token = uuidv4();+ const token = uuidv7();const canAcceptConnection = await monitor.acceptConnection();if (!canAcceptConnection) {console.log("Can't accept connection due to RAM/CPU load");diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex fa5711778..f107bd444 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -15,7 +15,7 @@ import {startBillingBatchProcessing,} from "../billing/batch_billing";import systemMonitor from "../system-monitor";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import {index_supabase_service,processIndexInsertJobs,@@ -475,7 +475,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {zeroDataRetention: false,};- const crawlId = uuidv4();+ const crawlId = uuidv7();// robots disabled for now// const crawler = crawlToCrawler(crawlId, sc, null);@@ -597,7 +597,7 @@ const workerFun = async (break;}- const token = uuidv4();+ const token = uuidv7();const canAcceptConnection = await monitor.acceptConnection();if (!canAcceptConnection) {diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 6a009c15b..fe0c62ace 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,4 +1,4 @@-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { NotificationType, RateLimiterMode, ScrapeJobData } from "../types";import {cleanOldConcurrencyLimitEntries,@@ -316,7 +316,7 @@ async function addScrapeJobRaw(export async function addScrapeJob(webScraperOptions: ScrapeJobData,- jobId: string = uuidv4(),+ jobId: string = uuidv7(),priority: number = 0,directToBullMQ: boolean = false,listenable: boolean = false,diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex c1a7cb076..ef033b6a7 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -10,7 +10,7 @@ import {import { Job, Queue, Worker } from "bullmq";import { logger as _logger } from "../lib/logger";import systemMonitor from "./system-monitor";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import { configDotenv } from "dotenv";import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";import { performDeepResearch } from "../lib/deep-research/deep-research-service";@@ -263,7 +263,7 @@ const workerFun = async (_logger.info("No longer accepting new jobs. SIGINT");break;}- const token = uuidv4();+ const token = uuidv7();const canAcceptConnection = await monitor.acceptConnection();if (!canAcceptConnection) {console.log("Can't accept connection due to RAM/CPU load");diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 1e5af10e8..35bc26ddb 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -10,7 +10,7 @@ import {} from "../../lib/concurrency-limit";import { addJobPriority, deleteJobPriority } from "../../lib/job-priority";import { cacheableLookup } from "../../scraper/scrapeURL/lib/cacheableLookup";-import { v4 as uuidv4 } from "uuid";+import { v7 as uuidv7 } from "uuid";import {addCrawlJob,addCrawlJobs,@@ -110,7 +110,7 @@ async function billScrapeJob(process.env.USE_DB_AUTHENTICATION === "true") {try {- const billingJobId = uuidv4();+ const billingJobId = uuidv7();logger.debug(`Adding billing job to queue for team ${job.data.team_id}`,{@@ -372,7 +372,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {team_id: sc.team_id,basePriority: job.data.crawl_id ? 20 : 10,});- const jobId = uuidv4();+ const jobId = uuidv7();logger.debug("Determined job priority " +@@ -779,7 +779,7 @@ async function addKickoffSitemapJob(return;}- const jobId = uuidv4();+ const jobId = uuidv7();await _addScrapeJobToBullMQ({mode: "kickoff_sitemap" as const,@@ -828,7 +828,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {logger.debug("Locking URL...");await lockURL(job.data.crawl_id, sc, job.data.url);- const jobId = uuidv4();+ const jobId = uuidv7();logger.debug("Adding scrape job to Redis...", { jobId });await addScrapeJob({@@ -914,7 +914,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {logger.debug("Using job priority " + jobPriority, { jobPriority });const jobs = indexLinks.map(url => {- const uuid = uuidv4();+ const uuid = uuidv7();return {jobId: uuid,data: {@@ -1040,7 +1040,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {job.data.zeroDataRetention || (sc.zeroDataRetention ?? false),apiKeyId: job.data.apiKeyId,} satisfies ScrapeJobSingleUrls,- jobId: uuidv4(),+ jobId: uuidv7(),priority: jobPriority,}));