Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): remove temporary migration thing
Gergő Móriczd4f051d51a30
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-migration.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-migration.tsdeleted file mode 100644index 65827d4c1..000000000--- a/apps/api/src/controllers/v0/admin/concurrency-queue-migration.ts+++ /dev/null@@ -1,88 +0,0 @@-import type { Request, Response } from "express";-import { logger as _logger } from "../../../lib/logger";-import { redisEvictConnection } from "../../../services/redis";-import { crawlGroup, scrapeQueue } from "../../../services/worker/nuq";-import { getCrawl } from "../../../lib/crawl-redis";--export async function migrateConcurrencyQueue(_: Request, res: Response) {- const logger = _logger.child({- module: "admin",- method: "migrateConcurrencyQueue"- });-- let crawlsCursor = "0";-- do {- const crawlsScan = await redisEvictConnection.sscan(- "ongoing_crawls",- crawlsCursor- );- crawlsCursor = crawlsScan[0];-- for (const crawlId of crawlsScan[1]) {- const crawlData = (await getCrawl(crawlId)) ?? { maxConcurrency: undefined }-- logger.info("Migrating crawl", { crawlId });-- await crawlGroup.addGroup(crawlId, [- {- queue: scrapeQueue,- maxConcurrency: crawlData.maxConcurrency ?? undefined,- },- ]);- }-- } while (crawlsCursor !== "0");-- let queuesCursor = "0";-- do {- const queuesScan = await redisEvictConnection.sscan(- "concurrency-limit-queues",- queuesCursor,- );- queuesCursor = queuesScan[0];-- for (const queueKey of queuesScan[1]) {- if (queueKey.startsWith("concurrency-limit-queue:preview_")) {- logger.warn("Skipping preview queue", { queueKey });- continue;- }-- let queueCursor = "0";-- logger.info("Migrating a queue", { queueKey });-- do {- const queueScan = await redisEvictConnection.zscan(- queueKey,- queueCursor,- );- queueCursor = queueScan[0];-- for (let i = 0; i < queueScan[1].length; i += 2) {- const jobData = JSON.parse(queueScan[1][i]);- const jobScore = queueScan[1][i + 1];-- logger.info("Migrating job", { queueKey, teamId: jobData?.data?.teamId, zeroDataRetention: jobData?.data?.zeroDataRetention, scrapeId: jobData?.id });-- const success = await scrapeQueue.tryAddJob(jobData.id, jobData.data, {- priority: jobData.priority,- listenable: jobData.listenable,- ownerId: jobData.data.team_id ?? undefined,- groupId: jobData.data.crawl_id ?? undefined,- timesOutAt: jobScore === "inf" ? undefined : new Date(parseInt(jobScore, 10)),- });-- if (success === null) {- logger.warn("Failed to migrate job due to conflict", { queueKey, teamId: jobData?.data?.teamId, zeroDataRetention: jobData?.data?.zeroDataRetention, scrapeId: jobData?.id });- }- }- } while (queueCursor !== "0");- }- } while (queuesCursor !== "0");-- logger.info("Migration complete! 🎉");-- res.json({ ok: true });-}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 3f2ae85e5..205b3aae3 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -12,7 +12,6 @@ import {} from "../controllers/v0/admin/metrics";import { crawlCheckController } from "../controllers/v0/admin/crawl-check";import { realtimeSearchController } from "../controllers/v2/f-search";-import { migrateConcurrencyQueue } from "../controllers/v0/admin/concurrency-queue-migration";export const adminRouter = express.Router();@@ -65,8 +64,3 @@ adminRouter.post(`/admin/${process.env.BULL_AUTH_KEY}/fsearch`,wrap(realtimeSearchController),);--adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/migrate-concurrency-queue`,- wrap(migrateConcurrencyQueue),-);