Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(api): remove one-use crawl migration endpoint
Gergő Móricz880d75158b5b
diff --git a/apps/api/src/controllers/v0/admin/crawl-migration.ts b/apps/api/src/controllers/v0/admin/crawl-migration.tsdeleted file mode 100644index f30f8eb4a..000000000--- a/apps/api/src/controllers/v0/admin/crawl-migration.ts+++ /dev/null@@ -1,118 +0,0 @@-import { Request, Response } from "express";-import { redisEvictConnection } from "../../../services/redis";-import { getCrawl } from "../../../lib/crawl-redis";-import { supabase_service } from "../../../services/supabase";-import { logger as _logger } from "../../../lib/logger";--type MigrationResult = {- id: string;- success: boolean;- error?: string;- skipped?: boolean;-};--export async function crawlMigrationController(req: Request, res: Response) {- const logger = _logger.child({- module: "admin",- method: "crawlMigrationController",- });-- logger.info("Starting crawl migration...");-- // Get all active crawl IDs from Redis- const activeCrawlIds = await redisEvictConnection.smembers("active_crawls");-- logger.info(`Found ${activeCrawlIds.length} active crawls to migrate`);-- const results: MigrationResult[] = [];-- for (const crawlId of activeCrawlIds) {- try {- // Check if requests entry already exists- const { data: existingRequest } = await supabase_service- .from("requests")- .select("id")- .eq("id", crawlId)- .single();-- if (existingRequest) {- logger.debug(`Crawl ${crawlId} already has a requests entry, skipping`);- results.push({ id: crawlId, success: true, skipped: true });- continue;- }-- // Fetch crawl data from Redis- const crawl = await getCrawl(crawlId);-- if (!crawl) {- logger.warn(`Crawl ${crawlId} not found in Redis, skipping`);- results.push({- id: crawlId,- success: false,- error: "Crawl not found in Redis",- });- continue;- }-- // Determine kind: 'crawl' if crawlerOptions !== null, else 'batch_scrape'- const kind = crawl.crawlerOptions !== null ? "crawl" : "batch_scrape";-- // Insert into requests table- const { error: insertError } = await supabase_service- .from("requests")- .insert({- id: crawlId,- kind: kind,- api_version: "legacy",- team_id:- crawl.team_id === "preview" || crawl.team_id?.startsWith("preview_")- ? null- : crawl.team_id,- origin: "migration",- integration: null,- target_hint: crawl.zeroDataRetention- ? "<redacted due to zero data retention>"- : (crawl.originUrl ?? "<unknown>"),- dr_clean_by: crawl.zeroDataRetention- ? new Date(Date.now() + 24 * 60 * 60 * 1000)- : null,- });-- if (insertError) {- logger.error(`Failed to migrate crawl ${crawlId}`, {- error: insertError,- });- results.push({- id: crawlId,- success: false,- error: insertError.message,- });- } else {- logger.info(`Successfully migrated crawl ${crawlId}`, { kind });- results.push({ id: crawlId, success: true });- }- } catch (error) {- logger.error(`Error migrating crawl ${crawlId}`, { error });- results.push({- id: crawlId,- success: false,- error: error instanceof Error ? error.message : String(error),- });- }- }-- const migrated = results.filter(r => r.success && !r.skipped).length;- const skipped = results.filter(r => r.skipped).length;- const failed = results.filter(r => !r.success).length;-- logger.info("Crawl migration complete", { migrated, skipped, failed });-- res.status(200).json({- success: true,- total: activeCrawlIds.length,- migrated,- skipped,- failed,- results,- });-}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex f4f8f9b7d..723d35ec3 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -15,7 +15,6 @@ import { realtimeSearchController } from "../controllers/v2/f-search";import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";import { integCreateUserController } from "../controllers/v0/admin/create-user";import { integValidateApiKeyController } from "../controllers/v0/admin/validate-api-key";-import { crawlMigrationController } from "../controllers/v0/admin/crawl-migration";export const adminRouter = express.Router();@@ -83,8 +82,3 @@ adminRouter.post(`/admin/integration/validate-api-key`,wrap(integValidateApiKeyController),);--adminRouter.post(- `/admin/${process.env.BULL_AUTH_KEY}/crawl-migration`,- wrap(crawlMigrationController),-);