Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(v0/crawl): force kickoff
Móricz Gergőae0d705f5d3d
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 675841789..5b19a0a8e 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -18,6 +18,7 @@ import {addCrawlJob,addCrawlJobs,crawlToCrawler,+ finishCrawlKickoff,lockURL,lockURLs,saveCrawl,@@ -177,6 +178,8 @@ export async function crawlController(req: Request, res: Response) {await saveCrawl(id, sc);+ await finishCrawlKickoff(id);+const sitemap = sc.crawlerOptions.ignoreSitemap? 0: await crawler.tryGetSitemap(async (urls) => {@@ -224,7 +227,7 @@ export async function crawlController(req: Request, res: Response) {await addScrapeJob(job.data as any, {}, job.opts.jobId);}});-+if (sitemap === 0) {await lockURL(id, sc, url);diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.tsindex 3f1a9dd9e..37c84cdc4 100644--- a/apps/api/src/controllers/v0/crawlPreview.ts+++ b/apps/api/src/controllers/v0/crawlPreview.ts@@ -7,6 +7,7 @@ import { logger } from "../../../src/lib/logger";import {addCrawlJob,crawlToCrawler,+ finishCrawlKickoff,lockURL,saveCrawl,StoredCrawl,@@ -112,6 +113,8 @@ export async function crawlPreviewController(req: Request, res: Response) {const crawler = crawlToCrawler(id, sc);+ await finishCrawlKickoff(id);+const sitemap = sc.crawlerOptions?.ignoreSitemap? 0: await crawler.tryGetSitemap(async (urls) => {