Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(v0/crawl): force kickoff

Móricz Gergőae0d705f5d3d
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 675841789..5b19a0a8e 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -18,6 +18,7 @@ import {
addCrawlJob,
addCrawlJobs,
crawlToCrawler,
+ finishCrawlKickoff,
lockURL,
lockURLs,
saveCrawl,
@@ -177,6 +178,8 @@ export async function crawlController(req: Request, res: Response) {
await saveCrawl(id, sc);
+ await finishCrawlKickoff(id);
+
const sitemap = sc.crawlerOptions.ignoreSitemap
? 0
: await crawler.tryGetSitemap(async (urls) => {
@@ -224,7 +227,7 @@ export async function crawlController(req: Request, res: Response) {
await addScrapeJob(job.data as any, {}, job.opts.jobId);
}
});
-
+
if (sitemap === 0) {
await lockURL(id, sc, url);
diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts
index 3f1a9dd9e..37c84cdc4 100644
--- a/apps/api/src/controllers/v0/crawlPreview.ts
+++ b/apps/api/src/controllers/v0/crawlPreview.ts
@@ -7,6 +7,7 @@ import { logger } from "../../../src/lib/logger";
import {
addCrawlJob,
crawlToCrawler,
+ finishCrawlKickoff,
lockURL,
saveCrawl,
StoredCrawl,
@@ -112,6 +113,8 @@ export async function crawlPreviewController(req: Request, res: Response) {
const crawler = crawlToCrawler(id, sc);
+ await finishCrawlKickoff(id);
+
const sitemap = sc.crawlerOptions?.ignoreSitemap
? 0
: await crawler.tryGetSitemap(async (urls) => {