Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl): only extend URLs at pre-finish if changeTracking is on (#1837)

Gergő Móricz3ecbc2f03527
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index f900f708d..73f334e76 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -149,7 +149,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
let lastUrls: string[] = [];
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
- if (useDbAuthentication) {
+ if (useDbAuthentication && sc.scrapeOptions.formats.includes("changeTracking")) {
lastUrls = (
(
await supabase_service.rpc("diff_get_last_crawl_urls", {