Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): forceZDR behaviour
Gergő Móricz8bc1c0703a53
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 2a8cbed31..42930369e 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -122,7 +122,7 @@ export async function getMapResults({let links: string[] = [url];let mapResults: MapDocument[] = [];- const zeroDataRetention = flags?.forceZDR ?? false;+ const zeroDataRetention = flags?.forceZDR || false;const sc: StoredCrawl = {originUrl: url,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex d58b28de4..899387e62 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -93,7 +93,7 @@ async function scrapeSearchResult(const costTracking = new CostTracking();- const zeroDataRetention = flags?.forceZDR ?? false;+ const zeroDataRetention = flags?.forceZDR || false;try {if (isUrlBlocked(searchResult.url, flags)) {diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex a62f0c3af..131d0e27f 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -48,7 +48,7 @@ async function scrapeX402SearchResult(const costTracking = new CostTracking();- const zeroDataRetention = flags?.forceZDR ?? false;+ const zeroDataRetention = flags?.forceZDR || false;try {if (isUrlBlocked(searchResult.url, flags)) {diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex b0d22723d..b82240f0b 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -49,7 +49,7 @@ export async function batchScrapeController(}const zeroDataRetention =- req.acuc?.flags?.forceZDR ?? req.body.zeroDataRetention ?? false;+ req.acuc?.flags?.forceZDR || (req.body.zeroDataRetention ?? false);const id = req.body.appendToId ?? uuidv7();const logger = _logger.child({diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 5a5244c42..696d00371 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -78,7 +78,7 @@ export async function scrapeController(}const zeroDataRetention =- req.acuc?.flags?.forceZDR ?? req.body.zeroDataRetention ?? false;+ req.acuc?.flags?.forceZDR || (req.body.zeroDataRetention ?? false);const logger = _logger.child({method: "scrapeController",diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex a61d696d0..5dc6d07dc 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -63,7 +63,7 @@ async function startScrapeJob(const jobId = uuidv7();const zeroDataRetention =- options.zeroDataRetention ?? flags?.forceZDR ?? false;+ flags?.forceZDR || (options.zeroDataRetention ?? false);logger.info("Adding scrape job", {scrapeId: jobId,diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex bc4a92452..89f5265b1 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -58,7 +58,7 @@ async function startX420ScrapeJob(): Promise<string> {const jobId = uuidv7();- const zeroDataRetention = flags?.forceZDR ?? false;+ const zeroDataRetention = flags?.forceZDR || false;logger.info("Adding scrape job [x402]", {scrapeId: jobId,diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex a66d09c06..00757fdcc 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -126,7 +126,7 @@ export async function getMapResults({const id = providedId ?? uuidv7();let mapResults: MapDocument[] = [];- const zeroDataRetention = flags?.forceZDR ?? false;+ const zeroDataRetention = flags?.forceZDR || false;const sc: StoredCrawl = {originUrl: url,