Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): only allow appending to batch scrapes (#5648)
tomsideguide7c8d58921c65
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex b637819a4..838a1e9b7 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -115,6 +115,23 @@ export async function batchScrapeController(req.body.zeroDataRetention ||(safeMode.safeMode?.lockdown ?? false);+ // Only batch scrapes can be appended to: crawls (including monitor crawls)+ // carry their own billing and scope.+ const appendTarget = req.body.appendToId+ ? ((await getCrawl(req.body.appendToId)) as StoredCrawl | null)+ : null;+ if (+ req.body.appendToId &&+ (!appendTarget ||+ appendTarget.team_id !== req.auth.team_id ||+ appendTarget.crawlerOptions != null)+ ) {+ return res.status(404).json({+ success: false,+ error: "Job not found",+ });+ }+const id = req.body.appendToId ?? uuidv7();const logger = _logger.child({crawlId: id,@@ -336,8 +353,8 @@ export async function batchScrapeController(scrapeOptions.maxAge = undefined;}- const sc: StoredCrawl = req.body.appendToId- ? ((await getCrawl(req.body.appendToId)) as StoredCrawl)+ const sc: StoredCrawl = appendTarget+ ? appendTarget: {crawlerOptions: null,scrapeOptions,@@ -365,13 +382,8 @@ export async function batchScrapeController(origin: req.body.origin,};- if (req.body.appendToId && (!sc || sc.team_id !== req.auth.team_id)) {- return res.status(404).json({- success: false,- error: "Job not found",- });- }- if (req.body.appendToId && sc?.internalOptions) {+ if (req.body.appendToId && sc.internalOptions) {+ sc.internalOptions.bypassBilling = false;// Refresh Safe Mode + threat-protection so appended jobs enforce the team's// current policy, not whatever was stored when the batch was created.sc.internalOptions.teamFlags = req.acuc?.flags ?? undefined;diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 285ac3fea..0159a14fe 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -144,6 +144,23 @@ export async function batchScrapeController(});}+ // Only batch scrapes can be appended to: crawls (including monitor crawls)+ // carry their own billing and scope.+ const appendTarget = req.body.appendToId+ ? ((await getCrawl(req.body.appendToId)) as StoredCrawl | null)+ : null;+ if (+ req.body.appendToId &&+ (!appendTarget ||+ appendTarget.team_id !== req.auth.team_id ||+ appendTarget.crawlerOptions != null)+ ) {+ return res.status(404).json({+ success: false,+ error: "Job not found",+ });+ }+const id = req.body.appendToId ?? uuidv7();const billing: BillingMetadata = {...(req.body.__agentInterop@@ -425,8 +442,8 @@ export async function batchScrapeController(});}- const sc: StoredCrawl = req.body.appendToId- ? ((await getCrawl(req.body.appendToId)) as StoredCrawl)+ const sc: StoredCrawl = appendTarget+ ? appendTarget: {crawlerOptions: null,scrapeOptions: req.body,@@ -457,12 +474,9 @@ export async function batchScrapeController(};if (req.body.appendToId) {- if (!sc || sc.team_id !== req.auth.team_id) {- return res.status(404).json({- success: false,- error: "Job not found",- });- }+ sc.internalOptions.bypassBilling = !(+ req.body.__agentInterop?.shouldBill ?? true+ );// Refresh Safe Mode + threat-protection context so appended jobs enforce// the team's current policy, not whatever was stored when the batch was// first created.