Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): only allow appending to batch scrapes (#5648)

tomsideguide7c8d58921c65
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index b637819a4..838a1e9b7 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -115,6 +115,23 @@ export async function batchScrapeController(
req.body.zeroDataRetention ||
(safeMode.safeMode?.lockdown ?? false);
+ // Only batch scrapes can be appended to: crawls (including monitor crawls)
+ // carry their own billing and scope.
+ const appendTarget = req.body.appendToId
+ ? ((await getCrawl(req.body.appendToId)) as StoredCrawl | null)
+ : null;
+ if (
+ req.body.appendToId &&
+ (!appendTarget ||
+ appendTarget.team_id !== req.auth.team_id ||
+ appendTarget.crawlerOptions != null)
+ ) {
+ return res.status(404).json({
+ success: false,
+ error: "Job not found",
+ });
+ }
+
const id = req.body.appendToId ?? uuidv7();
const logger = _logger.child({
crawlId: id,
@@ -336,8 +353,8 @@ export async function batchScrapeController(
scrapeOptions.maxAge = undefined;
}
- const sc: StoredCrawl = req.body.appendToId
- ? ((await getCrawl(req.body.appendToId)) as StoredCrawl)
+ const sc: StoredCrawl = appendTarget
+ ? appendTarget
: {
crawlerOptions: null,
scrapeOptions,
@@ -365,13 +382,8 @@ export async function batchScrapeController(
origin: req.body.origin,
};
- if (req.body.appendToId && (!sc || sc.team_id !== req.auth.team_id)) {
- return res.status(404).json({
- success: false,
- error: "Job not found",
- });
- }
- if (req.body.appendToId && sc?.internalOptions) {
+ if (req.body.appendToId && sc.internalOptions) {
+ sc.internalOptions.bypassBilling = false;
// Refresh Safe Mode + threat-protection so appended jobs enforce the team's
// current policy, not whatever was stored when the batch was created.
sc.internalOptions.teamFlags = req.acuc?.flags ?? undefined;
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 285ac3fea..0159a14fe 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -144,6 +144,23 @@ export async function batchScrapeController(
});
}
+ // Only batch scrapes can be appended to: crawls (including monitor crawls)
+ // carry their own billing and scope.
+ const appendTarget = req.body.appendToId
+ ? ((await getCrawl(req.body.appendToId)) as StoredCrawl | null)
+ : null;
+ if (
+ req.body.appendToId &&
+ (!appendTarget ||
+ appendTarget.team_id !== req.auth.team_id ||
+ appendTarget.crawlerOptions != null)
+ ) {
+ return res.status(404).json({
+ success: false,
+ error: "Job not found",
+ });
+ }
+
const id = req.body.appendToId ?? uuidv7();
const billing: BillingMetadata = {
...(req.body.__agentInterop
@@ -425,8 +442,8 @@ export async function batchScrapeController(
});
}
- const sc: StoredCrawl = req.body.appendToId
- ? ((await getCrawl(req.body.appendToId)) as StoredCrawl)
+ const sc: StoredCrawl = appendTarget
+ ? appendTarget
: {
crawlerOptions: null,
scrapeOptions: req.body,
@@ -457,12 +474,9 @@ export async function batchScrapeController(
};
if (req.body.appendToId) {
- if (!sc || sc.team_id !== req.auth.team_id) {
- return res.status(404).json({
- success: false,
- error: "Job not found",
- });
- }
+ sc.internalOptions.bypassBilling = !(
+ req.body.__agentInterop?.shouldBill ?? true
+ );
// Refresh Safe Mode + threat-protection context so appended jobs enforce
// the team's current policy, not whatever was stored when the batch was
// first created.