Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: add crawlTtlHours team flag to replace teamIdsExcludedFromExpiry (#1899)

devin-ai-integration[bot]cf5eea0ed469
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index de5eb83ba..5f115d18e 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -294,15 +294,12 @@ export async function crawlStatusController(
return res.status(403).json({ success: false, error: "Forbidden" });
}
- const teamIdsExcludedFromExpiry = [
- "8f819703-1b85-4f7f-a6eb-e03841ec6617",
- "f96ad1a4-8102-4b35-9904-36fd517d3616",
- ];
+ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;
if (
crawlJob
- && !teamIdsExcludedFromExpiry.includes(crawlJob.team_id)
- && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > 24 * 60 * 60 * 1000
+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs
) {
return res.status(404).json({ success: false, error: "Job expired" });
}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index cdf08a345..3945a5a45 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1110,6 +1110,7 @@ export type TeamFlags = {
zdrCost?: number;
checkRobotsOnScrape?: boolean;
allowTeammateInvites?: boolean;
+ crawlTtlHours?: number;
} | null;
export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;