Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add crawlTtlHours team flag to replace teamIdsExcludedFromExpiry (#1899)
devin-ai-integration[bot]cf5eea0ed469
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex de5eb83ba..5f115d18e 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -294,15 +294,12 @@ export async function crawlStatusController(return res.status(403).json({ success: false, error: "Forbidden" });}- const teamIdsExcludedFromExpiry = [- "8f819703-1b85-4f7f-a6eb-e03841ec6617",- "f96ad1a4-8102-4b35-9904-36fd517d3616",- ];+ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;if (crawlJob- && !teamIdsExcludedFromExpiry.includes(crawlJob.team_id)- && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > 24 * 60 * 60 * 1000+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs) {return res.status(404).json({ success: false, error: "Job expired" });}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex cdf08a345..3945a5a45 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1110,6 +1110,7 @@ export type TeamFlags = {zdrCost?: number;checkRobotsOnScrape?: boolean;allowTeammateInvites?: boolean;+ crawlTtlHours?: number;} | null;export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;