Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
FIR-2006: Fix maxUrls and timeLimit parameters in Deep Research API (#1569)
devin-ai-integration[bot]9949403b5974
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex 8b32c6bca..7c87d6542 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -276,3 +276,64 @@ export async function tokenUsage(): Promise<{ remaining_tokens: number }> {.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`).set("Content-Type", "application/json")).body.data;}++// =========================================+// =========================================++async function deepResearchStart(body: {+ query?: string;+ maxDepth?: number;+ maxUrls?: number;+ timeLimit?: number;+ analysisPrompt?: string;+ systemPrompt?: string;+ formats?: string[];+ topic?: string;+ jsonOptions?: any;+}) {+ return await request(TEST_URL)+ .post("/v1/deep-research")+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Content-Type", "application/json")+ .send(body);+}++async function deepResearchStatus(id: string) {+ return await request(TEST_URL)+ .get("/v1/deep-research/" + encodeURIComponent(id))+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .send();+}++function expectDeepResearchStartToSucceed(response: Awaited<ReturnType<typeof deepResearchStart>>) {+ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(typeof response.body.id).toBe("string");+}++export async function deepResearch(body: {+ query?: string;+ maxDepth?: number;+ maxUrls?: number;+ timeLimit?: number;+ analysisPrompt?: string;+ systemPrompt?: string;+ formats?: string[];+ topic?: string;+ jsonOptions?: any;+}) {+ const ds = await deepResearchStart(body);+ expectDeepResearchStartToSucceed(ds);++ let x;++ do {+ x = await deepResearchStatus(ds.body.id);+ expect(x.statusCode).toBe(200);+ expect(typeof x.body.status).toBe("string");+ } while (x.body.status === "processing");++ expect(x.body.success).toBe(true);+ expect(x.body.status).toBe("completed");+ return x.body;+}diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.tsindex ce50823fb..36d83eda7 100644--- a/apps/api/src/lib/deep-research/deep-research-service.ts+++ b/apps/api/src/lib/deep-research/deep-research-service.ts@@ -47,11 +47,29 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {const acuc = await getACUCTeam(teamId);+ const checkTimeLimit = () => {+ const timeElapsed = Date.now() - startTime;+ const isLimitReached = timeElapsed >= timeLimit * 1000;+ if (isLimitReached) {+ logger.debug("[Deep Research] Time limit reached", {+ timeElapsed: timeElapsed / 1000,+ timeLimit+ });+ }+ return isLimitReached;+ };+try {while (!state.hasReachedMaxDepth() && urlsAnalyzed < maxUrls) {logger.debug("[Deep Research] Current depth:", state.getCurrentDepth());- const timeElapsed = Date.now() - startTime;- if (timeElapsed >= timeLimit * 1000) {+ logger.debug("[Deep Research] URL analysis count:", {+ urlsAnalyzed,+ maxUrls,+ timeElapsed: (Date.now() - startTime) / 1000,+ timeLimit+ });++ if (checkTimeLimit()) {logger.debug("[Deep Research] Time limit reached, stopping research");break;}@@ -142,15 +160,25 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {}// Filter out already seen URLs and track new ones- const newSearchResults = searchResults.filter(async (result) => {+ const newSearchResults: typeof searchResults = [];+ for (const result of searchResults) {if (!result.url || state.hasSeenUrl(result.url)) {- return false;+ continue;}state.addSeenUrl(result.url);urlsAnalyzed++;- return true;- });+ if (urlsAnalyzed >= maxUrls) {+ logger.debug("[Deep Research] Max URLs limit reached", { urlsAnalyzed, maxUrls });+ break;+ }+ newSearchResults.push(result);+ }++ if (checkTimeLimit()) {+ logger.debug("[Deep Research] Time limit reached during URL filtering");+ break;+ }await state.addSources(newSearchResults.map((result) => ({url: result.url ?? "",@@ -205,6 +233,11 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {const timeRemaining = timeLimit * 1000 - (Date.now() - startTime);logger.debug("[Deep Research] Time remaining (ms):", { timeRemaining });+ if (checkTimeLimit()) {+ logger.debug("[Deep Research] Time limit reached before analysis");+ break;+ }+const analysis = await llmService.analyzeAndPlan(state.getFindings(),currentTopic,@@ -212,6 +245,11 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {options.systemPrompt ?? "",costTracking,);++ if (checkTimeLimit()) {+ logger.debug("[Deep Research] Time limit reached after analysis");+ break;+ }if (!analysis) {logger.debug("[Deep Research] Analysis failed");@@ -258,6 +296,12 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {// Final synthesislogger.debug("[Deep Research] Starting final synthesis");++ // Check time limit before final synthesis+ if (checkTimeLimit()) {+ logger.debug("[Deep Research] Time limit reached before final synthesis");+ }+await state.addActivity([{type: "synthesis",status: "processing",