Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(extract-status): be able to get extract status even after TTL lapses (#1599)
Gergő Móricza3145ccacc80
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex b1f97471b..4d9969357 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -33,11 +33,12 @@ export type PseudoJob<T> = {timestamp: number,data: {scrapeOptions: any,+ teamId?: string,},failedReason?: string,}-export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null }+export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string}export async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex 76a611f85..1795363e7 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -18,10 +18,11 @@ export async function getExtractJob(id: string): Promise<PseudoJob<ExtractResultconst job: PseudoJob<any> = {id,- getState: bullJob ? bullJob.getState : (() => dbJob!.success ? "completed" : "failed"),+ getState: bullJob ? bullJob.getState.bind(bullJob) : (() => dbJob!.success ? "completed" : "failed"),returnvalue: data,data: {scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,+ teamId: bullJob ? bullJob.data.teamId : dbJob!.team_id,},timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,@@ -36,7 +37,9 @@ export async function extractStatusController() {const extract = await getExtract(req.params.jobId);- if (!extract) {+ let status = extract?.status;++ if (extract && extract.team_id !== req.auth.team_id) {return res.status(404).json({success: false,error: "Extract job not found",@@ -45,34 +48,46 @@ export async function extractStatusController(let data: ExtractResult | [] = [];- if (extract.status === "completed") {+ if (!extract || extract.status === "completed") {const jobData = await getExtractJob(req.params.jobId);- if (!jobData) {+ if ((!jobData && !extract) || (jobData && jobData.data.teamId !== req.auth.team_id)) {return res.status(404).json({success: false,- error: "Job not found",+ error: "Extract job not found",});}- if (!jobData.returnvalue) {+ if (jobData) {+ const jobStatus = await jobData.getState();++ if (jobStatus === "completed") {+ status = "completed";+ } else if (jobStatus === "failed") {+ status = "failed";+ } else {+ status = "processing";+ }+ }++ if (!jobData?.returnvalue) {// if we got in the split-second where the redis is updated but the bull isn't// just pretend it's still processing - MG- extract.status = "processing";+ status = "processing";} else {data = jobData.returnvalue ?? [];}}return res.status(200).json({- success: extract.status === "failed" ? false : true,+ success: status === "failed" ? false : true,data,- status: extract.status,+ status,error: extract?.error ?? undefined,expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),- steps: extract.showSteps ? extract.steps : undefined,- llmUsage: extract.showLLMUsage ? extract.llmUsage : undefined,- sources: extract.showSources ? extract.sources : undefined,- costTracking: extract.showCostTracking ? extract.costTracking : undefined,- sessionIds: extract.sessionIds ? extract.sessionIds : undefined,+ steps: extract?.showSteps ? extract.steps : undefined,+ llmUsage: extract?.showLLMUsage ? extract.llmUsage : undefined,+ sources: extract?.showSources ? extract.sources : undefined,+ costTracking: extract?.showCostTracking ? extract.costTracking : undefined,+ sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,});}