Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(extract-status): be able to get extract status even after TTL lapses (#1599)

Gergő Móricza3145ccacc80
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index b1f97471b..4d9969357 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -33,11 +33,12 @@ export type PseudoJob<T> = {
timestamp: number,
data: {
scrapeOptions: any,
+ teamId?: string,
},
failedReason?: string,
}
-export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null }
+export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string}
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts
index 76a611f85..1795363e7 100644
--- a/apps/api/src/controllers/v1/extract-status.ts
+++ b/apps/api/src/controllers/v1/extract-status.ts
@@ -18,10 +18,11 @@ export async function getExtractJob(id: string): Promise<PseudoJob<ExtractResult
const job: PseudoJob<any> = {
id,
- getState: bullJob ? bullJob.getState : (() => dbJob!.success ? "completed" : "failed"),
+ getState: bullJob ? bullJob.getState.bind(bullJob) : (() => dbJob!.success ? "completed" : "failed"),
returnvalue: data,
data: {
scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,
+ teamId: bullJob ? bullJob.data.teamId : dbJob!.team_id,
},
timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),
failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
@@ -36,7 +37,9 @@ export async function extractStatusController(
) {
const extract = await getExtract(req.params.jobId);
- if (!extract) {
+ let status = extract?.status;
+
+ if (extract && extract.team_id !== req.auth.team_id) {
return res.status(404).json({
success: false,
error: "Extract job not found",
@@ -45,34 +48,46 @@ export async function extractStatusController(
let data: ExtractResult | [] = [];
- if (extract.status === "completed") {
+ if (!extract || extract.status === "completed") {
const jobData = await getExtractJob(req.params.jobId);
- if (!jobData) {
+ if ((!jobData && !extract) || (jobData && jobData.data.teamId !== req.auth.team_id)) {
return res.status(404).json({
success: false,
- error: "Job not found",
+ error: "Extract job not found",
});
}
- if (!jobData.returnvalue) {
+ if (jobData) {
+ const jobStatus = await jobData.getState();
+
+ if (jobStatus === "completed") {
+ status = "completed";
+ } else if (jobStatus === "failed") {
+ status = "failed";
+ } else {
+ status = "processing";
+ }
+ }
+
+ if (!jobData?.returnvalue) {
// if we got in the split-second where the redis is updated but the bull isn't
// just pretend it's still processing - MG
- extract.status = "processing";
+ status = "processing";
} else {
data = jobData.returnvalue ?? [];
}
}
return res.status(200).json({
- success: extract.status === "failed" ? false : true,
+ success: status === "failed" ? false : true,
data,
- status: extract.status,
+ status,
error: extract?.error ?? undefined,
expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),
- steps: extract.showSteps ? extract.steps : undefined,
- llmUsage: extract.showLLMUsage ? extract.llmUsage : undefined,
- sources: extract.showSources ? extract.sources : undefined,
- costTracking: extract.showCostTracking ? extract.costTracking : undefined,
- sessionIds: extract.sessionIds ? extract.sessionIds : undefined,
+ steps: extract?.showSteps ? extract.steps : undefined,
+ llmUsage: extract?.showLLMUsage ? extract.llmUsage : undefined,
+ sources: extract?.showSources ? extract.sources : undefined,
+ costTracking: extract?.showCostTracking ? extract.costTracking : undefined,
+ sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,
});
}