Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: extract race condition (#2548)
tom53665e3bebd2
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex 907819331..c1c73e345 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -10,7 +10,7 @@ import { logger as _logger } from "../../lib/logger";type DBExtract = {id: string;- success: boolean;+ is_successful: boolean;options: any;created_at: any;error: string | null;@@ -47,7 +47,7 @@ async function getExtractJob(const job: ExtractPseudoJob<any> = {id,getState: dbExtract- ? () => (dbExtract.success ? "completed" : "failed")+ ? () => (dbExtract.is_successful ? "completed" : "failed"): () => bullJob!.getState(),returnvalue: data,data: {diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 71a1894f6..bec78ab0b 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -64,6 +64,8 @@ export interface ExtractResult {llmUsage?: number;totalUrlsScraped?: number;sources?: Record<string, string[]>;+ tokensBilled?: number;+ creditsBilled?: number;}type completions = {@@ -134,7 +136,7 @@ export async function performExtraction(const tokens_billed = 300 + calculateThinkingCost(costTracking);const creditsToBill = Math.ceil(tokens_billed / 15);- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -667,7 +669,7 @@ export async function performExtraction(});const tokens_billed = 300 + calculateThinkingCost(costTracking);const creditsToBill = Math.ceil(tokens_billed / 15);- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -774,7 +776,7 @@ export async function performExtraction(} catch (error) {const tokens_billed = 300 + calculateThinkingCost(costTracking);const creditsToBill = Math.ceil(tokens_billed / 15);- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -816,7 +818,7 @@ export async function performExtraction();},);- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -1012,7 +1014,7 @@ export async function performExtraction(tokensBilled: tokensToBill,creditsBilled: creditsToBill,});- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -1025,32 +1027,16 @@ export async function performExtraction(cost_tracking: costTracking.toJSON(),}).then(() => {- logger.debug("Updating extract status to completed", {+ logger.debug("Extract completed successfully", {extractId,llmUsage,sources,tokensBilled: tokensToBill,creditsBilled: creditsToBill,});- updateExtract(extractId, {- status: "completed",- llmUsage,- sources,- tokensBilled: tokensToBill,- creditsBilled: creditsToBill,- // costTracking,- }).catch(error => {- logger.error("Failed to update extract status to completed", {- extractId,- error,- });- });}).catch(error => {- logger.error(- "Failed to log extract to database",- { extractId, error },- );+ logger.error("Failed to log extract to database", { extractId, error });});logger.debug("Done!");@@ -1084,6 +1070,8 @@ export async function performExtraction(llmUsage,totalUrlsScraped,sources,+ tokensBilled: tokensToBill,+ creditsBilled: creditsToBill,};} catch (error) {const tokens_billed = 300 + calculateThinkingCost(costTracking);diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 9f4ec51ef..031163b9b 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -62,6 +62,8 @@ interface ExtractResult {llmUsage?: number;totalUrlsScraped?: number;sources?: Record<string, string[]>;+ tokensBilled?: number;+ creditsBilled?: number;}type completions = {@@ -120,7 +122,7 @@ export async function performExtraction_F0(logger.error("No search results found", {query: request.prompt,});- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -221,7 +223,7 @@ export async function performExtraction_F0(logger.error("0 links! Bailing.", {linkCount: links.length,});- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -594,7 +596,7 @@ export async function performExtraction_F0(Object.assign(sources, multiEntitySources);} catch (error) {logger.error(`Failed to transform array to object`, { error });- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -686,7 +688,7 @@ export async function performExtraction_F0(logger.debug("Scrapes finished.", { docCount: validResults.length });} catch (error) {logger.error("Failed to scrape documents", { error });- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -709,7 +711,7 @@ export async function performExtraction_F0(if (docsMap.size == 0) {// All urls are invalidlogger.error("All provided URLs are invalid!");- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -884,7 +886,7 @@ export async function performExtraction_F0(});// Log job with token usage and sources- logExtract({+ await logExtract({id: extractId,request_id: extractId,urls: request.urls || [],@@ -922,17 +924,13 @@ export async function performExtraction_F0(creditsBilled: creditsToBill,});- updateExtract(extractId, {- status: "completed",+ // Redis status update moved to worker for consistent ordering+ logger.debug("Extract completed successfully", {+ extractId,llmUsage,sources,tokensBilled: tokensToBill,creditsBilled: creditsToBill,- }).catch(error => {- logger.error("Failed to update extract status to completed", {- extractId,- error,- });});}).catch(error => {@@ -965,5 +963,7 @@ export async function performExtraction_F0(llmUsage,totalUrlsScraped,sources,+ tokensBilled: tokensToBill,+ creditsBilled: creditsToBill,};}diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 007705d22..5eed8eb2f 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -96,7 +96,14 @@ const processExtractJobInternal = async (// });if (result && result.success) {- // Move job to completed state in Redis+ await updateExtract(job.data.extractId, {+ status: "completed",+ llmUsage: result.llmUsage,+ sources: result.sources,+ tokensBilled: result.tokensBilled,+ creditsBilled: result.creditsBilled,+ });+await job.moveToCompleted(result, token, false);if (sender) {@@ -108,13 +115,13 @@ const processExtractJobInternal = async (return result;} else {- // throw new Error(result.error || "Unknown error during extraction");-- await job.moveToCompleted(result, token, false);await updateExtract(job.data.extractId, {+ status: "failed",error: result?.error ?? getErrorContactMessage(job.data.extractId),});+ await job.moveToCompleted(result, token, false);+if (sender) {sender.send(WebhookEvent.EXTRACT_FAILED, {success: false,@@ -136,18 +143,17 @@ const processExtractJobInternal = async (});}- try {- // Move job to failed state in Redis- await job.moveToFailed(error, token, false);- } catch (e) {- logger.log("Failed to move job to failed state in Redis", { error });- }-await updateExtract(job.data.extractId, {status: "failed",error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),});+ try {+ await job.moveToFailed(error, token, false);+ } catch (e) {+ logger.log("Failed to move job to failed state in BullMQ", { error });+ }+if (sender) {sender.send(WebhookEvent.EXTRACT_FAILED, {success: false,