Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/extract): timing data
Gergő Móricz81e98a2e9db2
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 0400763a8..09218c158 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -110,6 +110,8 @@ export async function extractController(isUrlBlocked(url, req.acuc?.flags ?? null),) ?? [];+ const createdAt = Date.now();+if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {if (!res.headersSent) {return res.status(403).json({@@ -149,6 +151,7 @@ export async function extractController(extractId,agent: req.body.agent,apiKeyId: req.acuc?.api_key_id ?? null,+ createdAt,};if (@@ -164,7 +167,7 @@ export async function extractController(await saveExtract(extractId, {id: extractId,team_id: req.auth.team_id,- createdAt: Date.now(),+ createdAt,status: "processing",showSteps: req.body.__experimental_streamSteps,showLLMUsage: req.body.__experimental_llmUsage,diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 6a614e78f..08dce3a62 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -48,7 +48,7 @@ export async function extractController(}const extractId = crypto.randomUUID();-+ const createdAt = Date.now();_logger.info("Extract starting...", {request: req.body,originalRequest,@@ -65,12 +65,13 @@ export async function extractController(subId: req.acuc?.sub_id,extractId,agent: req.body.agent,+ createdAt,};await saveExtract(extractId, {id: extractId,team_id: req.auth.team_id,- createdAt: Date.now(),+ createdAt,status: "processing",showSteps: req.body.__experimental_streamSteps,showLLMUsage: req.body.__experimental_llmUsage,diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 4e70fc516..c4952a160 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -50,6 +50,7 @@ interface ExtractServiceOptions {cacheKey?: string;agent?: boolean;apiKeyId: number | null;+ createdAt?: number;}export interface ExtractResult {@@ -78,6 +79,7 @@ export async function performExtraction(options: ExtractServiceOptions,): Promise<ExtractResult> {const { request, teamId, subId, apiKeyId } = options;+ const createdAt = options.createdAt ? new Date(options.createdAt) : new Date();const urlTraces: URLTrace[] = [];let docsMap: Map<string, Document> = new Map();let singleAnswerCompletions: completions | null = null;@@ -134,7 +136,7 @@ export async function performExtraction(message: "No search results found",num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -680,7 +682,7 @@ export async function performExtraction(: "Failed to transform array to object",num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -797,7 +799,7 @@ export async function performExtraction(message: error.message,num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -855,7 +857,7 @@ export async function performExtraction(message: errorMessage,num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -1049,7 +1051,7 @@ export async function performExtraction(message: "Extract completed",num_docs: 1,docs: finalResult ?? {},- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -1127,7 +1129,7 @@ export async function performExtraction(: "An unexpected error occurred",num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 951c0a536..edaf96ca9 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -47,6 +47,7 @@ interface ExtractServiceOptions {cacheMode?: "load" | "save" | "direct";cacheKey?: string;apiKeyId: number | null;+ createdAt?: number;}interface ExtractResult {@@ -75,6 +76,7 @@ export async function performExtraction_F0(options: ExtractServiceOptions,): Promise<ExtractResult> {const { request, teamId, subId, apiKeyId } = options;+ const createdAt = options.createdAt ? new Date(options.createdAt) : new Date();const urlTraces: URLTrace[] = [];let docsMap: Map<string, Document> = new Map();let singleAnswerCompletions: completions | null = null;@@ -120,7 +122,7 @@ export async function performExtraction_F0(message: "No search results found",num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -228,7 +230,7 @@ export async function performExtraction_F0(message: "No valid URLs found to scrape",num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -604,7 +606,7 @@ export async function performExtraction_F0(message: "Failed to transform array to object",num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -702,7 +704,7 @@ export async function performExtraction_F0(message: "Failed to scrape documents",num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -732,7 +734,7 @@ export async function performExtraction_F0(message: "All provided URLs are invalid",num_docs: 1,docs: [],- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",@@ -904,7 +906,7 @@ export async function performExtraction_F0(message: "Extract completed",num_docs: 1,docs: finalResult ?? {},- time_taken: (new Date().getTime() - Date.now()) / 1000,+ time_taken: (new Date().getTime() - createdAt.getTime()) / 1000,team_id: teamId,mode: "extract",url: request.urls?.join(", ") || "",diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex b409e5554..4a1ba3b69 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -3,7 +3,6 @@ import { shutdownOtel } from "../otel";import "./sentry";import * as Sentry from "@sentry/node";import {- getExtractQueue,getDeepResearchQueue,getGenerateLlmsTxtQueue,getRedisConnection,@@ -13,24 +12,13 @@ import { logger as _logger } from "../lib/logger";import systemMonitor from "./system-monitor";import { v4 as uuidv4 } from "uuid";import { configDotenv } from "dotenv";-import {- ExtractResult,- performExtraction,-} from "../lib/extract/extraction-service";-import { updateExtract } from "../lib/extract/extract-redis";import { updateDeepResearch } from "../lib/deep-research/deep-research-redis";import { performDeepResearch } from "../lib/deep-research/deep-research-service";import { performGenerateLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-service";import { updateGeneratedLlmsTxt } from "../lib/generate-llmstxt/generate-llmstxt-redis";-import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";-import { createWebhookSender, WebhookEvent } from "./webhook";import Express from "express";-import http from "http";-import https from "https";-import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { BullMQOtel } from "bullmq-otel";-import { getErrorContactMessage } from "../lib/deployment";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";configDotenv();@@ -50,137 +38,6 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;const runningJobs: Set<string> = new Set();-// Install cacheable lookup for all other requests-// cacheableLookup.install(http.globalAgent);-// cacheableLookup.install(https.globalAgent);--const processExtractJobInternal = async (- token: string,- job: Job & { id: string },-) => {- const logger = _logger.child({- module: "extract-worker",- method: "processJobInternal",- jobId: job.id,- extractId: job.data.extractId,- teamId: job.data?.teamId ?? undefined,- });-- const extendLockInterval = setInterval(async () => {- logger.info(`🔄 Worker extending lock on job ${job.id}`);- await job.extendLock(token, jobLockExtensionTime);- }, jobLockExtendInterval);-- const sender = await createWebhookSender({- teamId: job.data.teamId,- jobId: job.data.extractId,- webhook: job.data.request.webhook,- v0: false,- });-- try {- if (sender) {- sender.send(WebhookEvent.EXTRACT_STARTED, {- success: true,- });- }-- let result: ExtractResult | null = null;-- const model = job.data.request.agent?.model;- if (- job.data.request.agent &&- model &&- model.toLowerCase().includes("fire-1")- ) {- result = await performExtraction(job.data.extractId, {- request: job.data.request,- teamId: job.data.teamId,- subId: job.data.subId,- apiKeyId: job.data.apiKeyId,- });- } else {- result = await performExtraction_F0(job.data.extractId, {- request: job.data.request,- teamId: job.data.teamId,- subId: job.data.subId,- apiKeyId: job.data.apiKeyId,- });- }- // result = await performExtraction_F0(job.data.extractId, {- // request: job.data.request,- // teamId: job.data.teamId,- // subId: job.data.subId,- // });-- if (result && result.success) {- // Move job to completed state in Redis- await job.moveToCompleted(result, token, false);-- if (sender) {- sender.send(WebhookEvent.EXTRACT_COMPLETED, {- success: true,- data: [result],- });- }-- return result;- } else {- // throw new Error(result.error || "Unknown error during extraction");-- await job.moveToCompleted(result, token, false);- await updateExtract(job.data.extractId, {- error: result?.error ?? getErrorContactMessage(job.data.extractId),- });-- if (sender) {- sender.send(WebhookEvent.EXTRACT_FAILED, {- success: false,- error: result?.error ?? getErrorContactMessage(job.data.extractId),- });- }-- return result;- }- } catch (error) {- logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });-- Sentry.captureException(error, {- data: {- job: job.id,- },- });-- try {- // Move job to failed state in Redis- await job.moveToFailed(error, token, false);- } catch (e) {- logger.log("Failed to move job to failed state in Redis", { error });- }-- await updateExtract(job.data.extractId, {- status: "failed",- error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),- });-- if (sender) {- sender.send(WebhookEvent.EXTRACT_FAILED, {- success: false,- error:- (error as any)?.message ?? getErrorContactMessage(job.data.extractId),- });- }-- return {- success: false,- error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),- };- // throw error;- } finally {- clearInterval(extendLockInterval);- }-};-const processDeepResearchJobInternal = async (token: string,job: Job & { id: string },