Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(extract): log failed extracts
Gergő Móriczd82f44c93e89
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 0b72f379a..6f14306a2 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -36,7 +36,7 @@ import { getCachedDocs, saveCachedDocs } from "./helpers/cached-docs";import { normalizeUrl } from "../canonical-url";import { search } from "../../search";import { buildRephraseToSerpPrompt } from "./build-prompts";-import fs from "fs/promises";+interface ExtractServiceOptions {request: ExtractRequest;teamId: string;@@ -109,845 +109,887 @@ export async function performExtraction(teamId,});- // If no URLs are provided, generate URLs from the prompt- if ((!request.urls || request.urls.length === 0) && request.prompt) {- logger.debug("Generating URLs from prompt...", {- prompt: request.prompt,- });- const rephrasedPrompt = await generateBasicCompletion(- buildRephraseToSerpPrompt(request.prompt),- );- let rptxt = rephrasedPrompt?.text.replace('"', "").replace("'", "") || "";- if (rephrasedPrompt) {- costTracking.otherCallCount++;- costTracking.otherCost += rephrasedPrompt.cost;- costTracking.totalCost += rephrasedPrompt.cost;+ try {++ // If no URLs are provided, generate URLs from the prompt+ if ((!request.urls || request.urls.length === 0) && request.prompt) {+ logger.debug("Generating URLs from prompt...", {+ prompt: request.prompt,+ });+ const rephrasedPrompt = await generateBasicCompletion(+ buildRephraseToSerpPrompt(request.prompt),+ );+ let rptxt = rephrasedPrompt?.text.replace('"', "").replace("'", "") || "";+ if (rephrasedPrompt) {+ costTracking.otherCallCount++;+ costTracking.otherCost += rephrasedPrompt.cost;+ costTracking.totalCost += rephrasedPrompt.cost;+ }+ const searchResults = await search({+ query: rptxt,+ num_results: 10,+ });++ request.urls = searchResults.map((result) => result.url) as string[];+ }+ if (request.urls && request.urls.length === 0) {+ logger.error("No search results found", {+ query: request.prompt,+ });+ logJob({+ job_id: extractId,+ success: false,+ message: "No search results found",+ num_docs: 1,+ docs: [],+ time_taken: (new Date().getTime() - Date.now()) / 1000,+ team_id: teamId,+ mode: "extract",+ url: request.urls?.join(", ") || "",+ scrapeOptions: request,+ origin: request.origin ?? "api",+ num_tokens: 0,+ tokens_billed: 0,+ sources,+ cost_tracking: costTracking,+ });+ return {+ success: false,+ error: "No search results found",+ extractId,+ };}- const searchResults = await search({- query: rptxt,- num_results: 10,- });- request.urls = searchResults.map((result) => result.url) as string[];- }- if (request.urls && request.urls.length === 0) {- logger.error("No search results found", {- query: request.prompt,- });- return {- success: false,- error: "No search results found",- extractId,- };- }+ const urls = request.urls || ([] as string[]);- const urls = request.urls || ([] as string[]);-- if (- request.__experimental_cacheMode == "load" &&- request.__experimental_cacheKey &&- urls- ) {- logger.debug("Loading cached docs...");- try {- const cache = await getCachedDocs(urls, request.__experimental_cacheKey);- for (const doc of cache) {- if (doc.metadata.url) {- docsMap.set(normalizeUrl(doc.metadata.url), doc);+ if (+ request.__experimental_cacheMode == "load" &&+ request.__experimental_cacheKey &&+ urls+ ) {+ logger.debug("Loading cached docs...");+ try {+ const cache = await getCachedDocs(urls, request.__experimental_cacheKey);+ for (const doc of cache) {+ if (doc.metadata.url) {+ docsMap.set(normalizeUrl(doc.metadata.url), doc);+ }}+ } catch (error) {+ logger.error("Error loading cached docs", { error });}- } catch (error) {- logger.error("Error loading cached docs", { error });}- }- // Token tracking- let tokenUsage: TokenUsage[] = [];-- await updateExtract(extractId, {- status: "processing",- steps: [- {- step: ExtractStep.INITIAL,- startedAt: Date.now(),- finishedAt: Date.now(),- discoveredLinks: request.urls,- },- ],- });-- let reqSchema = request.schema;- if (!reqSchema && request.prompt) {- const schemaGenRes = await generateSchemaFromPrompt(request.prompt, logger);- reqSchema = schemaGenRes.extract;- costTracking.otherCallCount++;- costTracking.otherCost += schemaGenRes.cost;- costTracking.totalCost += schemaGenRes.cost;+ // Token tracking+ let tokenUsage: TokenUsage[] = [];- logger.debug("Generated request schema.", {- originalSchema: request.schema,- schema: reqSchema,+ await updateExtract(extractId, {+ status: "processing",+ steps: [+ {+ step: ExtractStep.INITIAL,+ startedAt: Date.now(),+ finishedAt: Date.now(),+ discoveredLinks: request.urls,+ },+ ],});- }- if (reqSchema) {- reqSchema = await dereferenceSchema(reqSchema);- }-- logger.debug("Transformed schema.", {- originalSchema: request.schema,- schema: reqSchema,- });+ let reqSchema = request.schema;+ if (!reqSchema && request.prompt) {+ const schemaGenRes = await generateSchemaFromPrompt(request.prompt, logger);+ reqSchema = schemaGenRes.extract;+ costTracking.otherCallCount++;+ costTracking.otherCost += schemaGenRes.cost;+ costTracking.totalCost += schemaGenRes.cost;- let rSchema = reqSchema;-- // agent evaluates if the schema or the prompt has an array with big amount of items- // also it checks if the schema any other properties that are not arrays- // if so, it splits the results into 2 types of completions:- // 1. the first one is a completion that will extract the array of items- // 2. the second one is multiple completions that will extract the items from the array- let startAnalyze = Date.now();- const {- isMultiEntity,- multiEntityKeys,- reasoning,- keyIndicators,- tokenUsage: schemaAnalysisTokenUsage,- cost: schemaAnalysisCost,- } = await analyzeSchemaAndPrompt(urls, reqSchema, request.prompt ?? "", logger);-- logger.debug("Analyzed schema.", {- isMultiEntity,- multiEntityKeys,- reasoning,- keyIndicators,- });+ logger.debug("Generated request schema.", {+ originalSchema: request.schema,+ schema: reqSchema,+ });+ }- costTracking.otherCallCount++;- costTracking.otherCost += schemaAnalysisCost;- costTracking.totalCost += schemaAnalysisCost;+ if (reqSchema) {+ reqSchema = await dereferenceSchema(reqSchema);+ }- // Track schema analysis tokens- tokenUsage.push(schemaAnalysisTokenUsage);+ logger.debug("Transformed schema.", {+ originalSchema: request.schema,+ schema: reqSchema,+ });- let startMap = Date.now();- let aggMapLinks: string[] = [];- logger.debug("Processing URLs...", {- urlCount: request.urls?.length || 0,- });+ let rSchema = reqSchema;- const urlPromises = urls.map((url) =>- processUrl(- {- url,- prompt: request.prompt,- teamId,- allowExternalLinks: request.allowExternalLinks,- origin: request.origin,- limit: request.limit,- includeSubdomains: request.includeSubdomains,- schema: request.schema,- log,- isMultiEntity,- reasoning,- multiEntityKeys,- keyIndicators,- },- urlTraces,- (links: string[]) => {- aggMapLinks.push(...links);- updateExtract(extractId, {- steps: [- {- step: ExtractStep.MAP,- startedAt: startMap,- finishedAt: Date.now(),- discoveredLinks: aggMapLinks,- },- ],- });- },- logger.child({ module: "extract", method: "processUrl", url }),- costTracking,- ),- );-- const processedUrls = await Promise.all(urlPromises);- let links = processedUrls.flat().filter((url) => url);- logger.debug("Processed URLs.", {- linkCount: links.length,- });-- if (links.length === 0) {- links = urls.map(x => x.replace(/\*$/g, ""));- logger.warn("0 links! Doing just the original URLs. (without * wildcard)", {- linkCount: links.length,+ // agent evaluates if the schema or the prompt has an array with big amount of items+ // also it checks if the schema any other properties that are not arrays+ // if so, it splits the results into 2 types of completions:+ // 1. the first one is a completion that will extract the array of items+ // 2. the second one is multiple completions that will extract the items from the array+ let startAnalyze = Date.now();+ const {+ isMultiEntity,+ multiEntityKeys,+ reasoning,+ keyIndicators,+ tokenUsage: schemaAnalysisTokenUsage,+ cost: schemaAnalysisCost,+ } = await analyzeSchemaAndPrompt(urls, reqSchema, request.prompt ?? "", logger);++ logger.debug("Analyzed schema.", {+ isMultiEntity,+ multiEntityKeys,+ reasoning,+ keyIndicators,});- }- log["links"] = links;- log["linksLength"] = links.length;-- await updateExtract(extractId, {- status: "processing",- steps: [- {- step: ExtractStep.MAP_RERANK,- startedAt: startMap,- finishedAt: Date.now(),- discoveredLinks: links,- },- ],- });+ costTracking.otherCallCount++;+ costTracking.otherCost += schemaAnalysisCost;+ costTracking.totalCost += schemaAnalysisCost;- if (isMultiEntity && reqSchema) {- log["isMultiEntity"] = true;- logger.debug("=== MULTI-ENTITY ===");+ // Track schema analysis tokens+ tokenUsage.push(schemaAnalysisTokenUsage);- const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(- reqSchema,- multiEntityKeys,- );- rSchema = singleAnswerSchema;- logger.debug("Spread schemas.", { singleAnswerSchema, multiEntitySchema });+ let startMap = Date.now();+ let aggMapLinks: string[] = [];+ logger.debug("Processing URLs...", {+ urlCount: request.urls?.length || 0,+ });- await updateExtract(extractId, {- status: "processing",- steps: [+ const urlPromises = urls.map((url) =>+ processUrl({- step: ExtractStep.MULTI_ENTITY,- startedAt: startAnalyze,- finishedAt: Date.now(),- discoveredLinks: [],+ url,+ prompt: request.prompt,+ teamId,+ allowExternalLinks: request.allowExternalLinks,+ origin: request.origin,+ limit: request.limit,+ includeSubdomains: request.includeSubdomains,+ schema: request.schema,+ log,+ isMultiEntity,+ reasoning,+ multiEntityKeys,+ keyIndicators,},- ],+ urlTraces,+ (links: string[]) => {+ aggMapLinks.push(...links);+ updateExtract(extractId, {+ steps: [+ {+ step: ExtractStep.MAP,+ startedAt: startMap,+ finishedAt: Date.now(),+ discoveredLinks: aggMapLinks,+ },+ ],+ });+ },+ logger.child({ module: "extract", method: "processUrl", url }),+ costTracking,+ ),+ );++ const processedUrls = await Promise.all(urlPromises);+ let links = processedUrls.flat().filter((url) => url);+ logger.debug("Processed URLs.", {+ linkCount: links.length,});- const timeout = 60000;+ if (links.length === 0) {+ links = urls.map(x => x.replace(/\*$/g, ""));+ logger.warn("0 links! Doing just the original URLs. (without * wildcard)", {+ linkCount: links.length,+ });+ }++ log["links"] = links;+ log["linksLength"] = links.length;await updateExtract(extractId, {status: "processing",steps: [{- step: ExtractStep.MULTI_ENTITY_SCRAPE,- startedAt: startAnalyze,+ step: ExtractStep.MAP_RERANK,+ startedAt: startMap,finishedAt: Date.now(),discoveredLinks: links,},],});- logger.debug("Starting multi-entity scrape...");- let startScrape = Date.now();- log["docsSizeBeforeMultiEntityScrape"] = docsMap.size;+ if (isMultiEntity && reqSchema) {+ log["isMultiEntity"] = true;+ logger.debug("=== MULTI-ENTITY ===");- const scrapePromises = links.map((url) => {- if (!docsMap.has(normalizeUrl(url))) {- return scrapeDocument(+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ reqSchema,+ multiEntityKeys,+ );+ rSchema = singleAnswerSchema;+ logger.debug("Spread schemas.", { singleAnswerSchema, multiEntitySchema });++ await updateExtract(extractId, {+ status: "processing",+ steps: [{- url,- teamId,- origin: request.origin || "api",- timeout,+ step: ExtractStep.MULTI_ENTITY,+ startedAt: startAnalyze,+ finishedAt: Date.now(),+ discoveredLinks: [],},- urlTraces,- logger.child({- module: "extract",- method: "scrapeDocument",- url,- isMultiEntity: true,- }),- {- ...request.scrapeOptions,+ ],+ });++ const timeout = 60000;- // Needs to be true for multi-entity to work properly- onlyMainContent: true,+ await updateExtract(extractId, {+ status: "processing",+ steps: [+ {+ step: ExtractStep.MULTI_ENTITY_SCRAPE,+ startedAt: startAnalyze,+ finishedAt: Date.now(),+ discoveredLinks: links,},- );- }- return docsMap.get(normalizeUrl(url));- });+ ],+ });- let multyEntityDocs = (await Promise.all(scrapePromises)).filter(- (doc): doc is Document => doc !== null,- );+ logger.debug("Starting multi-entity scrape...");+ let startScrape = Date.now();+ log["docsSizeBeforeMultiEntityScrape"] = docsMap.size;- log["docsSizeAfterMultiEntityScrape"] = scrapePromises.length;+ const scrapePromises = links.map((url) => {+ if (!docsMap.has(normalizeUrl(url))) {+ return scrapeDocument(+ {+ url,+ teamId,+ origin: request.origin || "api",+ timeout,+ },+ urlTraces,+ logger.child({+ module: "extract",+ method: "scrapeDocument",+ url,+ isMultiEntity: true,+ }),+ {+ ...request.scrapeOptions,- logger.debug("Multi-entity scrape finished.", {- docCount: multyEntityDocs.length,- });+ // Needs to be true for multi-entity to work properly+ onlyMainContent: true,+ },+ );+ }+ return docsMap.get(normalizeUrl(url));+ });- totalUrlsScraped += multyEntityDocs.length;+ let multyEntityDocs = (await Promise.all(scrapePromises)).filter(+ (doc): doc is Document => doc !== null,+ );- let endScrape = Date.now();+ log["docsSizeAfterMultiEntityScrape"] = scrapePromises.length;- await updateExtract(extractId, {- status: "processing",- steps: [- {- step: ExtractStep.MULTI_ENTITY_SCRAPE,- startedAt: startScrape,- finishedAt: endScrape,- discoveredLinks: links,- },- ],- });-- for (const doc of multyEntityDocs) {- if (doc?.metadata?.url) {- docsMap.set(normalizeUrl(doc.metadata.url), doc);- }- }+ logger.debug("Multi-entity scrape finished.", {+ docCount: multyEntityDocs.length,+ });- logger.debug("Updated docsMap.", { docsMapSize: docsMap.size }); // useful for error probing+ totalUrlsScraped += multyEntityDocs.length;- // Process docs in chunks with queue style processing- const chunkSize = 50;- const timeoutCompletion = 45000; // 45 second timeout- const chunks: Document[][] = [];- const extractionResults: { extract: any; url: string }[] = [];+ let endScrape = Date.now();- // Split into chunks- for (let i = 0; i < multyEntityDocs.length; i += chunkSize) {- chunks.push(multyEntityDocs.slice(i, i + chunkSize));- }+ await updateExtract(extractId, {+ status: "processing",+ steps: [+ {+ step: ExtractStep.MULTI_ENTITY_SCRAPE,+ startedAt: startScrape,+ finishedAt: endScrape,+ discoveredLinks: links,+ },+ ],+ });- const sessionIds = chunks.map(() => 'fc-' + crypto.randomUUID());- await updateExtract(extractId, {- status: "processing",- steps: [- {- step: ExtractStep.MULTI_ENTITY_AGENT_SCRAPE,- startedAt: Date.now(),- finishedAt: null- },- ],- sessionIds- });+ for (const doc of multyEntityDocs) {+ if (doc?.metadata?.url) {+ docsMap.set(normalizeUrl(doc.metadata.url), doc);+ }+ }- // Process chunks sequentially with timeout- for (let i = 0; i < chunks.length; i++) {- const chunk = chunks[i];- const sessionId = sessionIds[i];- const chunkPromises = chunk.map(async (doc) => {- try {- ajv.compile(multiEntitySchema);+ logger.debug("Updated docsMap.", { docsMapSize: docsMap.size }); // useful for error probing- // Wrap in timeout promise- const timeoutPromise = new Promise((resolve) => {- setTimeout(() => resolve(null), timeoutCompletion);- });+ // Process docs in chunks with queue style processing+ const chunkSize = 50;+ const timeoutCompletion = 45000; // 45 second timeout+ const chunks: Document[][] = [];+ const extractionResults: { extract: any; url: string }[] = [];- const completionPromise = batchExtractPromise({- multiEntitySchema,- links,- prompt: request.prompt ?? "",- systemPrompt: request.systemPrompt ?? "",- doc,- useAgent: isAgentExtractModelValid(request.agent?.model),- extractId,- sessionId- }, logger);-- // Race between timeout and completion- const multiEntityCompletion = (await completionPromise) as Awaited<- ReturnType<typeof batchExtractPromise>- >;-- // TODO: merge multiEntityCompletion.extract to fit the multiEntitySchema-- // Track multi-entity extraction tokens- if (multiEntityCompletion) {- tokenUsage.push(multiEntityCompletion.totalUsage);-- costTracking.smartScrapeCallCount += multiEntityCompletion.smartScrapeCallCount;- costTracking.smartScrapeCost += multiEntityCompletion.smartScrapeCost;- costTracking.otherCallCount += multiEntityCompletion.otherCallCount;- costTracking.otherCost += multiEntityCompletion.otherCost;- costTracking.totalCost += multiEntityCompletion.smartScrapeCost + multiEntityCompletion.otherCost;-- if (multiEntityCompletion.extract) {- return {- extract: multiEntityCompletion.extract,- url: doc.metadata.url || doc.metadata.sourceURL || "",- };- }- }+ // Split into chunks+ for (let i = 0; i < multyEntityDocs.length; i += chunkSize) {+ chunks.push(multyEntityDocs.slice(i, i + chunkSize));+ }- // console.log(multiEntityCompletion.extract)- // if (!multiEntityCompletion.extract?.is_content_relevant) {- // console.log(`Skipping extraction for ${doc.metadata.url} as content is not relevant`);- // return null;- // }-- // Update token usage in traces- // if (multiEntityCompletion && multiEntityCompletion.numTokens) {- // const totalLength = docs.reduce(- // (sum, doc) => sum + (doc.markdown?.length || 0),- // 0,- // );- // docs.forEach((doc) => {- // if (doc.metadata?.sourceURL) {- // const trace = urlTraces.find(- // (t) => t.url === doc.metadata.sourceURL,- // );- // if (trace && trace.contentStats) {- // trace.contentStats.tokensUsed = Math.floor(- // ((doc.markdown?.length || 0) / totalLength) *- // (multiEntityCompletion?.numTokens || 0),- // );- // }- // }- // });- // }-- // if (multiEntityCompletion.extract && multiEntityCompletion.extract.extraction_confidence < 3) {- // console.log(`Skipping extraction for ${doc.metadata.url} as confidence is too low (${multiEntityCompletion.extract.extraction_confidence})`);- // return null;- // }-- return null;- } catch (error) {- logger.error(`Failed to process document.`, {- error,- url: doc.metadata.url ?? doc.metadata.sourceURL!,- });- return null;- }- });- // Wait for current chunk to complete before processing next chunk- const chunkResults = await Promise.all(chunkPromises);- const validResults = chunkResults.filter(- (result): result is { extract: any; url: string } => result !== null,- );- extractionResults.push(...validResults);- // Merge all extracts from valid results into a single array- const extractArrays = validResults.map((r) =>- Array.isArray(r.extract) ? r.extract : [r.extract],- );- const mergedExtracts = extractArrays.flat();- multiEntityCompletions.push(...mergedExtracts);- multiEntityCompletions = multiEntityCompletions.filter((c) => c !== null);- logger.debug("All multi-entity completion chunks finished.", {- completionCount: multiEntityCompletions.length,+ const sessionIds = chunks.map(() => 'fc-' + crypto.randomUUID());+ await updateExtract(extractId, {+ status: "processing",+ steps: [+ {+ step: ExtractStep.MULTI_ENTITY_AGENT_SCRAPE,+ startedAt: Date.now(),+ finishedAt: null+ },+ ],+ sessionIds});- log["multiEntityCompletionsLength"] = multiEntityCompletions.length;- }- try {- // Use SourceTracker to handle source tracking- const sourceTracker = new SourceTracker();- logger.debug("Created SourceTracker instance");+ // Process chunks sequentially with timeout+ for (let i = 0; i < chunks.length; i++) {+ const chunk = chunks[i];+ const sessionId = sessionIds[i];+ const chunkPromises = chunk.map(async (doc) => {+ try {+ ajv.compile(multiEntitySchema);++ // Wrap in timeout promise+ const timeoutPromise = new Promise((resolve) => {+ setTimeout(() => resolve(null), timeoutCompletion);+ });++ const completionPromise = batchExtractPromise({+ multiEntitySchema,+ links,+ prompt: request.prompt ?? "",+ systemPrompt: request.systemPrompt ?? "",+ doc,+ useAgent: isAgentExtractModelValid(request.agent?.model),+ extractId,+ sessionId+ }, logger);++ // Race between timeout and completion+ const multiEntityCompletion = (await completionPromise) as Awaited<+ ReturnType<typeof batchExtractPromise>+ >;++ // TODO: merge multiEntityCompletion.extract to fit the multiEntitySchema++ // Track multi-entity extraction tokens+ if (multiEntityCompletion) {+ tokenUsage.push(multiEntityCompletion.totalUsage);++ costTracking.smartScrapeCallCount += multiEntityCompletion.smartScrapeCallCount;+ costTracking.smartScrapeCost += multiEntityCompletion.smartScrapeCost;+ costTracking.otherCallCount += multiEntityCompletion.otherCallCount;+ costTracking.otherCost += multiEntityCompletion.otherCost;+ costTracking.totalCost += multiEntityCompletion.smartScrapeCost + multiEntityCompletion.otherCost;++ if (multiEntityCompletion.extract) {+ return {+ extract: multiEntityCompletion.extract,+ url: doc.metadata.url || doc.metadata.sourceURL || "",+ };+ }+ }- // Transform and merge results while preserving sources- try {- sourceTracker.transformResults(- extractionResults,- multiEntitySchema,- false,+ // console.log(multiEntityCompletion.extract)+ // if (!multiEntityCompletion.extract?.is_content_relevant) {+ // console.log(`Skipping extraction for ${doc.metadata.url} as content is not relevant`);+ // return null;+ // }++ // Update token usage in traces+ // if (multiEntityCompletion && multiEntityCompletion.numTokens) {+ // const totalLength = docs.reduce(+ // (sum, doc) => sum + (doc.markdown?.length || 0),+ // 0,+ // );+ // docs.forEach((doc) => {+ // if (doc.metadata?.sourceURL) {+ // const trace = urlTraces.find(+ // (t) => t.url === doc.metadata.sourceURL,+ // );+ // if (trace && trace.contentStats) {+ // trace.contentStats.tokensUsed = Math.floor(+ // ((doc.markdown?.length || 0) / totalLength) *+ // (multiEntityCompletion?.numTokens || 0),+ // );+ // }+ // }+ // });+ // }++ // if (multiEntityCompletion.extract && multiEntityCompletion.extract.extraction_confidence < 3) {+ // console.log(`Skipping extraction for ${doc.metadata.url} as confidence is too low (${multiEntityCompletion.extract.extraction_confidence})`);+ // return null;+ // }++ return null;+ } catch (error) {+ logger.error(`Failed to process document.`, {+ error,+ url: doc.metadata.url ?? doc.metadata.sourceURL!,+ });+ return null;+ }+ });+ // Wait for current chunk to complete before processing next chunk+ const chunkResults = await Promise.all(chunkPromises);+ const validResults = chunkResults.filter(+ (result): result is { extract: any; url: string } => result !== null,);- logger.debug("Successfully transformed results with sourceTracker");- } catch (error) {- const errorLog = `[${new Date().toISOString()}] Error in sourceTracker.transformResults: ${JSON.stringify(error, null, 2)}\n`;- await fs.appendFile('logs/extraction-errors.log', errorLog);- logger.error(`Error in sourceTracker.transformResults:`, { error });- throw error;- }-- try {- multiEntityResult = transformArrayToObject(- multiEntitySchema,- multiEntityCompletions,+ extractionResults.push(...validResults);+ // Merge all extracts from valid results into a single array+ const extractArrays = validResults.map((r) =>+ Array.isArray(r.extract) ? r.extract : [r.extract],);- logger.debug("Successfully transformed array to object");- } catch (error) {- const errorLog = `[${new Date().toISOString()}] Error in transformArrayToObject: ${JSON.stringify(error, null, 2)}\n`;- await fs.appendFile('logs/extraction-errors.log', errorLog);- logger.error(`Error in transformArrayToObject:`, { error });- throw error;+ const mergedExtracts = extractArrays.flat();+ multiEntityCompletions.push(...mergedExtracts);+ multiEntityCompletions = multiEntityCompletions.filter((c) => c !== null);+ logger.debug("All multi-entity completion chunks finished.", {+ completionCount: multiEntityCompletions.length,+ });+ log["multiEntityCompletionsLength"] = multiEntityCompletions.length;}- // Track sources before deduplicationtry {- sourceTracker.trackPreDeduplicationSources(multiEntityResult);- logger.debug("Successfully tracked pre-deduplication sources");- } catch (error) {- const errorLog = `[${new Date().toISOString()}] Error in trackPreDeduplicationSources: ${JSON.stringify(error, null, 2)}\n`;- await fs.appendFile('logs/extraction-errors.log', errorLog);- logger.error(`Error in trackPreDeduplicationSources:`, { error });- throw error;- }+ // Use SourceTracker to handle source tracking+ const sourceTracker = new SourceTracker();+ logger.debug("Created SourceTracker instance");- // Apply deduplication and merge- try {- multiEntityResult = deduplicateObjectsArray(multiEntityResult);- logger.debug("Successfully deduplicated objects array");- } catch (error) {- const errorLog = `[${new Date().toISOString()}] Error in deduplicateObjectsArray: ${JSON.stringify(error, null, 2)}\n`;- await fs.appendFile('logs/extraction-errors.log', errorLog);- logger.error(`Error in deduplicateObjectsArray:`, { error });- throw error;- }+ // Transform and merge results while preserving sources+ try {+ sourceTracker.transformResults(+ extractionResults,+ multiEntitySchema,+ false,+ );+ logger.debug("Successfully transformed results with sourceTracker");+ } catch (error) {+ logger.error(`Error in sourceTracker.transformResults:`, { error });+ throw error;+ }- try {- multiEntityResult = mergeNullValObjs(multiEntityResult);- logger.debug("Successfully merged null value objects");- } catch (error) {- const errorLog = `[${new Date().toISOString()}] Error in mergeNullValObjs: ${JSON.stringify(error, null, 2)}\n`;- await fs.appendFile('logs/extraction-errors.log', errorLog);- logger.error(`Error in mergeNullValObjs:`, { error });- throw error;- }+ try {+ multiEntityResult = transformArrayToObject(+ multiEntitySchema,+ multiEntityCompletions,+ );+ logger.debug("Successfully transformed array to object");+ } catch (error) {+ logger.error(`Error in transformArrayToObject:`, { error });+ throw error;+ }- // Map sources to final deduplicated/merged items- try {- const multiEntitySources = sourceTracker.mapSourcesToFinalItems(- multiEntityResult,- multiEntityKeys,- );- Object.assign(sources, multiEntitySources);- logger.debug("Successfully mapped sources to final items");+ // Track sources before deduplication+ try {+ sourceTracker.trackPreDeduplicationSources(multiEntityResult);+ logger.debug("Successfully tracked pre-deduplication sources");+ } catch (error) {+ logger.error(`Error in trackPreDeduplicationSources:`, { error });+ throw error;+ }++ // Apply deduplication and merge+ try {+ multiEntityResult = deduplicateObjectsArray(multiEntityResult);+ logger.debug("Successfully deduplicated objects array");+ } catch (error) {+ logger.error(`Error in deduplicateObjectsArray:`, { error });+ throw error;+ }++ try {+ multiEntityResult = mergeNullValObjs(multiEntityResult);+ logger.debug("Successfully merged null value objects");+ } catch (error) {+ logger.error(`Error in mergeNullValObjs:`, { error });+ throw error;+ }++ // Map sources to final deduplicated/merged items+ try {+ const multiEntitySources = sourceTracker.mapSourcesToFinalItems(+ multiEntityResult,+ multiEntityKeys,+ );+ Object.assign(sources, multiEntitySources);+ logger.debug("Successfully mapped sources to final items");+ } catch (error) {+ logger.error(`Error in mapSourcesToFinalItems:`, { error });+ throw error;+ }} catch (error) {- const errorLog = `[${new Date().toISOString()}] Error in mapSourcesToFinalItems: ${JSON.stringify(error, null, 2)}\n`;- await fs.appendFile('logs/extraction-errors.log', errorLog);- logger.error(`Error in mapSourcesToFinalItems:`, { error });- throw error;+ logger.error(`Failed to transform array to object`, {+ error,+ errorMessage: error.message,+ errorStack: error.stack,+ multiEntityResult: JSON.stringify(multiEntityResult),+ multiEntityCompletions: JSON.stringify(multiEntityCompletions),+ multiEntitySchema: JSON.stringify(multiEntitySchema)+ });+ logJob({+ job_id: extractId,+ success: false,+ message: (error instanceof Error ? error.message : "Failed to transform array to object"),+ num_docs: 1,+ docs: [],+ time_taken: (new Date().getTime() - Date.now()) / 1000,+ team_id: teamId,+ mode: "extract",+ url: request.urls?.join(", ") || "",+ scrapeOptions: request,+ origin: request.origin ?? "api",+ num_tokens: 0,+ tokens_billed: 0,+ sources,+ cost_tracking: costTracking,+ });+ return {+ success: false,+ error:+ "An unexpected error occurred. Please contact [email protected] for help.",+ extractId,+ urlTrace: urlTraces,+ totalUrlsScraped,+ };}- } catch (error) {- const errorLog = `[${new Date().toISOString()}] Failed to transform array to object\nError: ${JSON.stringify(error, null, 2)}\nStack: ${error.stack}\nMultiEntityResult: ${JSON.stringify(multiEntityResult, null, 2)}\nMultiEntityCompletions: ${JSON.stringify(multiEntityCompletions, null, 2)}\nMultiEntitySchema: ${JSON.stringify(multiEntitySchema, null, 2)}\n\n`;- await fs.appendFile('logs/extraction-errors.log', errorLog);- logger.error(`Failed to transform array to object`, {- error,- errorMessage: error.message,- errorStack: error.stack,- multiEntityResult: JSON.stringify(multiEntityResult),- multiEntityCompletions: JSON.stringify(multiEntityCompletions),- multiEntitySchema: JSON.stringify(multiEntitySchema)- });- return {- success: false,- error:- "An unexpected error occurred. Please contact [email protected] for help.",- extractId,- urlTrace: urlTraces,- totalUrlsScraped,- };}- }- if (- rSchema &&- Object.keys(rSchema).length > 0 &&- rSchema.properties &&- Object.keys(rSchema.properties).length > 0- ) {- log["isSingleEntity"] = true;- logger.debug("=== SINGLE PAGES ===", {- linkCount: links.length,- schema: rSchema,- });+ if (+ rSchema &&+ Object.keys(rSchema).length > 0 &&+ rSchema.properties &&+ Object.keys(rSchema.properties).length > 0+ ) {+ log["isSingleEntity"] = true;+ logger.debug("=== SINGLE PAGES ===", {+ linkCount: links.length,+ schema: rSchema,+ });- // Scrape documents- const timeout = 60000;- let singleAnswerDocs: Document[] = [];+ // Scrape documents+ const timeout = 60000;+ let singleAnswerDocs: Document[] = [];- // let rerank = await rerankLinks(links.map((url) => ({ url })), request.prompt ?? JSON.stringify(request.schema), urlTraces);+ // let rerank = await rerankLinks(links.map((url) => ({ url })), request.prompt ?? JSON.stringify(request.schema), urlTraces);- await updateExtract(extractId, {- status: "processing",- steps: [- {- step: ExtractStep.SCRAPE,- startedAt: Date.now(),- finishedAt: Date.now(),- discoveredLinks: links,- },- ],- });- log["docsSizeBeforeSingleEntityScrape"] = docsMap.size;- const scrapePromises = links.map((url) => {- if (!docsMap.has(normalizeUrl(url))) {- return scrapeDocument(+ await updateExtract(extractId, {+ status: "processing",+ steps: [{- url,- teamId,- origin: request.origin || "api",- timeout,+ step: ExtractStep.SCRAPE,+ startedAt: Date.now(),+ finishedAt: Date.now(),+ discoveredLinks: links,},- urlTraces,- logger.child({- module: "extract",- method: "scrapeDocument",- url,- isMultiEntity: false,- }),- request.scrapeOptions,- );- }- return docsMap.get(normalizeUrl(url));- });+ ],+ });+ log["docsSizeBeforeSingleEntityScrape"] = docsMap.size;+ const scrapePromises = links.map((url) => {+ if (!docsMap.has(normalizeUrl(url))) {+ return scrapeDocument(+ {+ url,+ teamId,+ origin: request.origin || "api",+ timeout,+ },+ urlTraces,+ logger.child({+ module: "extract",+ method: "scrapeDocument",+ url,+ isMultiEntity: false,+ }),+ request.scrapeOptions,+ );+ }+ return docsMap.get(normalizeUrl(url));+ });- try {- const results = await Promise.all(scrapePromises);- log["docsSizeAfterSingleEntityScrape"] = docsMap.size;+ try {+ const results = await Promise.all(scrapePromises);+ log["docsSizeAfterSingleEntityScrape"] = docsMap.size;- for (const doc of results) {- if (doc?.metadata?.url) {- docsMap.set(normalizeUrl(doc.metadata.url), doc);+ for (const doc of results) {+ if (doc?.metadata?.url) {+ docsMap.set(normalizeUrl(doc.metadata.url), doc);+ }}+ logger.debug("Updated docsMap.", { docsMapSize: docsMap.size }); // useful for error probing++ const validResults = results.filter(+ (doc): doc is Document => doc !== null,+ );+ singleAnswerDocs.push(...validResults);+ totalUrlsScraped += validResults.length;++ logger.debug("Scrapes finished.", { docCount: validResults.length });+ } catch (error) {+ return {+ success: false,+ error: error.message,+ extractId,+ urlTrace: urlTraces,+ totalUrlsScraped,+ };}- logger.debug("Updated docsMap.", { docsMapSize: docsMap.size }); // useful for error probing- const validResults = results.filter(- (doc): doc is Document => doc !== null,- );- singleAnswerDocs.push(...validResults);- totalUrlsScraped += validResults.length;+ if (docsMap.size == 0) {+ // All urls are invalid+ logger.error("All provided URLs are invalid!");+ return {+ success: false,+ error:+ "All provided URLs are invalid. Please check your input and try again.",+ extractId,+ urlTrace: request.urlTrace ? urlTraces : undefined,+ totalUrlsScraped: 0,+ };+ }- logger.debug("Scrapes finished.", { docCount: validResults.length });- } catch (error) {- return {- success: false,- error: error.message,- extractId,- urlTrace: urlTraces,- totalUrlsScraped,- };- }+ await updateExtract(extractId, {+ status: "processing",+ steps: [+ {+ step: ExtractStep.EXTRACT,+ startedAt: Date.now(),+ finishedAt: Date.now(),+ discoveredLinks: links,+ },+ ],+ });- if (docsMap.size == 0) {- // All urls are invalid- logger.error("All provided URLs are invalid!");- return {- success: false,- error:- "All provided URLs are invalid. Please check your input and try again.",+ // Generate completions+ logger.debug("Generating singleAnswer completions...");+ log["singleAnswerDocsLength"] = singleAnswerDocs.length;+ let {+ extract: completionResult,+ tokenUsage: singleAnswerTokenUsage,+ sources: singleAnswerSources,+ smartScrapeCost: singleAnswerSmartScrapeCost,+ otherCost: singleAnswerOtherCost,+ smartScrapeCallCount: singleAnswerSmartScrapeCallCount,+ otherCallCount: singleAnswerOtherCallCount,+ } = await singleAnswerCompletion({+ singleAnswerDocs,+ rSchema,+ links,+ prompt: request.prompt ?? "",+ systemPrompt: request.systemPrompt ?? "",+ useAgent: isAgentExtractModelValid(request.agent?.model),extractId,- urlTrace: request.urlTrace ? urlTraces : undefined,- totalUrlsScraped: 0,- };+ });+ costTracking.smartScrapeCost += singleAnswerSmartScrapeCost;+ costTracking.smartScrapeCallCount += singleAnswerSmartScrapeCallCount;+ costTracking.otherCost += singleAnswerOtherCost;+ costTracking.otherCallCount += singleAnswerOtherCallCount;+ costTracking.totalCost += singleAnswerSmartScrapeCost + singleAnswerOtherCost;+ logger.debug("Done generating singleAnswer completions.");++ singleAnswerResult = transformArrayToObject(rSchema, completionResult);++ singleAnswerResult = deduplicateObjectsArray(singleAnswerResult);+ // Track single answer extraction tokens and sources+ if (completionResult) {+ tokenUsage.push(singleAnswerTokenUsage);++ // Add sources for top-level properties in single answer+ if (rSchema?.properties) {+ Object.keys(rSchema.properties).forEach((key) => {+ if (completionResult[key] !== undefined) {+ sources[key] =+ singleAnswerSources ||+ singleAnswerDocs.map(+ (doc) => doc.metadata.url || doc.metadata.sourceURL || "",+ );+ }+ });+ }+ }++ // singleAnswerResult = completionResult;+ // singleAnswerCompletions = singleAnswerResult;++ // Update token usage in traces+ // if (completions && completions.numTokens) {+ // const totalLength = docs.reduce(+ // (sum, doc) => sum + (doc.markdown?.length || 0),+ // 0,+ // );+ // docs.forEach((doc) => {+ // if (doc.metadata?.sourceURL) {+ // const trace = urlTraces.find((t) => t.url === doc.metadata.sourceURL);+ // if (trace && trace.contentStats) {+ // trace.contentStats.tokensUsed = Math.floor(+ // ((doc.markdown?.length || 0) / totalLength) *+ // (completions?.numTokens || 0),+ // );+ // }+ // }+ // });+ // }}- await updateExtract(extractId, {- status: "processing",- steps: [- {- step: ExtractStep.EXTRACT,- startedAt: Date.now(),- finishedAt: Date.now(),- discoveredLinks: links,- },- ],- });+ log["singleAnswerResult"] = singleAnswerResult;+ log["multiEntityResult"] = multiEntityResult;- // Generate completions- logger.debug("Generating singleAnswer completions...");- log["singleAnswerDocsLength"] = singleAnswerDocs.length;- let {- extract: completionResult,- tokenUsage: singleAnswerTokenUsage,- sources: singleAnswerSources,- smartScrapeCost: singleAnswerSmartScrapeCost,- otherCost: singleAnswerOtherCost,- smartScrapeCallCount: singleAnswerSmartScrapeCallCount,- otherCallCount: singleAnswerOtherCallCount,- } = await singleAnswerCompletion({- singleAnswerDocs,- rSchema,- links,- prompt: request.prompt ?? "",- systemPrompt: request.systemPrompt ?? "",- useAgent: isAgentExtractModelValid(request.agent?.model),- extractId,- });- costTracking.smartScrapeCost += singleAnswerSmartScrapeCost;- costTracking.smartScrapeCallCount += singleAnswerSmartScrapeCallCount;- costTracking.otherCost += singleAnswerOtherCost;- costTracking.otherCallCount += singleAnswerOtherCallCount;- costTracking.totalCost += singleAnswerSmartScrapeCost + singleAnswerOtherCost;- logger.debug("Done generating singleAnswer completions.");-- singleAnswerResult = transformArrayToObject(rSchema, completionResult);-- singleAnswerResult = deduplicateObjectsArray(singleAnswerResult);- // Track single answer extraction tokens and sources- if (completionResult) {- tokenUsage.push(singleAnswerTokenUsage);-- // Add sources for top-level properties in single answer- if (rSchema?.properties) {- Object.keys(rSchema.properties).forEach((key) => {- if (completionResult[key] !== undefined) {- sources[key] =- singleAnswerSources ||- singleAnswerDocs.map(- (doc) => doc.metadata.url || doc.metadata.sourceURL || "",- );- }- });- }- }+ let finalResult = reqSchema+ ? await mixSchemaObjects(+ reqSchema,+ singleAnswerResult,+ multiEntityResult,+ logger.child({ method: "mixSchemaObjects" }),+ )+ : singleAnswerResult || multiEntityResult;- // singleAnswerResult = completionResult;- // singleAnswerCompletions = singleAnswerResult;+ // Tokenize final result to get token count+ // let finalResultTokens = 0;+ // if (finalResult) {+ // const finalResultStr = JSON.stringify(finalResult);+ // finalResultTokens = numTokensFromString(finalResultStr, "gpt-4o");- // Update token usage in traces- // if (completions && completions.numTokens) {- // const totalLength = docs.reduce(- // (sum, doc) => sum + (doc.markdown?.length || 0),- // 0,- // );- // docs.forEach((doc) => {- // if (doc.metadata?.sourceURL) {- // const trace = urlTraces.find((t) => t.url === doc.metadata.sourceURL);- // if (trace && trace.contentStats) {- // trace.contentStats.tokensUsed = Math.floor(- // ((doc.markdown?.length || 0) / totalLength) *- // (completions?.numTokens || 0),- // );- // }- // }- // });// }- }+ // // Deduplicate and validate final result against schema+ // if (reqSchema && finalResult && finalResult.length <= extractConfig.DEDUPLICATION.MAX_TOKENS) {+ // const schemaValidation = await generateCompletions(+ // logger.child({ method: "extractService/validateAndDeduplicate" }),+ // {+ // mode: "llm",+ // systemPrompt: `You are a data validator and deduplicator. Your task is to:+ // 1. Remove any duplicate entries in the data extracted by merging that into a single object according to the provided shcema+ // 2. Ensure all data matches the provided schema+ // 3. Keep only the highest quality and most complete entries when duplicates are found.++ // Do not change anything else. If data is null keep it null. If the schema is not provided, return the data as is.`,+ // prompt: `Please validate and merge the duplicate entries in this data according to the schema provided:\n++ // <start of extract data>++ // ${JSON.stringify(finalResult)}++ // <end of extract data>++ // <start of schema>++ // ${JSON.stringify(reqSchema)}++ // <end of schema>+ // `,+ // schema: reqSchema,+ // },+ // undefined,+ // undefined,+ // true,+ // "gpt-4o"+ // );+ // console.log("schemaValidation", schemaValidation);- log["singleAnswerResult"] = singleAnswerResult;- log["multiEntityResult"] = multiEntityResult;+ // console.log("schemaValidation", finalResult);- let finalResult = reqSchema- ? await mixSchemaObjects(- reqSchema,- singleAnswerResult,- multiEntityResult,- logger.child({ method: "mixSchemaObjects" }),- )- : singleAnswerResult || multiEntityResult;-- // Tokenize final result to get token count- // let finalResultTokens = 0;- // if (finalResult) {- // const finalResultStr = JSON.stringify(finalResult);- // finalResultTokens = numTokensFromString(finalResultStr, "gpt-4o");-- // }- // // Deduplicate and validate final result against schema- // if (reqSchema && finalResult && finalResult.length <= extractConfig.DEDUPLICATION.MAX_TOKENS) {- // const schemaValidation = await generateCompletions(- // logger.child({ method: "extractService/validateAndDeduplicate" }),- // {- // mode: "llm",- // systemPrompt: `You are a data validator and deduplicator. Your task is to:- // 1. Remove any duplicate entries in the data extracted by merging that into a single object according to the provided shcema- // 2. Ensure all data matches the provided schema- // 3. Keep only the highest quality and most complete entries when duplicates are found.-- // Do not change anything else. If data is null keep it null. If the schema is not provided, return the data as is.`,- // prompt: `Please validate and merge the duplicate entries in this data according to the schema provided:\n-- // <start of extract data>-- // ${JSON.stringify(finalResult)}-- // <end of extract data>-- // <start of schema>-- // ${JSON.stringify(reqSchema)}-- // <end of schema>- // `,- // schema: reqSchema,- // },- // undefined,- // undefined,- // true,- // "gpt-4o"- // );- // console.log("schemaValidation", schemaValidation);-- // console.log("schemaValidation", finalResult);-- // if (schemaValidation?.extract) {- // tokenUsage.push(schemaValidation.totalUsage);- // finalResult = schemaValidation.extract;- // }- // }-- const totalTokensUsed = tokenUsage.reduce((a, b) => a + b.totalTokens, 0);- const llmUsage = estimateTotalCost(tokenUsage);- let tokensToBill = calculateFinalResultCost(finalResult);-- if (CUSTOM_U_TEAMS.includes(teamId)) {- tokensToBill = 1;- }+ // if (schemaValidation?.extract) {+ // tokenUsage.push(schemaValidation.totalUsage);+ // finalResult = schemaValidation.extract;+ // }+ // }- // Bill team for usage- billTeam(teamId, subId, tokensToBill, logger, true).catch((error) => {- logger.error(- `Failed to bill team ${teamId} for ${tokensToBill} tokens: ${error}`,- );- });+ const totalTokensUsed = tokenUsage.reduce((a, b) => a + b.totalTokens, 0);+ const llmUsage = estimateTotalCost(tokenUsage);+ let tokensToBill = calculateFinalResultCost(finalResult);- // Log job with token usage and sources- logJob({- job_id: extractId,- success: true,- message: "Extract completed",- num_docs: 1,- docs: finalResult ?? {},- time_taken: (new Date().getTime() - Date.now()) / 1000,- team_id: teamId,- mode: "extract",- url: request.urls?.join(", ") || "",- scrapeOptions: request,- origin: request.origin ?? "api",- num_tokens: totalTokensUsed,- tokens_billed: tokensToBill,- sources,- cost_tracking: costTracking,- }).then(() => {- updateExtract(extractId, {- status: "completed",- llmUsage,- sources,- costTracking,- }).catch((error) => {+ if (CUSTOM_U_TEAMS.includes(teamId)) {+ tokensToBill = 1;+ }++ // Bill team for usage+ billTeam(teamId, subId, tokensToBill, logger, true).catch((error) => {logger.error(- `Failed to update extract ${extractId} status to completed: ${error}`,+ `Failed to bill team ${teamId} for ${tokensToBill} tokens: ${error}`,);});- });- logger.debug("Done!");-- if (- request.__experimental_cacheMode == "save" &&- request.__experimental_cacheKey- ) {- logger.debug("Saving cached docs...");- try {- await saveCachedDocs(- [...docsMap.values()],- request.__experimental_cacheKey,- );- } catch (error) {- logger.error("Error saving cached docs", { error });+ // Log job with token usage and sources+ logJob({+ job_id: extractId,+ success: true,+ message: "Extract completed",+ num_docs: 1,+ docs: finalResult ?? {},+ time_taken: (new Date().getTime() - Date.now()) / 1000,+ team_id: teamId,+ mode: "extract",+ url: request.urls?.join(", ") || "",+ scrapeOptions: request,+ origin: request.origin ?? "api",+ num_tokens: totalTokensUsed,+ tokens_billed: tokensToBill,+ sources,+ cost_tracking: costTracking,+ }).then(() => {+ updateExtract(extractId, {+ status: "completed",+ llmUsage,+ sources,+ costTracking,+ }).catch((error) => {+ logger.error(+ `Failed to update extract ${extractId} status to completed: ${error}`,+ );+ });+ });++ logger.debug("Done!");++ if (+ request.__experimental_cacheMode == "save" &&+ request.__experimental_cacheKey+ ) {+ logger.debug("Saving cached docs...");+ try {+ await saveCachedDocs(+ [...docsMap.values()],+ request.__experimental_cacheKey,+ );+ } catch (error) {+ logger.error("Error saving cached docs", { error });+ }}- }- // fs.writeFile(- // `logs/${request.urls?.[0].replaceAll("https://", "").replaceAll("http://", "").replaceAll("/", "-").replaceAll(".", "-")}-extract-${extractId}.json`,- // JSON.stringify(log, null, 2),- // );+ // fs.writeFile(+ // `logs/${request.urls?.[0].replaceAll("https://", "").replaceAll("http://", "").replaceAll("/", "-").replaceAll(".", "-")}-extract-${extractId}.json`,+ // JSON.stringify(log, null, 2),+ // );- return {- success: true,- data: finalResult ?? {},- extractId,- warning: undefined,- urlTrace: request.urlTrace ? urlTraces : undefined,- llmUsage,- totalUrlsScraped,- sources,- };+ return {+ success: true,+ data: finalResult ?? {},+ extractId,+ warning: undefined,+ urlTrace: request.urlTrace ? urlTraces : undefined,+ llmUsage,+ totalUrlsScraped,+ sources,+ };+ } catch (error) {+ await logJob({+ job_id: extractId,+ success: false,+ message: (error instanceof Error ? error.message : typeof error === "string" ? error : "An unexpected error occurred"),+ num_docs: 1,+ docs: [],+ time_taken: (new Date().getTime() - Date.now()) / 1000,+ team_id: teamId,+ mode: "extract",+ url: request.urls?.join(", ") || "",+ scrapeOptions: request,+ origin: request.origin ?? "api",+ num_tokens: 0,+ tokens_billed: 0,+ sources,+ cost_tracking: costTracking,+ });+ throw error;+ }}