Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/otel): much improved tracing (#2219)
Gergő Móricz2bf96926497e
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 69245b839..0b6181a09 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -77,6 +77,7 @@"@google-cloud/storage": "^7.16.0","@mendable/firecrawl-rs": "workspace:*","@openrouter/ai-sdk-provider": "^0.4.5",+ "@opentelemetry/api": "^1.9.0","@opentelemetry/auto-instrumentations-node": "^0.62.0","@opentelemetry/core": "^2.0.1","@opentelemetry/exporter-trace-otlp-http": "^0.205.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 6e7dc8b3d..afcfca3f7 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -60,6 +60,9 @@ importers:'@openrouter/ai-sdk-provider':specifier: ^0.4.5version: 0.4.5([email protected])+ '@opentelemetry/api':+ specifier: ^1.9.0+ version: 1.9.0'@opentelemetry/auto-instrumentations-node':specifier: ^0.62.0version: 0.62.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected])diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 22d69d7a9..48d662c0a 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -14,161 +14,262 @@ import { hasFormatOfType } from "../../lib/format-utils";import { TransportableError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";+import { withSpan, setSpanAttributes, SpanKind } from "../../lib/otel-tracer";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,res: Response<ScrapeResponse>,) {- // Get timing data from middleware (includes all middleware processing time)- const middlewareStartTime = (req as any).requestTiming?.startTime || new Date().getTime();- const controllerStartTime = new Date().getTime();-- const jobId = uuidv4();- const preNormalizedBody = { ...req.body };- req.body = scrapeRequestSchema.parse(req.body);-- const permissions = checkPermissions(req.body, req.acuc?.flags);- if (permissions.error) {- return res.status(403).json({- success: false,- error: permissions.error,- });- }-- const zeroDataRetention =- req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;-- const logger = _logger.child({- method: "scrapeController",- jobId,- scrapeId: jobId,- teamId: req.auth.team_id,- team_id: req.auth.team_id,- zeroDataRetention,- });-- const middlewareTime = controllerStartTime - middlewareStartTime;-- logger.debug("Scrape " + jobId + " starting", {- version: "v2",- scrapeId: jobId,- request: req.body,- originalRequest: preNormalizedBody,- account: req.account,- });-- const origin = req.body.origin;- const timeout = req.body.timeout;--- const isDirectToBullMQ =- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;-- const jobPriority = await getJobPriority({- team_id: req.auth.team_id,- basePriority: 10,- });-- const job = await addScrapeJob(- {- url: req.body.url,- mode: "single_urls",- team_id: req.auth.team_id,- scrapeOptions: {- ...req.body,- ...(req.body.__experimental_cache- ? {- maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours- }- : {}),- },- internalOptions: {+ return withSpan(+ "api.scrape.request",+ async span => {+ // Get timing data from middleware (includes all middleware processing time)+ const middlewareStartTime =+ (req as any).requestTiming?.startTime || new Date().getTime();+ const controllerStartTime = new Date().getTime();++ const jobId = uuidv4();+ const preNormalizedBody = { ...req.body };++ // Set initial span attributes+ setSpanAttributes(span, {+ "scrape.job_id": jobId,+ "scrape.url": req.body.url,+ "scrape.team_id": req.auth.team_id,+ "scrape.api_key_id": req.acuc?.api_key_id,+ "scrape.middleware_time_ms": controllerStartTime - middlewareStartTime,+ });++ // Validation span+ await withSpan("api.scrape.validate", async validateSpan => {+ req.body = scrapeRequestSchema.parse(req.body);+ setSpanAttributes(validateSpan, {+ "validation.success": true,+ });+ });++ // Permission check span+ const permissions = await withSpan(+ "api.scrape.check_permissions",+ async permSpan => {+ const perms = checkPermissions(req.body, req.acuc?.flags);+ setSpanAttributes(permSpan, {+ "permissions.success": !perms.error,+ "permissions.error": perms.error,+ });+ return perms;+ },+ );++ if (permissions.error) {+ setSpanAttributes(span, {+ "scrape.error": permissions.error,+ "scrape.status_code": 403,+ });+ return res.status(403).json({+ success: false,+ error: permissions.error,+ });+ }++ const zeroDataRetention =+ req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;++ const logger = _logger.child({+ method: "scrapeController",+ jobId,+ scrapeId: jobId,teamId: req.auth.team_id,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME- ? true- : false,- unnormalizedSourceURL: preNormalizedBody.url,- bypassBilling: isDirectToBullMQ,+ team_id: req.auth.team_id,zeroDataRetention,- teamFlags: req.acuc?.flags ?? null,- },- origin,- integration: req.body.integration,- startTime: controllerStartTime,- zeroDataRetention,- apiKeyId: req.acuc?.api_key_id ?? null,- },- jobId,- jobPriority,- isDirectToBullMQ,- );+ });++ const middlewareTime = controllerStartTime - middlewareStartTime;++ logger.debug("Scrape " + jobId + " starting", {+ version: "v2",+ scrapeId: jobId,+ request: req.body,+ originalRequest: preNormalizedBody,+ account: req.account,+ });++ setSpanAttributes(span, {+ "scrape.zero_data_retention": zeroDataRetention,+ "scrape.origin": req.body.origin,+ "scrape.timeout": req.body.timeout,+ });++ const origin = req.body.origin;+ const timeout = req.body.timeout;++ const isDirectToBullMQ =+ process.env.SEARCH_PREVIEW_TOKEN !== undefined &&+ process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;++ const jobPriority = await getJobPriority({+ team_id: req.auth.team_id,+ basePriority: 10,+ });++ // Job enqueuing span+ const job = await withSpan(+ "api.scrape.enqueue_job",+ async enqueueSpan => {+ const result = await addScrapeJob(+ {+ url: req.body.url,+ mode: "single_urls",+ team_id: req.auth.team_id,+ scrapeOptions: {+ ...req.body,+ ...(req.body.__experimental_cache+ ? {+ maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours+ }+ : {}),+ },+ internalOptions: {+ teamId: req.auth.team_id,+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME+ ? true+ : false,+ unnormalizedSourceURL: preNormalizedBody.url,+ bypassBilling: isDirectToBullMQ,+ zeroDataRetention,+ teamFlags: req.acuc?.flags ?? null,+ },+ origin,+ integration: req.body.integration,+ startTime: controllerStartTime,+ zeroDataRetention,+ apiKeyId: req.acuc?.api_key_id ?? null,+ },+ jobId,+ jobPriority,+ isDirectToBullMQ,+ );++ setSpanAttributes(enqueueSpan, {+ "job.priority": jobPriority,+ "job.direct_to_bullmq": isDirectToBullMQ,+ });++ return result;+ },+ );++ const totalWait =+ (req.body.waitFor ?? 0) ++ (req.body.actions ?? []).reduce(+ (a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,+ 0,+ );++ let doc: Document;+ try {+ // Wait for job completion span+ doc = await withSpan("api.scrape.wait_for_job", async waitSpan => {+ setSpanAttributes(waitSpan, {+ "wait.timeout": timeout !== undefined ? timeout + totalWait : null,+ "wait.job_id": jobId,+ });++ const result = await waitForJob(+ job ?? jobId,+ timeout !== undefined ? timeout + totalWait : null,+ zeroDataRetention,+ logger,+ );++ setSpanAttributes(waitSpan, {+ "wait.success": true,+ });++ return result;+ });+ } catch (e) {+ logger.error(`Error in scrapeController`, {+ version: "v2",+ error: e,+ });++ setSpanAttributes(span, {+ "scrape.error": e instanceof Error ? e.message : String(e),+ "scrape.error_type":+ e instanceof TransportableError ? e.code : "unknown",+ });++ if (zeroDataRetention) {+ await scrapeQueue.removeJob(jobId, logger);+ }++ if (e instanceof TransportableError) {+ const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;+ setSpanAttributes(span, {+ "scrape.status_code": statusCode,+ });+ return res.status(statusCode).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ } else {+ setSpanAttributes(span, {+ "scrape.status_code": 500,+ });+ return res.status(500).json({+ success: false,+ error: `(Internal server error) - ${e && e.message ? e.message : e}`,+ });+ }+ }- const totalWait =- (req.body.waitFor ?? 0) +- (req.body.actions ?? []).reduce(- (a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,- 0,- );-- let doc: Document;- try {- doc = await waitForJob(- job ?? jobId,- timeout !== undefined ? timeout + totalWait : null,- zeroDataRetention,- logger,- );- } catch (e) {- logger.error(`Error in scrapeController`, {- version: "v2",- error: e,- });-- if (zeroDataRetention) {await scrapeQueue.removeJob(jobId, logger);- }- if (e instanceof TransportableError) {- return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({- success: false,- code: e.code,- error: e.message,+ if (!hasFormatOfType(req.body.formats, "rawHtml")) {+ if (doc && doc.rawHtml) {+ delete doc.rawHtml;+ }+ }++ const totalRequestTime = new Date().getTime() - middlewareStartTime;+ const controllerTime = new Date().getTime() - controllerStartTime;++ // Set final span attributes+ setSpanAttributes(span, {+ "scrape.success": true,+ "scrape.status_code": 200,+ "scrape.total_request_time_ms": totalRequestTime,+ "scrape.controller_time_ms": controllerTime,+ "scrape.document.status_code": doc?.metadata?.statusCode,+ "scrape.document.content_type": doc?.metadata?.contentType,+ "scrape.document.error": doc?.metadata?.error,});- } else {- return res.status(500).json({- success: false,- error: `(Internal server error) - ${e && e.message ? e.message : e}`,++ logger.info("Request metrics", {+ version: "v2",+ scrapeId: jobId,+ mode: "scrape",+ middlewareStartTime,+ controllerStartTime,+ middlewareTime,+ controllerTime,+ totalRequestTime,+ });++ return res.status(200).json({+ success: true,+ data: doc,+ scrape_id: origin?.includes("website") ? jobId : undefined,});- }- }-- await scrapeQueue.removeJob(jobId, logger);--- if (!hasFormatOfType(req.body.formats, "rawHtml")) {- if (doc && doc.rawHtml) {- delete doc.rawHtml;- }- }-- const totalRequestTime = new Date().getTime() - middlewareStartTime;- const controllerTime = new Date().getTime() - controllerStartTime;- logger.info("Request metrics", {- version: "v2",- scrapeId: jobId,- mode: "scrape",- middlewareStartTime,- controllerStartTime,- middlewareTime,- controllerTime,- totalRequestTime,- });--- return res.status(200).json({- success: true,- data: doc,- scrape_id: origin?.includes("website") ? jobId : undefined,- });+ },+ {+ attributes: {+ "http.method": "POST",+ "http.route": "/v2/scrape",+ },+ kind: SpanKind.SERVER,+ },+ );}diff --git a/apps/api/src/lib/otel-tracer.ts b/apps/api/src/lib/otel-tracer.tsnew file mode 100644index 000000000..3ef40ae7b--- /dev/null+++ b/apps/api/src/lib/otel-tracer.ts@@ -0,0 +1,173 @@+import {+ trace,+ context,+ SpanStatusCode,+ Span,+ SpanKind,+ Attributes,+ propagation,+} from "@opentelemetry/api";+// import { Logger } from 'winston';++const tracer = trace.getTracer("firecrawl-api", "1.0.0");++export { SpanKind };++// Trace context propagation utilities+export interface SerializedTraceContext {+ traceParent?: string;+ traceState?: string;+}++export function serializeTraceContext(): SerializedTraceContext {+ const carrier: Record<string, string> = {};+ propagation.inject(context.active(), carrier);++ return {+ traceParent: carrier.traceparent,+ traceState: carrier.tracestate,+ };+}++function deserializeTraceContext(serialized: SerializedTraceContext): any {+ if (!serialized.traceParent) {+ return context.active();+ }++ const carrier: Record<string, string> = {+ traceparent: serialized.traceParent,+ };++ if (serialized.traceState) {+ carrier.tracestate = serialized.traceState;+ }++ return propagation.extract(context.active(), carrier);+}++// export function withTraceContext<T>(serializedContext: SerializedTraceContext, fn: () => T): T {+// const ctx = deserializeTraceContext(serializedContext);+// return context.with(ctx, fn);+// }++export async function withTraceContextAsync<T>(+ serializedContext: SerializedTraceContext,+ fn: () => Promise<T>,+): Promise<T> {+ const ctx = deserializeTraceContext(serializedContext);+ return context.with(ctx, fn);+}++interface SpanOptions {+ attributes?: Attributes;+ kind?: SpanKind;+}++function startSpan(name: string, options?: SpanOptions): Span {+ return tracer.startSpan(name, {+ attributes: options?.attributes,+ kind: options?.kind || SpanKind.INTERNAL,+ });+}++export async function withSpan<T>(+ name: string,+ fn: (span: Span) => Promise<T>,+ options?: SpanOptions,+): Promise<T> {+ const span = startSpan(name, options);++ try {+ const result = await context.with(+ trace.setSpan(context.active(), span),+ () => fn(span),+ );+ span.setStatus({ code: SpanStatusCode.OK });+ return result;+ } catch (error) {+ span.setStatus({+ code: SpanStatusCode.ERROR,+ message: error instanceof Error ? error.message : String(error),+ });++ if (error instanceof Error) {+ span.recordException(error);+ }++ throw error;+ } finally {+ span.end();+ }+}++// export function withSpanSync<T>(+// name: string,+// fn: (span: Span) => T,+// options?: SpanOptions+// ): T {+// const span = startSpan(name, options);++// try {+// const result = context.with(+// trace.setSpan(context.active(), span),+// () => fn(span)+// );+// span.setStatus({ code: SpanStatusCode.OK });+// return result;+// } catch (error) {+// span.setStatus({+// code: SpanStatusCode.ERROR,+// message: error instanceof Error ? error.message : String(error),+// });++// if (error instanceof Error) {+// span.recordException(error);+// }++// throw error;+// } finally {+// span.end();+// }+// }++export function setSpanAttributes(span: Span, attributes: Attributes): void {+ Object.entries(attributes).forEach(([key, value]) => {+ if (value !== undefined && value !== null) {+ span.setAttribute(key, value);+ }+ });+}++// export function setSpanError(span: Span, error: Error | unknown, logger?: Logger): void {+// const errorMessage = error instanceof Error ? error.message : String(error);++// span.setStatus({+// code: SpanStatusCode.ERROR,+// message: errorMessage,+// });++// if (error instanceof Error) {+// span.recordException(error);+// span.setAttributes({+// 'error.type': error.constructor.name,+// 'error.message': error.message,+// 'error.stack': error.stack,+// });+// }++// if (logger) {+// logger.error('Span error recorded', { error, spanName: span.spanContext().traceId });+// }+// }++// export function getActiveSpan(): Span | undefined {+// return trace.getActiveSpan();+// }++// export function createChildSpan(name: string, parentSpan?: Span, options?: SpanOptions): Span {+// const ctx = parentSpan+// ? trace.setSpan(context.active(), parentSpan)+// : context.active();++// return context.with(ctx, () => startSpan(name, options));+// }diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 116768348..b8d575112 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -32,6 +32,7 @@ import { hasFormatOfType } from "../../../../lib/format-utils";import { Action } from "../../../../controllers/v1/types";import { AbortManagerThrownError } from "../../lib/abortManager";import { youtubePostprocessor } from "../../postprocessors/youtube";+import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";// This function does not take `Meta` on purpose. It may not access any// meta values to construct the request -- that must be done by the@@ -49,407 +50,457 @@ async function performFireEngineScrape<abort?: AbortSignal,production = true,): Promise<FireEngineCheckStatusSuccess> {- const scrape = await fireEngineScrape(- meta,- logger.child({ method: "fireEngineScrape" }),- request,- mock,- abort,- production,- );-- let status: FireEngineCheckStatusSuccess | undefined = undefined;- if ((scrape as any).processing) {- const errorLimit = 3;- let errors: any[] = [];-- while (status === undefined) {- if (errors.length >= errorLimit) {- logger.error("Error limit hit.", { errors });- fireEngineDelete(- logger.child({- method: "performFireEngineScrape/fireEngineDelete",- afterErrors: errors,- }),- (scrape as any).jobId,- mock,- undefined,- production,- );- throw new Error("Error limit hit. See e.cause.errors for errors.", {- cause: { errors },- });- }+ return withSpan("engine.fire-engine.perform_scrape", async span => {+ const startTime = Date.now();+ let pollCount = 0;++ setSpanAttributes(span, {+ "fire-engine.url": request.url,+ "fire-engine.priority": request.priority,+ "fire-engine.wait": (request as any).wait,+ "fire-engine.screenshot": (request as any).screenshot,+ "fire-engine.fullpage": (request as any).fullPage,+ "fire-engine.proxy": (request as any).proxy,+ "fire-engine.mobile": (request as any).mobile,+ "fire-engine.skip_tls": (request as any).skipTlsVerification,+ "fire-engine.production": production,+ });+ const scrape = await fireEngineScrape(+ meta,+ logger.child({ method: "fireEngineScrape" }),+ request,+ mock,+ abort,+ production,+ );++ let status: FireEngineCheckStatusSuccess | undefined = undefined;+ if ((scrape as any).processing) {+ const errorLimit = 3;+ let errors: any[] = [];- meta.abort.throwIfAborted();-- try {- status = await fireEngineCheckStatus(- meta,- logger.child({ method: "fireEngineCheckStatus" }),- (scrape as any).jobId,- mock,- abort,- production,- );- } catch (error) {- if (error instanceof StillProcessingError) {- // nop- } else if (- error instanceof EngineError ||- error instanceof SiteError ||- error instanceof SSLError ||- error instanceof DNSResolutionError ||- error instanceof ActionError ||- error instanceof UnsupportedFileError ||- error instanceof FEPageLoadFailed ||- error instanceof ProxySelectionError- ) {+ while (status === undefined) {+ if (errors.length >= errorLimit) {+ logger.error("Error limit hit.", { errors });fireEngineDelete(logger.child({method: "performFireEngineScrape/fireEngineDelete",- afterError: error,+ afterErrors: errors,}),(scrape as any).jobId,mock,undefined,production,);- logger.debug("Fire-engine scrape job failed.", {- error,- jobId: (scrape as any).jobId,+ throw new Error("Error limit hit. See e.cause.errors for errors.", {+ cause: { errors },});- throw error;- } else if (error instanceof AbortManagerThrownError) {- fireEngineDelete(- logger.child({- method: "performFireEngineScrape/fireEngineDelete",- afterError: error,- }),+ }++ meta.abort.throwIfAborted();++ try {+ pollCount++;+ status = await fireEngineCheckStatus(+ meta,+ logger.child({ method: "fireEngineCheckStatus" }),(scrape as any).jobId,mock,- undefined,+ abort,production,);- throw error;- } else {- errors.push(error);- logger.debug(- `An unexpeceted error occurred while calling checkStatus. Error counter is now at ${errors.length}.`,- { error, jobId: (scrape as any).jobId },- );- Sentry.captureException(error);+ } catch (error) {+ if (error instanceof StillProcessingError) {+ // nop+ } else if (+ error instanceof EngineError ||+ error instanceof SiteError ||+ error instanceof SSLError ||+ error instanceof DNSResolutionError ||+ error instanceof ActionError ||+ error instanceof UnsupportedFileError ||+ error instanceof FEPageLoadFailed ||+ error instanceof ProxySelectionError+ ) {+ fireEngineDelete(+ logger.child({+ method: "performFireEngineScrape/fireEngineDelete",+ afterError: error,+ }),+ (scrape as any).jobId,+ mock,+ undefined,+ production,+ );+ logger.debug("Fire-engine scrape job failed.", {+ error,+ jobId: (scrape as any).jobId,+ });+ throw error;+ } else if (error instanceof AbortManagerThrownError) {+ fireEngineDelete(+ logger.child({+ method: "performFireEngineScrape/fireEngineDelete",+ afterError: error,+ }),+ (scrape as any).jobId,+ mock,+ undefined,+ production,+ );+ throw error;+ } else {+ errors.push(error);+ logger.debug(+ `An unexpeceted error occurred while calling checkStatus. Error counter is now at ${errors.length}.`,+ { error, jobId: (scrape as any).jobId },+ );+ Sentry.captureException(error);+ }}}++ await new Promise(resolve => setTimeout(resolve, 500));+ } else {+ status = scrape as FireEngineCheckStatusSuccess;}- await new Promise(resolve => setTimeout(resolve, 500));- } else {- status = scrape as FireEngineCheckStatusSuccess;- }+ await specialtyScrapeCheck(+ logger.child({+ method: "performFireEngineScrape/specialtyScrapeCheck",+ }),+ status.responseHeaders,+ status,+ );- await specialtyScrapeCheck(- logger.child({- method: "performFireEngineScrape/specialtyScrapeCheck",- }),- status.responseHeaders,- status,- );-- const contentType =- (Object.entries(status.responseHeaders ?? {}).find(- x => x[0].toLowerCase() === "content-type",- ) ?? [])[1] ?? "";-- if (contentType.includes("application/json")) {- status.content = await getInnerJson(status.content);- }+ const contentType =+ (Object.entries(status.responseHeaders ?? {}).find(+ x => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? "";- if (status.file) {- const content = status.file.content;- delete status.file;- status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag- }+ if (contentType.includes("application/json")) {+ status.content = await getInnerJson(status.content);+ }- fireEngineDelete(- logger.child({- method: "performFireEngineScrape/fireEngineDelete",- }),- (scrape as any).jobId,- mock,- undefined,- production,- );-- return status;+ if (status.file) {+ const content = status.file.content;+ delete status.file;+ status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag+ }++ fireEngineDelete(+ logger.child({+ method: "performFireEngineScrape/fireEngineDelete",+ }),+ (scrape as any).jobId,+ mock,+ undefined,+ production,+ );++ setSpanAttributes(span, {+ "fire-engine.poll_count": pollCount,+ "fire-engine.duration_ms": Date.now() - startTime,+ "fire-engine.status_code": status.pageStatusCode,+ "fire-engine.content_length": status.content?.length,+ "fire-engine.has_screenshot": !!status.screenshot,+ "fire-engine.has_pdf": !!(status as any).pdf,+ "fire-engine.job_id": (scrape as any).jobId,+ });++ return status;+ });}export async function scrapeURLWithFireEngineChromeCDP(meta: Meta,): Promise<EngineScrapeResult> {- const actions: Action[] = [- // Transform waitFor option into an action (unsupported by chrome-cdp)- ...(meta.options.waitFor !== 0- ? [- {- type: "wait" as const,- milliseconds:- meta.options.waitFor > 30000 ? 30000 : meta.options.waitFor,- },- ]- : []),-- // Include specified actions- ...(meta.options.actions ?? []),-- // Transform screenshot format into an action (unsupported by chrome-cdp)- ...(hasFormatOfType(meta.options.formats, "screenshot")- ? [- {- type: "screenshot" as const,- fullPage:- hasFormatOfType(meta.options.formats, "screenshot")?.fullPage ||- false,- ...(hasFormatOfType(meta.options.formats, "screenshot")?.viewport- ? {- viewport: hasFormatOfType(meta.options.formats, "screenshot")!- .viewport,- }- : {}),- },- ]- : []),- ];-- const totalWait = actions.reduce(- (a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),- 0,- );-- const request: FireEngineScrapeRequestCommon &- FireEngineScrapeRequestChromeCDP = {- url: meta.rewrittenUrl ?? meta.url,- engine: "chrome-cdp",- instantReturn: false,- skipTlsVerification: meta.options.skipTlsVerification,- headers: meta.options.headers,- ...(actions.length > 0- ? {- actions,- }- : {}),- priority: meta.internalOptions.priority,- geolocation: meta.options.location,- mobile: meta.options.mobile,- timeout: meta.abort.scrapeTimeout() ?? 300000,- disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,- mobileProxy: meta.featureFlags.has("stealthProxy"),- saveScrapeResultToGCS:- !meta.internalOptions.zeroDataRetention &&- meta.internalOptions.saveScrapeResultToGCS,- zeroDataRetention: meta.internalOptions.zeroDataRetention,- ...(youtubePostprocessor.shouldRun(- meta,- new URL(meta.rewrittenUrl ?? meta.url),- )- ? { blockMedia: false }- : {}),- };-- let response = await performFireEngineScrape(- meta,- meta.logger.child({- method: "scrapeURLWithFireEngineChromeCDP/callFireEngine",- request,- }),- request,- meta.mock,- meta.abort.asSignal(),- true,- );-- if (hasFormatOfType(meta.options.formats, "screenshot")) {- // meta.logger.debug(- // "Transforming screenshots from actions into screenshot field",- // { screenshots: response.screenshots },- // );- if (response.screenshots) {- response.screenshot = response.screenshots.slice(-1)[0];- response.screenshots = response.screenshots.slice(0, -1);- }- // meta.logger.debug("Screenshot transformation done", {- // screenshots: response.screenshots,- // screenshot: response.screenshot,- // });- }-- if (!response.url) {- meta.logger.warn("Fire-engine did not return the response's URL", {- response,- sourceURL: meta.url,+ return withSpan("engine.fire-engine.chrome-cdp", async span => {+ setSpanAttributes(span, {+ "engine.type": "fire-engine-chrome-cdp",+ "engine.url": meta.url,+ "engine.team_id": meta.internalOptions.teamId,});- }+ const actions: Action[] = [+ // Transform waitFor option into an action (unsupported by chrome-cdp)+ ...(meta.options.waitFor !== 0+ ? [+ {+ type: "wait" as const,+ milliseconds:+ meta.options.waitFor > 30000 ? 30000 : meta.options.waitFor,+ },+ ]+ : []),++ // Include specified actions+ ...(meta.options.actions ?? []),++ // Transform screenshot format into an action (unsupported by chrome-cdp)+ ...(hasFormatOfType(meta.options.formats, "screenshot")+ ? [+ {+ type: "screenshot" as const,+ fullPage:+ hasFormatOfType(meta.options.formats, "screenshot")?.fullPage ||+ false,+ ...(hasFormatOfType(meta.options.formats, "screenshot")?.viewport+ ? {+ viewport: hasFormatOfType(+ meta.options.formats,+ "screenshot",+ )!.viewport,+ }+ : {}),+ },+ ]+ : []),+ ];++ const totalWait = actions.reduce(+ (a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),+ 0,+ );- return {- url: response.url ?? meta.url,+ const request: FireEngineScrapeRequestCommon &+ FireEngineScrapeRequestChromeCDP = {+ url: meta.rewrittenUrl ?? meta.url,+ engine: "chrome-cdp",+ instantReturn: false,+ skipTlsVerification: meta.options.skipTlsVerification,+ headers: meta.options.headers,+ ...(actions.length > 0+ ? {+ actions,+ }+ : {}),+ priority: meta.internalOptions.priority,+ geolocation: meta.options.location,+ mobile: meta.options.mobile,+ timeout: meta.abort.scrapeTimeout() ?? 300000,+ disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,+ mobileProxy: meta.featureFlags.has("stealthProxy"),+ saveScrapeResultToGCS:+ !meta.internalOptions.zeroDataRetention &&+ meta.internalOptions.saveScrapeResultToGCS,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ ...(youtubePostprocessor.shouldRun(+ meta,+ new URL(meta.rewrittenUrl ?? meta.url),+ )+ ? { blockMedia: false }+ : {}),+ };++ let response = await performFireEngineScrape(+ meta,+ meta.logger.child({+ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine",+ request,+ }),+ request,+ meta.mock,+ meta.abort.asSignal(),+ true,+ );- html: response.content,- error: response.pageError,- statusCode: response.pageStatusCode,+ if (hasFormatOfType(meta.options.formats, "screenshot")) {+ // meta.logger.debug(+ // "Transforming screenshots from actions into screenshot field",+ // { screenshots: response.screenshots },+ // );+ if (response.screenshots) {+ response.screenshot = response.screenshots.slice(-1)[0];+ response.screenshots = response.screenshots.slice(0, -1);+ }+ // meta.logger.debug("Screenshot transformation done", {+ // screenshots: response.screenshots,+ // screenshot: response.screenshot,+ // });+ }- contentType:- (Object.entries(response.responseHeaders ?? {}).find(- x => x[0].toLowerCase() === "content-type",- ) ?? [])[1] ?? undefined,-- screenshot: response.screenshot,- ...(actions.length > 0- ? {- actions: {- screenshots: response.screenshots ?? [],- scrapes: response.actionContent ?? [],- javascriptReturns: (response.actionResults ?? [])- .filter(x => x.type === "executeJavascript")- .map(x =>- JSON.parse((x.result as any as { return: string }).return),- ),- pdfs: (response.actionResults ?? [])- .filter(x => x.type === "pdf")- .map(x => x.result.link),- },- }- : {}),+ if (!response.url) {+ meta.logger.warn("Fire-engine did not return the response's URL", {+ response,+ sourceURL: meta.url,+ });+ }- proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- youtubeTranscriptContent: response.youtubeTranscriptContent,- };+ return {+ url: response.url ?? meta.url,++ html: response.content,+ error: response.pageError,+ statusCode: response.pageStatusCode,++ contentType:+ (Object.entries(response.responseHeaders ?? {}).find(+ x => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined,++ screenshot: response.screenshot,+ ...(actions.length > 0+ ? {+ actions: {+ screenshots: response.screenshots ?? [],+ scrapes: response.actionContent ?? [],+ javascriptReturns: (response.actionResults ?? [])+ .filter(x => x.type === "executeJavascript")+ .map(x =>+ JSON.parse((x.result as any as { return: string }).return),+ ),+ pdfs: (response.actionResults ?? [])+ .filter(x => x.type === "pdf")+ .map(x => x.result.link),+ },+ }+ : {}),++ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",+ youtubeTranscriptContent: response.youtubeTranscriptContent,+ };+ });}export async function scrapeURLWithFireEnginePlaywright(meta: Meta,): Promise<EngineScrapeResult> {- const totalWait = meta.options.waitFor;-- const request: FireEngineScrapeRequestCommon &- FireEngineScrapeRequestPlaywright = {- url: meta.rewrittenUrl ?? meta.url,- engine: "playwright",- instantReturn: false,-- headers: meta.options.headers,- priority: meta.internalOptions.priority,- screenshot:- hasFormatOfType(meta.options.formats, "screenshot") !== undefined,- fullPageScreenshot: hasFormatOfType(meta.options.formats, "screenshot")- ?.fullPage,- wait: meta.options.waitFor,- geolocation: meta.options.location,- blockAds: meta.options.blockAds,- mobileProxy: meta.featureFlags.has("stealthProxy"),-- timeout: meta.abort.scrapeTimeout() ?? 300000,- saveScrapeResultToGCS:- !meta.internalOptions.zeroDataRetention &&- meta.internalOptions.saveScrapeResultToGCS,- zeroDataRetention: meta.internalOptions.zeroDataRetention,- };-- let response = await performFireEngineScrape(- meta,- meta.logger.child({- method: "scrapeURLWithFireEnginePlaywright/callFireEngine",- request,- }),- request,- meta.mock,- meta.abort.asSignal(),- );-- if (!response.url) {- meta.logger.warn("Fire-engine did not return the response's URL", {- response,- sourceURL: meta.url,+ return withSpan("engine.fire-engine.playwright", async span => {+ setSpanAttributes(span, {+ "engine.type": "fire-engine-playwright",+ "engine.url": meta.url,+ "engine.team_id": meta.internalOptions.teamId,});- }+ const totalWait = meta.options.waitFor;++ const request: FireEngineScrapeRequestCommon &+ FireEngineScrapeRequestPlaywright = {+ url: meta.rewrittenUrl ?? meta.url,+ engine: "playwright",+ instantReturn: false,++ headers: meta.options.headers,+ priority: meta.internalOptions.priority,+ screenshot:+ hasFormatOfType(meta.options.formats, "screenshot") !== undefined,+ fullPageScreenshot: hasFormatOfType(meta.options.formats, "screenshot")+ ?.fullPage,+ wait: meta.options.waitFor,+ geolocation: meta.options.location,+ blockAds: meta.options.blockAds,+ mobileProxy: meta.featureFlags.has("stealthProxy"),++ timeout: meta.abort.scrapeTimeout() ?? 300000,+ saveScrapeResultToGCS:+ !meta.internalOptions.zeroDataRetention &&+ meta.internalOptions.saveScrapeResultToGCS,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ };++ let response = await performFireEngineScrape(+ meta,+ meta.logger.child({+ method: "scrapeURLWithFireEnginePlaywright/callFireEngine",+ request,+ }),+ request,+ meta.mock,+ meta.abort.asSignal(),+ );++ if (!response.url) {+ meta.logger.warn("Fire-engine did not return the response's URL", {+ response,+ sourceURL: meta.url,+ });+ }- return {- url: response.url ?? meta.url,+ return {+ url: response.url ?? meta.url,- html: response.content,- error: response.pageError,- statusCode: response.pageStatusCode,+ html: response.content,+ error: response.pageError,+ statusCode: response.pageStatusCode,- contentType:- (Object.entries(response.responseHeaders ?? {}).find(- x => x[0].toLowerCase() === "content-type",- ) ?? [])[1] ?? undefined,+ contentType:+ (Object.entries(response.responseHeaders ?? {}).find(+ x => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined,- ...(response.screenshots !== undefined && response.screenshots.length > 0- ? {- screenshot: response.screenshots[0],- }- : {}),+ ...(response.screenshots !== undefined && response.screenshots.length > 0+ ? {+ screenshot: response.screenshots[0],+ }+ : {}),- proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- };+ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",+ };+ });}export async function scrapeURLWithFireEngineTLSClient(meta: Meta,): Promise<EngineScrapeResult> {- const request: FireEngineScrapeRequestCommon &- FireEngineScrapeRequestTLSClient = {- url: meta.rewrittenUrl ?? meta.url,- engine: "tlsclient",- instantReturn: false,-- headers: meta.options.headers,- priority: meta.internalOptions.priority,-- atsv: meta.internalOptions.atsv,- geolocation: meta.options.location,- disableJsDom: meta.internalOptions.v0DisableJsDom,- mobileProxy: meta.featureFlags.has("stealthProxy"),-- timeout: meta.abort.scrapeTimeout() ?? 300000,- saveScrapeResultToGCS:- !meta.internalOptions.zeroDataRetention &&- meta.internalOptions.saveScrapeResultToGCS,- zeroDataRetention: meta.internalOptions.zeroDataRetention,- };-- let response = await performFireEngineScrape(- meta,- meta.logger.child({- method: "scrapeURLWithFireEngineTLSClient/callFireEngine",- request,- }),- request,- meta.mock,- meta.abort.asSignal(),- );-- if (!response.url) {- meta.logger.warn("Fire-engine did not return the response's URL", {- response,- sourceURL: meta.url,+ return withSpan("engine.fire-engine.tlsclient", async span => {+ setSpanAttributes(span, {+ "engine.type": "fire-engine-tlsclient",+ "engine.url": meta.url,+ "engine.team_id": meta.internalOptions.teamId,});- }+ const request: FireEngineScrapeRequestCommon &+ FireEngineScrapeRequestTLSClient = {+ url: meta.rewrittenUrl ?? meta.url,+ engine: "tlsclient",+ instantReturn: false,++ headers: meta.options.headers,+ priority: meta.internalOptions.priority,++ atsv: meta.internalOptions.atsv,+ geolocation: meta.options.location,+ disableJsDom: meta.internalOptions.v0DisableJsDom,+ mobileProxy: meta.featureFlags.has("stealthProxy"),++ timeout: meta.abort.scrapeTimeout() ?? 300000,+ saveScrapeResultToGCS:+ !meta.internalOptions.zeroDataRetention &&+ meta.internalOptions.saveScrapeResultToGCS,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ };++ let response = await performFireEngineScrape(+ meta,+ meta.logger.child({+ method: "scrapeURLWithFireEngineTLSClient/callFireEngine",+ request,+ }),+ request,+ meta.mock,+ meta.abort.asSignal(),+ );- return {- url: response.url ?? meta.url,+ if (!response.url) {+ meta.logger.warn("Fire-engine did not return the response's URL", {+ response,+ sourceURL: meta.url,+ });+ }- html: response.content,- error: response.pageError,- statusCode: response.pageStatusCode,+ return {+ url: response.url ?? meta.url,- contentType:- (Object.entries(response.responseHeaders ?? {}).find(- x => x[0].toLowerCase() === "content-type",- ) ?? [])[1] ?? undefined,+ html: response.content,+ error: response.pageError,+ statusCode: response.pageStatusCode,++ contentType:+ (Object.entries(response.responseHeaders ?? {}).find(+ x => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined,- proxyUsed: response.usedMobileProxy ? "stealth" : "basic",- };+ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",+ };+ });}export function fireEngineMaxReasonableTime(diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex e733c1d74..7326d038a 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1,5 +1,6 @@import { Logger } from "winston";import * as Sentry from "@sentry/node";+import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import {type Document,@@ -383,321 +384,366 @@ class WrappedEngineError extends Error {}async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {- meta.logger.info(- `Scraping URL ${JSON.stringify(meta.rewrittenUrl ?? meta.url)}...`,- );-- if (meta.internalOptions.zeroDataRetention) {- if (meta.featureFlags.has("screenshot")) {- throw new ZDRViolationError("screenshot");- }-- if (meta.featureFlags.has("screenshot@fullScreen")) {- throw new ZDRViolationError("screenshot@fullScreen");- }+ return withSpan("scrape.engine_loop", async span => {+ meta.logger.info(+ `Scraping URL ${JSON.stringify(meta.rewrittenUrl ?? meta.url)}...`,+ );- if (- meta.options.actions &&- meta.options.actions.find(x => x.type === "screenshot")- ) {- throw new ZDRViolationError("screenshot action");- }+ setSpanAttributes(span, {+ "engine.url": meta.rewrittenUrl ?? meta.url,+ "engine.features": Array.from(meta.featureFlags).join(","),+ });- if (- meta.options.actions &&- meta.options.actions.find(x => x.type === "pdf")- ) {- throw new ZDRViolationError("pdf action");- }- }+ if (meta.internalOptions.zeroDataRetention) {+ if (meta.featureFlags.has("screenshot")) {+ throw new ZDRViolationError("screenshot");+ }- // TODO: handle sitemap data, see WebScraper/index.ts:280- // TODO: ScrapeEvents+ if (meta.featureFlags.has("screenshot@fullScreen")) {+ throw new ZDRViolationError("screenshot@fullScreen");+ }- const fallbackList = buildFallbackList(meta);+ if (+ meta.options.actions &&+ meta.options.actions.find(x => x.type === "screenshot")+ ) {+ throw new ZDRViolationError("screenshot action");+ }- const snipeAbortController = new AbortController();- const snipeAbort: AbortInstance = {- signal: snipeAbortController.signal,- tier: "engine",- throwable() {- return new EngineSnipedError();- },- };+ if (+ meta.options.actions &&+ meta.options.actions.find(x => x.type === "pdf")+ ) {+ throw new ZDRViolationError("pdf action");+ }+ }- type EngineBundlePromise = {- engine: Engine;- unsupportedFeatures: Set<FeatureFlag>;- promise: Promise<EngineScrapeResultWithContext>;- };+ // TODO: handle sitemap data, see WebScraper/index.ts:280+ // TODO: ScrapeEvents- const remainingEngines = [...fallbackList];- let enginePromises: EngineBundlePromise[] = [];+ const fallbackList = buildFallbackList(meta);- meta.abort.throwIfAborted();+ setSpanAttributes(span, {+ "engine.fallback_list": fallbackList.map(f => f.engine).join(","),+ });- let result: EngineScrapeResultWithContext | null = null;+ const snipeAbortController = new AbortController();+ const snipeAbort: AbortInstance = {+ signal: snipeAbortController.signal,+ tier: "engine",+ throwable() {+ return new EngineSnipedError();+ },+ };- while (remainingEngines.length > 0) {- const { engine, unsupportedFeatures } = remainingEngines.shift()!;+ type EngineBundlePromise = {+ engine: Engine;+ unsupportedFeatures: Set<FeatureFlag>;+ promise: Promise<EngineScrapeResultWithContext>;+ };- const waitUntilWaterfall =- getEngineMaxReasonableTime(meta, engine) +- parseInt(process.env.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS || "0", 10);+ const remainingEngines = [...fallbackList];+ let enginePromises: EngineBundlePromise[] = [];+ const enginesAttempted: string[] = [];++ meta.abort.throwIfAborted();++ let result: EngineScrapeResultWithContext | null = null;++ while (remainingEngines.length > 0) {+ const { engine, unsupportedFeatures } = remainingEngines.shift()!;+ enginesAttempted.push(engine);++ const waitUntilWaterfall =+ getEngineMaxReasonableTime(meta, engine) ++ parseInt(process.env.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS || "0", 10);++ if (+ !isFinite(waitUntilWaterfall) ||+ isNaN(waitUntilWaterfall) ||+ waitUntilWaterfall <= 0+ ) {+ meta.logger.warn("Invalid waitUntilWaterfall value", {+ waitUntilWaterfall,+ timeout: meta.options.timeout,+ actions: !!meta.options.actions,+ hasJson: !!meta.options.formats?.find(x => x.type === "json"),+ remainingEngines: remainingEngines.length,+ });+ }- if (- !isFinite(waitUntilWaterfall) ||- isNaN(waitUntilWaterfall) ||- waitUntilWaterfall <= 0- ) {- meta.logger.warn("Invalid waitUntilWaterfall value", {+ meta.logger.info("Scraping via " + engine + "...", {waitUntilWaterfall,- timeout: meta.options.timeout,- actions: !!meta.options.actions,- hasJson: !!meta.options.formats?.find(x => x.type === "json"),- remainingEngines: remainingEngines.length,});- }- meta.logger.info("Scraping via " + engine + "...", {- waitUntilWaterfall,- });+ enginePromises.push({+ engine,+ unsupportedFeatures,+ promise: (async () => {+ try {+ return {+ engine,+ unsupportedFeatures,+ result: await scrapeURLLoopIter(meta, engine, snipeAbort),+ };+ } catch (error) {+ throw new WrappedEngineError(engine, error);+ }+ })(),+ });- enginePromises.push({- engine,- unsupportedFeatures,- promise: (async () => {+ while (true) {try {- return {- engine,- unsupportedFeatures,- result: await scrapeURLLoopIter(meta, engine, snipeAbort),- };+ result = await Promise.race([+ ...enginePromises.map(x => x.promise),+ ...(remainingEngines.length > 0+ ? [+ new Promise<EngineScrapeResultWithContext>((_, reject) => {+ setTimeout(() => {+ reject(new WaterfallNextEngineSignal());+ }, waitUntilWaterfall);+ }),+ ]+ : []),+ new Promise<EngineScrapeResultWithContext>((_, reject) => {+ setTimeout(() => {+ try {+ meta.abort.throwIfAborted();++ // Fallback error if above doesn't throw+ const usingDefaultTimeout =+ meta.abort.scrapeTimeout() === undefined;+ throw new ScrapeJobTimeoutError(+ usingDefaultTimeout+ ? "Scrape timed out due to maximum length of 5 minutes"+ : "Scrape timed out",+ );+ } catch (error) {+ reject(error);+ }+ }, meta.abort.scrapeTimeout() ?? 300000);+ }),+ ]);+ break;} catch (error) {- throw new WrappedEngineError(engine, error);- }- })(),- });-- while (true) {- try {- result = await Promise.race([- ...enginePromises.map(x => x.promise),- ...(remainingEngines.length > 0- ? [- new Promise<EngineScrapeResultWithContext>((_, reject) => {- setTimeout(() => {- reject(new WaterfallNextEngineSignal());- }, waitUntilWaterfall);- }),- ]- : []),- new Promise<EngineScrapeResultWithContext>((_, reject) => {- setTimeout(() => {- try {- meta.abort.throwIfAborted();-- // Fallback error if above doesn't throw- const usingDefaultTimeout =- meta.abort.scrapeTimeout() === undefined;- throw new ScrapeJobTimeoutError(- usingDefaultTimeout- ? "Scrape timed out due to maximum length of 5 minutes"- : "Scrape timed out",- );- } catch (error) {- reject(error);- }- }, meta.abort.scrapeTimeout() ?? 300000);- }),- ]);- break;- } catch (error) {- if (error instanceof WrappedEngineError) {- if (error.error instanceof EngineError) {- meta.logger.warn(- "Engine " + error.engine + " could not scrape the page.",- {+ if (error instanceof WrappedEngineError) {+ if (error.error instanceof EngineError) {+ meta.logger.warn(+ "Engine " + error.engine + " could not scrape the page.",+ {+ error: error.error,+ },+ );+ } else if (error.error instanceof IndexMissError) {+ meta.logger.warn(+ "Engine " ++ error.engine ++ " could not find the page in the index.",+ {+ error: error.error,+ },+ );+ } else if (+ error.error instanceof AddFeatureError ||+ error.error instanceof RemoveFeatureError ||+ error.error instanceof SiteError ||+ error.error instanceof SSLError ||+ error.error instanceof DNSResolutionError ||+ error.error instanceof ActionError ||+ error.error instanceof UnsupportedFileError ||+ error.error instanceof PDFAntibotError ||+ error.error instanceof PDFInsufficientTimeError ||+ error.error instanceof ProxySelectionError+ ) {+ throw error.error;+ } else if (error.error instanceof LLMRefusalError) {+ meta.logger.warn("LLM refusal encountered", {error: error.error,- },- );- } else if (error.error instanceof IndexMissError) {- meta.logger.warn(- "Engine " +- error.engine +- " could not find the page in the index.",- {+ });+ throw error.error;+ } else if (error.error instanceof FEPageLoadFailed) {+ // This is the internal timeout bug on f-e and should be treated as an EngineError.+ meta.logger.warn("FEPageLoadFailed encountered", {error: error.error,- },- );- } else if (- error.error instanceof AddFeatureError ||- error.error instanceof RemoveFeatureError ||- error.error instanceof SiteError ||- error.error instanceof SSLError ||- error.error instanceof DNSResolutionError ||- error.error instanceof ActionError ||- error.error instanceof UnsupportedFileError ||- error.error instanceof PDFAntibotError ||- error.error instanceof PDFInsufficientTimeError ||- error.error instanceof ProxySelectionError- ) {- throw error.error;- } else if (error.error instanceof LLMRefusalError) {- meta.logger.warn("LLM refusal encountered", { error: error.error });- throw error.error;- } else if (error.error instanceof FEPageLoadFailed) {- // This is the internal timeout bug on f-e and should be treated as an EngineError.- meta.logger.warn("FEPageLoadFailed encountered", {- error: error.error,- });- } else if (error.error instanceof AbortManagerThrownError) {- if (error.error.tier === "engine") {+ });+ } else if (error.error instanceof AbortManagerThrownError) {+ if (error.error.tier === "engine") {+ meta.logger.warn(+ "Engine " + error.engine + " timed out while scraping.",+ { error: error.error },+ );+ } else {+ throw error.error;+ }+ } else {meta.logger.warn(- "Engine " + error.engine + " timed out while scraping.",- { error: error.error },+ "An unexpected error happened while scraping with " ++ error.engine ++ ".",+ { error },);- } else {- throw error.error;}- } else {- meta.logger.warn(- "An unexpected error happened while scraping with " +- error.engine +- ".",- { error },++ // Filter out the failed engine+ enginePromises = enginePromises.filter(+ x => x.engine !== error.engine,);- }- // Filter out the failed engine- enginePromises = enginePromises.filter(- x => x.engine !== error.engine,- );+ // If we don't have any engines waterfalled, let's waterfall the next engine+ if (enginePromises.length === 0) {+ break;+ }- // If we don't have any engines waterfalled, let's waterfall the next engine- if (enginePromises.length === 0) {+ // Otherwise, just keep racing+ } else if (+ error instanceof AddFeatureError ||+ error instanceof RemoveFeatureError+ ) {+ throw error;+ } else if (error instanceof WaterfallNextEngineSignal) {+ // It's time to waterfall the next enginebreak;- }+ } else if (error instanceof AbortManagerThrownError) {+ if (error.tier === "engine") {+ meta.logger.warn(+ "Engine-scoped timeout error received here. Weird!",+ { error },+ );+ }- // Otherwise, just keep racing- } else if (- error instanceof AddFeatureError ||- error instanceof RemoveFeatureError- ) {- throw error;- } else if (error instanceof WaterfallNextEngineSignal) {- // It's time to waterfall the next engine- break;- } else if (error instanceof AbortManagerThrownError) {- if (error.tier === "engine") {- meta.logger.warn(- "Engine-scoped timeout error received here. Weird!",- { error },- );+ throw error;+ } else {+ meta.logger.warn("Unexpected error while racing engines", {+ error,+ });+ throw error;}-- throw error;- } else {- meta.logger.warn("Unexpected error while racing engines", { error });- throw error;}}++ if (result === null) {+ meta.logger.info("Waterfalling to next engine...", {+ waitUntilWaterfall,+ });+ } else {+ break;+ }}+ snipeAbortController.abort();+if (result === null) {- meta.logger.info("Waterfalling to next engine...", {- waitUntilWaterfall,+ setSpanAttributes(span, {+ "engine.no_engines_left": true,+ "engine.engines_attempted": enginesAttempted.join(","),});- } else {- break;+ throw new NoEnginesLeftError(fallbackList.map(x => x.engine));}- }-- snipeAbortController.abort();- if (result === null) {- throw new NoEnginesLeftError(fallbackList.map(x => x.engine));- }+ // Set winner engine attributes+ setSpanAttributes(span, {+ "engine.winner": result.engine,+ "engine.engines_attempted": enginesAttempted.join(","),+ "engine.unsupported_features":+ result.unsupportedFeatures.size > 0+ ? Array.from(result.unsupportedFeatures).join(",")+ : undefined,+ });- let engineResult: EngineScrapeResult = result.result;-- for (const postprocessor of postprocessors) {- if (- postprocessor.shouldRun(- meta,- new URL(engineResult.url),- engineResult.postprocessorsUsed,- )- ) {- meta.logger.info("Running postprocessor " + postprocessor.name);- try {- engineResult = await postprocessor.run(- {- ...meta,- logger: meta.logger.child({- method: "postprocessors/" + postprocessor.name,- }),- },- engineResult,- );- } catch (error) {- meta.logger.warn("Failed to run postprocessor " + postprocessor.name, {- error,- });+ meta.winnerEngine = result.engine;+ let engineResult: EngineScrapeResult = result.result;++ for (const postprocessor of postprocessors) {+ if (+ postprocessor.shouldRun(+ meta,+ new URL(engineResult.url),+ engineResult.postprocessorsUsed,+ )+ ) {+ meta.logger.info("Running postprocessor " + postprocessor.name);+ try {+ engineResult = await postprocessor.run(+ {+ ...meta,+ logger: meta.logger.child({+ method: "postprocessors/" + postprocessor.name,+ }),+ },+ engineResult,+ );+ } catch (error) {+ meta.logger.warn(+ "Failed to run postprocessor " + postprocessor.name,+ {+ error,+ },+ );+ }}}- }- let document: Document = {- markdown: engineResult.markdown,- rawHtml: engineResult.html,- screenshot: engineResult.screenshot,- actions: engineResult.actions,- metadata: {- sourceURL: meta.internalOptions.unnormalizedSourceURL ?? meta.url,- url: engineResult.url,- statusCode: engineResult.statusCode,- error: engineResult.error,- numPages: engineResult.pdfMetadata?.numPages,- ...(engineResult.pdfMetadata?.title- ? { title: engineResult.pdfMetadata.title }- : {}),- contentType: engineResult.contentType,- proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",- ...(fallbackList.find(x =>- ["index", "index;documents"].includes(x.engine),- )- ? engineResult.cacheInfo- ? {- cacheState: "hit",- cachedAt: engineResult.cacheInfo.created_at.toISOString(),- }- : {- cacheState: "miss",- }- : {}),- postprocessorsUsed: engineResult.postprocessorsUsed,- },- };+ let document: Document = {+ markdown: engineResult.markdown,+ rawHtml: engineResult.html,+ screenshot: engineResult.screenshot,+ actions: engineResult.actions,+ metadata: {+ sourceURL: meta.internalOptions.unnormalizedSourceURL ?? meta.url,+ url: engineResult.url,+ statusCode: engineResult.statusCode,+ error: engineResult.error,+ numPages: engineResult.pdfMetadata?.numPages,+ ...(engineResult.pdfMetadata?.title+ ? { title: engineResult.pdfMetadata.title }+ : {}),+ contentType: engineResult.contentType,+ proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",+ ...(fallbackList.find(x =>+ ["index", "index;documents"].includes(x.engine),+ )+ ? engineResult.cacheInfo+ ? {+ cacheState: "hit",+ cachedAt: engineResult.cacheInfo.created_at.toISOString(),+ }+ : {+ cacheState: "miss",+ }+ : {}),+ postprocessorsUsed: engineResult.postprocessorsUsed,+ },+ };- if (result.unsupportedFeatures.size > 0) {- const warning = `The engine used does not support the following features: ${[...result.unsupportedFeatures].join(", ")} -- your scrape may be partial.`;- meta.logger.warn(warning, {- engine: result.engine,- unsupportedFeatures: result.unsupportedFeatures,- });- document.warning =- document.warning !== undefined- ? document.warning + " " + warning- : warning;- }+ if (result.unsupportedFeatures.size > 0) {+ const warning = `The engine used does not support the following features: ${[...result.unsupportedFeatures].join(", ")} -- your scrape may be partial.`;+ meta.logger.warn(warning, {+ engine: result.engine,+ unsupportedFeatures: result.unsupportedFeatures,+ });+ document.warning =+ document.warning !== undefined+ ? document.warning + " " + warning+ : warning;+ }- document = await executeTransformers(meta, document);+ document = await executeTransformers(meta, document);- return {- success: true,- document,- };+ // Set final span attributes+ setSpanAttributes(span, {+ "engine.final_status_code": document.metadata.statusCode,+ "engine.final_url": document.metadata.url,+ "engine.content_type": document.metadata.contentType,+ "engine.proxy_used": document.metadata.proxyUsed,+ "engine.cache_state": document.metadata.cacheState,+ "engine.postprocessors_used": engineResult.postprocessorsUsed?.join(","),+ });++ return {+ success: true,+ document,+ };+ });}export async function scrapeURL(@@ -707,242 +753,321 @@ export async function scrapeURL(internalOptions: InternalOptions,costTracking: CostTracking,): Promise<ScrapeUrlResponse> {- const meta = await buildMetaObject(- id,- url,- options,- internalOptions,- costTracking,- );+ return withSpan("scrape.pipeline", async span => {+ const meta = await buildMetaObject(+ id,+ url,+ options,+ internalOptions,+ costTracking,+ );- const startTime = Date.now();+ const startTime = Date.now();++ // Set initial span attributes+ setSpanAttributes(span, {+ "scrape.id": id,+ "scrape.url": url,+ "scrape.team_id": internalOptions.teamId,+ "scrape.crawl_id": internalOptions.crawlId,+ "scrape.zero_data_retention": internalOptions.zeroDataRetention,+ "scrape.force_engine": Array.isArray(internalOptions.forceEngine)+ ? internalOptions.forceEngine.join(",")+ : internalOptions.forceEngine,+ "scrape.features": Array.from(meta.featureFlags).join(","),+ });- meta.logger.info("scrapeURL entered");+ meta.logger.info("scrapeURL entered");- if (meta.rewrittenUrl) {- meta.logger.info("Rewriting URL");- }+ if (meta.rewrittenUrl) {+ meta.logger.info("Rewriting URL");+ setSpanAttributes(span, {+ "scrape.rewritten_url": meta.rewrittenUrl,+ });+ }- if (internalOptions.teamFlags?.checkRobotsOnScrape) {- const urlToCheck = meta.rewrittenUrl || meta.url;- meta.logger.info("Checking robots.txt", { url: urlToCheck });-- const urlObj = new URL(urlToCheck);- const isRobotsTxtPath = urlObj.pathname === "/robots.txt";-- if (!isRobotsTxtPath) {- try {- const { content: robotsTxt } = await fetchRobotsTxt(- {- url: urlToCheck,- zeroDataRetention: internalOptions.zeroDataRetention || false,- location: options.location,- },- id,- meta.logger,- meta.abort.asSignal(),- );+ if (internalOptions.teamFlags?.checkRobotsOnScrape) {+ await withSpan("scrape.robots_check", async robotsSpan => {+ const urlToCheck = meta.rewrittenUrl || meta.url;+ meta.logger.info("Checking robots.txt", { url: urlToCheck });++ const urlObj = new URL(urlToCheck);+ const isRobotsTxtPath = urlObj.pathname === "/robots.txt";- const checker = createRobotsChecker(urlToCheck, robotsTxt);- const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);+ setSpanAttributes(robotsSpan, {+ "robots.url": urlToCheck,+ "robots.is_robots_txt_path": isRobotsTxtPath,+ });- if (!isAllowed) {- meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });+ if (!isRobotsTxtPath) {+ try {+ const { content: robotsTxt } = await fetchRobotsTxt(+ {+ url: urlToCheck,+ zeroDataRetention: internalOptions.zeroDataRetention || false,+ location: options.location,+ },+ id,+ meta.logger,+ meta.abort.asSignal(),+ );++ const checker = createRobotsChecker(urlToCheck, robotsTxt);+ const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);++ setSpanAttributes(robotsSpan, {+ "robots.allowed": isAllowed,+ });++ if (!isAllowed) {+ meta.logger.info("URL blocked by robots.txt", {+ url: urlToCheck,+ });+ setSpanAttributes(span, {+ "scrape.blocked_by_robots": true,+ });+ throw new Error("URL blocked by robots.txt");+ }+ } catch (error) {+ if (error.message === "URL blocked by robots.txt") {+ throw error;+ }+ meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {+ error,+ url: urlToCheck,+ });+ setSpanAttributes(robotsSpan, {+ "robots.fetch_failed": true,+ });+ }+ }+ }).catch(error => {+ if (error.message === "URL blocked by robots.txt") {return {success: false,- error: new Error("URL blocked by robots.txt"),+ error,};}- } catch (error) {- meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {- error,- url: urlToCheck,- });- }+ throw error;+ });}- }- meta.logger.info("Pre-recording frequency");-- const shouldRecordFrequency =- useIndex &&- meta.options.storeInCache &&- !meta.internalOptions.zeroDataRetention &&- internalOptions.teamId !== process.env.PRECRAWL_TEAM_ID;- if (shouldRecordFrequency) {- (async () => {- try {- meta.logger.info("Recording frequency");- const normalizedURL = normalizeURLForIndex(meta.url);- const urlHash = hashURL(normalizedURL);-- let { data, error } = await index_supabase_service- .from("index")- .select("id, created_at, status")- .eq("url_hash", urlHash)- .order("created_at", { ascending: false })- .limit(1);-- if (error) {- meta.logger.warn("Failed to get age data", { error });- }+ meta.logger.info("Pre-recording frequency");- const age = data?.[0]- ? Date.now() - new Date(data[0].created_at).getTime()- : -1;+ const shouldRecordFrequency =+ useIndex &&+ meta.options.storeInCache &&+ !meta.internalOptions.zeroDataRetention &&+ internalOptions.teamId !== process.env.PRECRAWL_TEAM_ID;+ if (shouldRecordFrequency) {+ (async () => {+ try {+ meta.logger.info("Recording frequency");+ const normalizedURL = normalizeURLForIndex(meta.url);+ const urlHash = hashURL(normalizedURL);++ let { data, error } = await index_supabase_service+ .from("index")+ .select("id, created_at, status")+ .eq("url_hash", urlHash)+ .order("created_at", { ascending: false })+ .limit(1);++ if (error) {+ meta.logger.warn("Failed to get age data", { error });+ }- const fakeDomain = meta.options.__experimental_omceDomain;- const domainSplits = generateDomainSplits(- new URL(normalizeURLForIndex(meta.url)).hostname,- fakeDomain,- );- const domainHash = hashURL(domainSplits.slice(-1)[0]);-- const out = {- domain_hash: domainHash,- url: meta.url,- age2: age,- };-- await addIndexRFInsertJob(out);- meta.logger.info("Recorded frequency", { out });- } catch (error) {- meta.logger.warn("Failed to record frequency", { error });- }- })();- } else {- meta.logger.info("Not recording frequency", {- useIndex,- storeInCache: meta.options.storeInCache,- zeroDataRetention: meta.internalOptions.zeroDataRetention,- });- }+ const age = data?.[0]+ ? Date.now() - new Date(data[0].created_at).getTime()+ : -1;- try {- let result: ScrapeUrlResponse;- while (true) {- try {- result = await scrapeURLLoop(meta);- break;- } catch (error) {- if (- error instanceof AddFeatureError &&- (meta.internalOptions.forceEngine === undefined ||- Array.isArray(meta.internalOptions.forceEngine))- ) {- meta.logger.debug(- "More feature flags requested by scraper: adding " +- error.featureFlags.join(", "),- { error, existingFlags: meta.featureFlags },+ const fakeDomain = meta.options.__experimental_omceDomain;+ const domainSplits = generateDomainSplits(+ new URL(normalizeURLForIndex(meta.url)).hostname,+ fakeDomain,);- meta.featureFlags = new Set(- [...meta.featureFlags].concat(error.featureFlags),- );- if (error.pdfPrefetch) {- meta.pdfPrefetch = error.pdfPrefetch;- }- } else if (- error instanceof RemoveFeatureError &&- (meta.internalOptions.forceEngine === undefined ||- Array.isArray(meta.internalOptions.forceEngine))- ) {- meta.logger.debug(- "Incorrect feature flags reported by scraper: removing " +- error.featureFlags.join(","),- { error, existingFlags: meta.featureFlags },- );- meta.featureFlags = new Set(- [...meta.featureFlags].filter(x => !error.featureFlags.includes(x)),- );- } else if (- error instanceof PDFAntibotError &&- meta.internalOptions.forceEngine === undefined- ) {- if (meta.pdfPrefetch !== undefined) {- meta.logger.error(- "PDF was prefetched and still blocked by antibot, failing",+ const domainHash = hashURL(domainSplits.slice(-1)[0]);++ const out = {+ domain_hash: domainHash,+ url: meta.url,+ age2: age,+ };++ await addIndexRFInsertJob(out);+ meta.logger.info("Recorded frequency", { out });+ } catch (error) {+ meta.logger.warn("Failed to record frequency", { error });+ }+ })();+ } else {+ meta.logger.info("Not recording frequency", {+ useIndex,+ storeInCache: meta.options.storeInCache,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ });+ }++ try {+ let result: ScrapeUrlResponse;+ while (true) {+ try {+ result = await scrapeURLLoop(meta);+ break;+ } catch (error) {+ if (+ error instanceof AddFeatureError &&+ (meta.internalOptions.forceEngine === undefined ||+ Array.isArray(meta.internalOptions.forceEngine))+ ) {+ meta.logger.debug(+ "More feature flags requested by scraper: adding " ++ error.featureFlags.join(", "),+ { error, existingFlags: meta.featureFlags },);- throw error;- } else {+ meta.featureFlags = new Set(+ [...meta.featureFlags].concat(error.featureFlags),+ );+ if (error.pdfPrefetch) {+ meta.pdfPrefetch = error.pdfPrefetch;+ }+ } else if (+ error instanceof RemoveFeatureError &&+ (meta.internalOptions.forceEngine === undefined ||+ Array.isArray(meta.internalOptions.forceEngine))+ ) {meta.logger.debug(- "PDF was blocked by anti-bot, prefetching with chrome-cdp",+ "Incorrect feature flags reported by scraper: removing " ++ error.featureFlags.join(","),+ { error, existingFlags: meta.featureFlags },);meta.featureFlags = new Set(- [...meta.featureFlags].filter(x => x !== "pdf"),+ [...meta.featureFlags].filter(+ x => !error.featureFlags.includes(x),+ ),);+ } else if (+ error instanceof PDFAntibotError &&+ meta.internalOptions.forceEngine === undefined+ ) {+ if (meta.pdfPrefetch !== undefined) {+ meta.logger.error(+ "PDF was prefetched and still blocked by antibot, failing",+ );+ throw error;+ } else {+ meta.logger.debug(+ "PDF was blocked by anti-bot, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "pdf"),+ );+ }+ } else {+ throw error;}- } else {- throw error;}}- }- meta.logger.debug("scrapeURL metrics", {- module: "scrapeURL/metrics",- timeTaken: Date.now() - startTime,- maxAgeValid: (meta.options.maxAge ?? 0) > 0,- shouldUseIndex: shouldUseIndex(meta),- success: result.success,- indexHit: result.success && result.document.metadata.cacheState === "hit",- });+ meta.logger.debug("scrapeURL metrics", {+ module: "scrapeURL/metrics",+ timeTaken: Date.now() - startTime,+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,+ shouldUseIndex: shouldUseIndex(meta),+ success: result.success,+ indexHit:+ result.success && result.document.metadata.cacheState === "hit",+ });- return result;- } catch (error) {- // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {- // throw new FEPageLoadFailed();- // } else- meta.logger.debug("scrapeURL metrics", {- module: "scrapeURL/metrics",- timeTaken: Date.now() - startTime,- maxAgeValid: (meta.options.maxAge ?? 0) > 0,- shouldUseIndex: shouldUseIndex(meta),- success: false,- indexHit: false,- });+ setSpanAttributes(span, {+ "scrape.success": true,+ "scrape.duration_ms": Date.now() - startTime,+ "scrape.index_hit":+ result.success && result.document.metadata.cacheState === "hit",+ });- if (error instanceof NoEnginesLeftError) {- meta.logger.warn("scrapeURL: All scraping engines failed!", { error });- } else if (error instanceof LLMRefusalError) {- meta.logger.warn("scrapeURL: LLM refused to extract content", { error });- } else if (- error instanceof Error &&- error.message.includes("Invalid schema for response_format")- ) {- // TODO: seperate into custom error- meta.logger.warn("scrapeURL: LLM schema error", { error });- // TODO: results?- } else if (error instanceof SiteError) {- meta.logger.warn("scrapeURL: Site failed to load in browser", { error });- } else if (error instanceof SSLError) {- meta.logger.warn("scrapeURL: SSL error", { error });- } else if (error instanceof ActionError) {- meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });- } else if (error instanceof UnsupportedFileError) {- meta.logger.warn("scrapeURL: Tried to scrape unsupported file", {- error,+ return result;+ } catch (error) {+ // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {+ // throw new FEPageLoadFailed();+ // } else+ meta.logger.debug("scrapeURL metrics", {+ module: "scrapeURL/metrics",+ timeTaken: Date.now() - startTime,+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,+ shouldUseIndex: shouldUseIndex(meta),+ success: false,+ indexHit: false,});- } else if (error instanceof PDFInsufficientTimeError) {- meta.logger.warn("scrapeURL: Insufficient time to process PDF", {- error,++ // Set error attributes on span+ let errorType = "unknown";+ if (error instanceof NoEnginesLeftError) {+ errorType = "NoEnginesLeftError";+ meta.logger.warn("scrapeURL: All scraping engines failed!", { error });+ } else if (error instanceof LLMRefusalError) {+ errorType = "LLMRefusalError";+ meta.logger.warn("scrapeURL: LLM refused to extract content", {+ error,+ });+ } else if (+ error instanceof Error &&+ error.message.includes("Invalid schema for response_format")+ ) {+ errorType = "LLMSchemaError";+ // TODO: seperate into custom error+ meta.logger.warn("scrapeURL: LLM schema error", { error });+ // TODO: results?+ } else if (error instanceof SiteError) {+ errorType = "SiteError";+ meta.logger.warn("scrapeURL: Site failed to load in browser", {+ error,+ });+ } else if (error instanceof SSLError) {+ errorType = "SSLError";+ meta.logger.warn("scrapeURL: SSL error", { error });+ } else if (error instanceof ActionError) {+ errorType = "ActionError";+ meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });+ } else if (error instanceof UnsupportedFileError) {+ errorType = "UnsupportedFileError";+ meta.logger.warn("scrapeURL: Tried to scrape unsupported file", {+ error,+ });+ } else if (error instanceof PDFInsufficientTimeError) {+ errorType = "PDFInsufficientTimeError";+ meta.logger.warn("scrapeURL: Insufficient time to process PDF", {+ error,+ });+ } else if (error instanceof PDFPrefetchFailed) {+ errorType = "PDFPrefetchFailed";+ meta.logger.warn(+ "scrapeURL: Failed to prefetch PDF that is protected by anti-bot",+ { error },+ );+ } else if (error instanceof ProxySelectionError) {+ errorType = "ProxySelectionError";+ meta.logger.warn("scrapeURL: Proxy selection error", { error });+ } else if (error instanceof AbortManagerThrownError) {+ errorType = "AbortManagerThrownError";+ throw error.inner;+ } else {+ Sentry.captureException(error);+ meta.logger.error("scrapeURL: Unexpected error happened", { error });+ // TODO: results?+ }++ setSpanAttributes(span, {+ "scrape.success": false,+ "scrape.error": error instanceof Error ? error.message : String(error),+ "scrape.error_type": errorType,+ "scrape.duration_ms": Date.now() - startTime,});- } else if (error instanceof PDFPrefetchFailed) {- meta.logger.warn(- "scrapeURL: Failed to prefetch PDF that is protected by anti-bot",- { error },- );- } else if (error instanceof ProxySelectionError) {- meta.logger.warn("scrapeURL: Proxy selection error", { error });- } else if (error instanceof AbortManagerThrownError) {- throw error.inner;- } else {- Sentry.captureException(error);- meta.logger.error("scrapeURL: Unexpected error happened", { error });- // TODO: results?- }- return {- success: false,- error,- };- }+ return {+ success: false,+ error,+ };+ }+ });}diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex da0bda268..194252f70 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -21,6 +21,7 @@ import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";import { deserializeTransportableError } from "../lib/error-serde";import { abTestJob } from "./ab-test";import { NuQJob, scrapeQueue } from "./worker/nuq";+import { serializeTraceContext } from "../lib/otel-tracer";/*** Checks if a job is a crawl or batch scrape based on its options@@ -189,8 +190,15 @@ export async function addScrapeJob(priority: number = 0,directToBullMQ: boolean = false,): Promise<NuQJob<ScrapeJobData> | null> {+ // Capture trace context to propagate to worker+ const traceContext = serializeTraceContext();+ const optionsWithTrace: ScrapeJobData = {+ ...webScraperOptions,+ traceContext,+ };+return await addScrapeJobRaw(- webScraperOptions,+ optionsWithTrace,jobId,priority,directToBullMQ,@@ -206,6 +214,9 @@ export async function addScrapeJobs() {if (jobs.length === 0) return true;+ // Capture trace context for all jobs+ const traceContext = serializeTraceContext();+const jobsByTeam = new Map<string,{@@ -349,7 +360,7 @@ export async function addScrapeJobs(addToCQ.map(async job => {const size = JSON.stringify(job.data).length;await _addScrapeJobToConcurrencyQueue(- job.data,+ { ...job.data, traceContext },job.jobId,job.priority,);@@ -358,7 +369,11 @@ export async function addScrapeJobs(await Promise.all(addToBull.map(async job => {- await _addScrapeJobToBullMQ(job.data, job.jobId, job.priority);+ await _addScrapeJobToBullMQ(+ { ...job.data, traceContext },+ job.jobId,+ job.priority,+ );}),);}diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 110b0af77..4365a39e1 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -2,6 +2,7 @@ import { Logger } from "winston";import { logger } from "../../lib/logger";import { Client, Pool } from "pg";import { type ScrapeJobData } from "../../types";+import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";// === Basics@@ -145,24 +146,42 @@ class NuQ<JobData = any, JobReturnValue = any> {id: string,_logger = logger,): Promise<NuQJob<JobData, JobReturnValue> | null> {- const start = Date.now();- try {- return this.rowToJob(- (- await nuqPool.query(- `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = $1;`,- [id],- )- ).rows[0],- );- } finally {- _logger.info("nuqGetJob metrics", {- module: "nuq/metrics",- method: "nuqGetJob",- duration: Date.now() - start,- scrapeId: id,+ return withSpan("nuq.getJob", async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,});- }++ const start = Date.now();+ try {+ const result = this.rowToJob(+ (+ await nuqPool.query(+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = $1;`,+ [id],+ )+ ).rows[0],+ );++ setSpanAttributes(span, {+ "nuq.job_found": result !== null,+ "nuq.job_status": result?.status,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ _logger.info("nuqGetJob metrics", {+ module: "nuq/metrics",+ method: "nuqGetJob",+ duration,+ scrapeId: id,+ });+ }+ });}public async getJobs(@@ -296,25 +315,44 @@ class NuQ<JobData = any, JobReturnValue = any> {data: JobData,priority: number = 0,): Promise<NuQJob<JobData, JobReturnValue>> {- const start = Date.now();- try {- return this.rowToJob(- (- await nuqPool.query(- `INSERT INTO ${this.queueName} (id, data, priority) VALUES ($1, $2, $3) RETURNING ${this.jobReturning.join(", ")};`,- [id, data, priority],- )- ).rows[0],- )!;- } finally {- logger.info("nuqAddJob metrics", {- module: "nuq/metrics",- method: "nuqAddJob",- duration: Date.now() - start,- scrapeId: id,- zeroDataRetention: (data as any)?.zeroDataRetention ?? false,+ return withSpan("nuq.addJob", async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,+ "nuq.priority": priority,+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,});- }++ const start = Date.now();+ try {+ const result = this.rowToJob(+ (+ await nuqPool.query(+ `INSERT INTO ${this.queueName} (id, data, priority) VALUES ($1, $2, $3) RETURNING ${this.jobReturning.join(", ")};`,+ [id, data, priority],+ )+ ).rows[0],+ )!;++ setSpanAttributes(span, {+ "nuq.job_created": true,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ logger.info("nuqAddJob metrics", {+ module: "nuq/metrics",+ method: "nuqAddJob",+ duration,+ scrapeId: id,+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,+ });+ }+ });}private readonly nuqWaitMode =@@ -327,89 +365,107 @@ class NuQ<JobData = any, JobReturnValue = any> {timeout: number | null,_logger: Logger = logger,): Promise<JobReturnValue> {- const done = new Promise<JobReturnValue>(- (async (resolve, reject) => {- if (this.nuqWaitMode === "listen") {- let timer: NodeJS.Timeout | null = null;- if (timeout !== null) {- timer = setTimeout(- (() => {- this.removeListener(id, listener);- reject(new Error("Timed out"));- }).bind(this),- timeout,- );- }+ return withSpan("nuq.waitForJob", async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,+ "nuq.timeout": timeout ?? undefined,+ "nuq.wait_mode": this.nuqWaitMode,+ });- const listener = async function (_msg: "completed" | "failed") {- if (timer) clearTimeout(timer);- const job = await this.getJob(id, _logger);- if (!job) {- reject(new Error("Job raced out while waiting for it"));- } else {- if (job.status === "completed") {- resolve(job.returnvalue!);- } else {- reject(new Error(job.failedReason!));- }+ const startTime = Date.now();++ const done = new Promise<JobReturnValue>(+ (async (resolve, reject) => {+ if (this.nuqWaitMode === "listen") {+ let timer: NodeJS.Timeout | null = null;+ if (timeout !== null) {+ timer = setTimeout(+ (() => {+ this.removeListener(id, listener);+ reject(new Error("Timed out"));+ }).bind(this),+ timeout,+ );}- }.bind(this);-- try {- await this.addListener(id, listener);- } catch (e) {- reject(e);- }- try {- const job = await this.getJob(id, _logger);- if (job && ["completed", "failed"].includes(job.status)) {- this.removeListener(id, listener);+ const listener = async function (_msg: "completed" | "failed") {if (timer) clearTimeout(timer);- if (job.status === "completed") {- resolve(job.returnvalue!);+ const job = await this.getJob(id, _logger);+ if (!job) {+ reject(new Error("Job raced out while waiting for it"));} else {- reject(new Error(job.failedReason!));+ if (job.status === "completed") {+ resolve(job.returnvalue!);+ } else {+ reject(new Error(job.failedReason!));+ }}- return;+ }.bind(this);++ try {+ await this.addListener(id, listener);+ } catch (e) {+ reject(e);}- } catch (e) {- _logger.warn("nuqGetJob ensure check failed", {- module: "nuq",- method: "nuqWaitForJob",- error: e,- scrapeId: id,- });- }- } else {- const timeoutAt = timeout !== null ? Date.now() + timeout : null;- const poll = async function poll() {+try {const job = await this.getJob(id, _logger);if (job && ["completed", "failed"].includes(job.status)) {+ this.removeListener(id, listener);+ if (timer) clearTimeout(timer);if (job.status === "completed") {- return resolve(job.returnvalue!);+ resolve(job.returnvalue!);} else {- return reject(new Error(job.failedReason!));+ reject(new Error(job.failedReason!));}+ return;}} catch (e) {- return reject(e);+ _logger.warn("nuqGetJob ensure check failed", {+ module: "nuq",+ method: "nuqWaitForJob",+ error: e,+ scrapeId: id,+ });}+ } else {+ const timeoutAt = timeout !== null ? Date.now() + timeout : null;+ const poll = async function poll() {+ try {+ const job = await this.getJob(id, _logger);+ if (job && ["completed", "failed"].includes(job.status)) {+ if (job.status === "completed") {+ return resolve(job.returnvalue!);+ } else {+ return reject(new Error(job.failedReason!));+ }+ }+ } catch (e) {+ return reject(e);+ }- if (timeoutAt && Date.now() > timeoutAt) {- return reject(new Error("Timed out"));- }+ if (timeoutAt && Date.now() > timeoutAt) {+ return reject(new Error("Timed out"));+ }- setTimeout(poll.bind(this), 500);- }.bind(this);+ setTimeout(poll.bind(this), 500);+ }.bind(this);- poll();- }- }).bind(this),- );+ poll();+ }+ }).bind(this),+ );++ const result = await done;++ setSpanAttributes(span, {+ "nuq.wait_duration_ms": Date.now() - startTime,+ "nuq.wait_success": true,+ });- return done;+ return result;+ });}// === Consumer@@ -468,27 +524,43 @@ class NuQ<JobData = any, JobReturnValue = any> {returnvalue: any | null,_logger: Logger = logger,): Promise<boolean> {- const start = Date.now();- try {- return (- (- await nuqPool.query(- `- WITH updated AS (UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id)- SELECT pg_notify('${this.queueName}', (id::text || '|completed')) FROM updated;- `,- [id, lock, returnvalue],- )- ).rowCount !== 0- );- } finally {- _logger.info("nuqJobFinish metrics", {- module: "nuq/metrics",- method: "nuqJobFinish",- duration: Date.now() - start,- scrapeId: id,+ return withSpan("nuq.jobFinish", async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,});- }++ const start = Date.now();+ try {+ const result =+ (+ await nuqPool.query(+ `+ WITH updated AS (UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id)+ SELECT pg_notify('${this.queueName}', (id::text || '|completed')) FROM updated;+ `,+ [id, lock, returnvalue],+ )+ ).rowCount !== 0;++ setSpanAttributes(span, {+ "nuq.job_finished": result,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ _logger.info("nuqJobFinish metrics", {+ module: "nuq/metrics",+ method: "nuqJobFinish",+ duration,+ scrapeId: id,+ });+ }+ });}public async jobFail(@@ -497,27 +569,44 @@ class NuQ<JobData = any, JobReturnValue = any> {failedReason: string,_logger: Logger = logger,): Promise<boolean> {- const start = Date.now();- try {- return (- (- await nuqPool.query(- `- WITH updated AS (UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id)- SELECT pg_notify('${this.queueName}', (id::text || '|failed')) FROM updated;- `,- [id, lock, failedReason],- )- ).rowCount !== 0- );- } finally {- _logger.info("nuqJobFail metrics", {- module: "nuq/metrics",- method: "nuqJobFail",- duration: Date.now() - start,- scrapeId: id,+ return withSpan("nuq.jobFail", async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,+ "nuq.failed_reason": failedReason,});- }++ const start = Date.now();+ try {+ const result =+ (+ await nuqPool.query(+ `+ WITH updated AS (UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id)+ SELECT pg_notify('${this.queueName}', (id::text || '|failed')) FROM updated;+ `,+ [id, lock, failedReason],+ )+ ).rowCount !== 0;++ setSpanAttributes(span, {+ "nuq.job_failed": result,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ _logger.info("nuqJobFail metrics", {+ module: "nuq/metrics",+ method: "nuqJobFail",+ duration,+ scrapeId: id,+ });+ }+ });}// === Metricsdiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 17bcd4bea..674929d76 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -66,6 +66,11 @@ import {} from "../../types";import { scrapeSitemap } from "../../scraper/crawler/sitemap";import { shutdownOtel } from "../../otel";+import {+ withTraceContextAsync,+ withSpan,+ setSpanAttributes,+} from "../../lib/otel-tracer";const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));@@ -1055,6 +1060,27 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {zeroDataRetention: job.data?.zeroDataRetention ?? false,});+ // Restore trace context if available and execute within span+ if (job.data.traceContext) {+ return withTraceContextAsync(job.data.traceContext, () =>+ withSpan("worker.scrape.process", async span => {+ setSpanAttributes(span, {+ "worker.job_id": job.id,+ "worker.mode": job.data.mode,+ "worker.team_id": job.data.team_id,+ "worker.crawl_id": job.data.crawl_id || "none",+ "worker.url": job.data.mode === "single_urls" ? job.data.url : "n/a",+ });++ return processJobWithTracing(job, logger);+ }),+ );+ } else {+ return processJobWithTracing(job, logger);+ }+};++async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {try {try {let extendLockInterval: NodeJS.Timeout | null = null;@@ -1135,7 +1161,7 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {throw new Error(serializeTransportableError(new UnknownError(error)));}}-};+}const exitHandler = () => {shutdownOtel().finally(() => {diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 0690ec1e2..86f96b6b9 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -10,11 +10,13 @@ import { ExtractorOptions, Document } from "./lib/entities";import { InternalOptions } from "./scraper/scrapeURL";import type { CostTracking } from "./lib/cost-tracking";import { webhookSchema } from "./services/webhook/schema";+import { SerializedTraceContext } from "./lib/otel-tracer";type ScrapeJobCommon = {concurrencyLimited?: boolean;team_id: string;zeroDataRetention: boolean;+ traceContext?: SerializedTraceContext;};export type ScrapeJobData = ScrapeJobCommon &