Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/otel): much improved tracing (#2219)

Gergő Móricz2bf96926497e
diff --git a/apps/api/package.json b/apps/api/package.json
index 69245b839..0b6181a09 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -77,6 +77,7 @@
"@google-cloud/storage": "^7.16.0",
"@mendable/firecrawl-rs": "workspace:*",
"@openrouter/ai-sdk-provider": "^0.4.5",
+ "@opentelemetry/api": "^1.9.0",
"@opentelemetry/auto-instrumentations-node": "^0.62.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/exporter-trace-otlp-http": "^0.205.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index 6e7dc8b3d..afcfca3f7 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -60,6 +60,9 @@ importers:
'@openrouter/ai-sdk-provider':
specifier: ^0.4.5
version: 0.4.5([email protected])
+ '@opentelemetry/api':
+ specifier: ^1.9.0
+ version: 1.9.0
'@opentelemetry/auto-instrumentations-node':
specifier: ^0.62.0
version: 0.62.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected])
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 22d69d7a9..48d662c0a 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -14,161 +14,262 @@ import { hasFormatOfType } from "../../lib/format-utils";
import { TransportableError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
+import { withSpan, setSpanAttributes, SpanKind } from "../../lib/otel-tracer";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
res: Response<ScrapeResponse>,
) {
- // Get timing data from middleware (includes all middleware processing time)
- const middlewareStartTime = (req as any).requestTiming?.startTime || new Date().getTime();
- const controllerStartTime = new Date().getTime();
-
- const jobId = uuidv4();
- const preNormalizedBody = { ...req.body };
- req.body = scrapeRequestSchema.parse(req.body);
-
- const permissions = checkPermissions(req.body, req.acuc?.flags);
- if (permissions.error) {
- return res.status(403).json({
- success: false,
- error: permissions.error,
- });
- }
-
- const zeroDataRetention =
- req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
-
- const logger = _logger.child({
- method: "scrapeController",
- jobId,
- scrapeId: jobId,
- teamId: req.auth.team_id,
- team_id: req.auth.team_id,
- zeroDataRetention,
- });
-
- const middlewareTime = controllerStartTime - middlewareStartTime;
-
- logger.debug("Scrape " + jobId + " starting", {
- version: "v2",
- scrapeId: jobId,
- request: req.body,
- originalRequest: preNormalizedBody,
- account: req.account,
- });
-
- const origin = req.body.origin;
- const timeout = req.body.timeout;
-
-
- const isDirectToBullMQ =
- process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
- process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
-
- const jobPriority = await getJobPriority({
- team_id: req.auth.team_id,
- basePriority: 10,
- });
-
- const job = await addScrapeJob(
- {
- url: req.body.url,
- mode: "single_urls",
- team_id: req.auth.team_id,
- scrapeOptions: {
- ...req.body,
- ...(req.body.__experimental_cache
- ? {
- maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
- }
- : {}),
- },
- internalOptions: {
+ return withSpan(
+ "api.scrape.request",
+ async span => {
+ // Get timing data from middleware (includes all middleware processing time)
+ const middlewareStartTime =
+ (req as any).requestTiming?.startTime || new Date().getTime();
+ const controllerStartTime = new Date().getTime();
+
+ const jobId = uuidv4();
+ const preNormalizedBody = { ...req.body };
+
+ // Set initial span attributes
+ setSpanAttributes(span, {
+ "scrape.job_id": jobId,
+ "scrape.url": req.body.url,
+ "scrape.team_id": req.auth.team_id,
+ "scrape.api_key_id": req.acuc?.api_key_id,
+ "scrape.middleware_time_ms": controllerStartTime - middlewareStartTime,
+ });
+
+ // Validation span
+ await withSpan("api.scrape.validate", async validateSpan => {
+ req.body = scrapeRequestSchema.parse(req.body);
+ setSpanAttributes(validateSpan, {
+ "validation.success": true,
+ });
+ });
+
+ // Permission check span
+ const permissions = await withSpan(
+ "api.scrape.check_permissions",
+ async permSpan => {
+ const perms = checkPermissions(req.body, req.acuc?.flags);
+ setSpanAttributes(permSpan, {
+ "permissions.success": !perms.error,
+ "permissions.error": perms.error,
+ });
+ return perms;
+ },
+ );
+
+ if (permissions.error) {
+ setSpanAttributes(span, {
+ "scrape.error": permissions.error,
+ "scrape.status_code": 403,
+ });
+ return res.status(403).json({
+ success: false,
+ error: permissions.error,
+ });
+ }
+
+ const zeroDataRetention =
+ req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
+
+ const logger = _logger.child({
+ method: "scrapeController",
+ jobId,
+ scrapeId: jobId,
teamId: req.auth.team_id,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
- ? true
- : false,
- unnormalizedSourceURL: preNormalizedBody.url,
- bypassBilling: isDirectToBullMQ,
+ team_id: req.auth.team_id,
zeroDataRetention,
- teamFlags: req.acuc?.flags ?? null,
- },
- origin,
- integration: req.body.integration,
- startTime: controllerStartTime,
- zeroDataRetention,
- apiKeyId: req.acuc?.api_key_id ?? null,
- },
- jobId,
- jobPriority,
- isDirectToBullMQ,
- );
+ });
+
+ const middlewareTime = controllerStartTime - middlewareStartTime;
+
+ logger.debug("Scrape " + jobId + " starting", {
+ version: "v2",
+ scrapeId: jobId,
+ request: req.body,
+ originalRequest: preNormalizedBody,
+ account: req.account,
+ });
+
+ setSpanAttributes(span, {
+ "scrape.zero_data_retention": zeroDataRetention,
+ "scrape.origin": req.body.origin,
+ "scrape.timeout": req.body.timeout,
+ });
+
+ const origin = req.body.origin;
+ const timeout = req.body.timeout;
+
+ const isDirectToBullMQ =
+ process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
+ process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+
+ const jobPriority = await getJobPriority({
+ team_id: req.auth.team_id,
+ basePriority: 10,
+ });
+
+ // Job enqueuing span
+ const job = await withSpan(
+ "api.scrape.enqueue_job",
+ async enqueueSpan => {
+ const result = await addScrapeJob(
+ {
+ url: req.body.url,
+ mode: "single_urls",
+ team_id: req.auth.team_id,
+ scrapeOptions: {
+ ...req.body,
+ ...(req.body.__experimental_cache
+ ? {
+ maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
+ }
+ : {}),
+ },
+ internalOptions: {
+ teamId: req.auth.team_id,
+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
+ ? true
+ : false,
+ unnormalizedSourceURL: preNormalizedBody.url,
+ bypassBilling: isDirectToBullMQ,
+ zeroDataRetention,
+ teamFlags: req.acuc?.flags ?? null,
+ },
+ origin,
+ integration: req.body.integration,
+ startTime: controllerStartTime,
+ zeroDataRetention,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ },
+ jobId,
+ jobPriority,
+ isDirectToBullMQ,
+ );
+
+ setSpanAttributes(enqueueSpan, {
+ "job.priority": jobPriority,
+ "job.direct_to_bullmq": isDirectToBullMQ,
+ });
+
+ return result;
+ },
+ );
+
+ const totalWait =
+ (req.body.waitFor ?? 0) +
+ (req.body.actions ?? []).reduce(
+ (a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,
+ 0,
+ );
+
+ let doc: Document;
+ try {
+ // Wait for job completion span
+ doc = await withSpan("api.scrape.wait_for_job", async waitSpan => {
+ setSpanAttributes(waitSpan, {
+ "wait.timeout": timeout !== undefined ? timeout + totalWait : null,
+ "wait.job_id": jobId,
+ });
+
+ const result = await waitForJob(
+ job ?? jobId,
+ timeout !== undefined ? timeout + totalWait : null,
+ zeroDataRetention,
+ logger,
+ );
+
+ setSpanAttributes(waitSpan, {
+ "wait.success": true,
+ });
+
+ return result;
+ });
+ } catch (e) {
+ logger.error(`Error in scrapeController`, {
+ version: "v2",
+ error: e,
+ });
+
+ setSpanAttributes(span, {
+ "scrape.error": e instanceof Error ? e.message : String(e),
+ "scrape.error_type":
+ e instanceof TransportableError ? e.code : "unknown",
+ });
+
+ if (zeroDataRetention) {
+ await scrapeQueue.removeJob(jobId, logger);
+ }
+
+ if (e instanceof TransportableError) {
+ const statusCode = e.code === "SCRAPE_TIMEOUT" ? 408 : 500;
+ setSpanAttributes(span, {
+ "scrape.status_code": statusCode,
+ });
+ return res.status(statusCode).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ } else {
+ setSpanAttributes(span, {
+ "scrape.status_code": 500,
+ });
+ return res.status(500).json({
+ success: false,
+ error: `(Internal server error) - ${e && e.message ? e.message : e}`,
+ });
+ }
+ }
- const totalWait =
- (req.body.waitFor ?? 0) +
- (req.body.actions ?? []).reduce(
- (a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,
- 0,
- );
-
- let doc: Document;
- try {
- doc = await waitForJob(
- job ?? jobId,
- timeout !== undefined ? timeout + totalWait : null,
- zeroDataRetention,
- logger,
- );
- } catch (e) {
- logger.error(`Error in scrapeController`, {
- version: "v2",
- error: e,
- });
-
- if (zeroDataRetention) {
await scrapeQueue.removeJob(jobId, logger);
- }
- if (e instanceof TransportableError) {
- return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({
- success: false,
- code: e.code,
- error: e.message,
+ if (!hasFormatOfType(req.body.formats, "rawHtml")) {
+ if (doc && doc.rawHtml) {
+ delete doc.rawHtml;
+ }
+ }
+
+ const totalRequestTime = new Date().getTime() - middlewareStartTime;
+ const controllerTime = new Date().getTime() - controllerStartTime;
+
+ // Set final span attributes
+ setSpanAttributes(span, {
+ "scrape.success": true,
+ "scrape.status_code": 200,
+ "scrape.total_request_time_ms": totalRequestTime,
+ "scrape.controller_time_ms": controllerTime,
+ "scrape.document.status_code": doc?.metadata?.statusCode,
+ "scrape.document.content_type": doc?.metadata?.contentType,
+ "scrape.document.error": doc?.metadata?.error,
});
- } else {
- return res.status(500).json({
- success: false,
- error: `(Internal server error) - ${e && e.message ? e.message : e}`,
+
+ logger.info("Request metrics", {
+ version: "v2",
+ scrapeId: jobId,
+ mode: "scrape",
+ middlewareStartTime,
+ controllerStartTime,
+ middlewareTime,
+ controllerTime,
+ totalRequestTime,
+ });
+
+ return res.status(200).json({
+ success: true,
+ data: doc,
+ scrape_id: origin?.includes("website") ? jobId : undefined,
});
- }
- }
-
- await scrapeQueue.removeJob(jobId, logger);
-
-
- if (!hasFormatOfType(req.body.formats, "rawHtml")) {
- if (doc && doc.rawHtml) {
- delete doc.rawHtml;
- }
- }
-
- const totalRequestTime = new Date().getTime() - middlewareStartTime;
- const controllerTime = new Date().getTime() - controllerStartTime;
- logger.info("Request metrics", {
- version: "v2",
- scrapeId: jobId,
- mode: "scrape",
- middlewareStartTime,
- controllerStartTime,
- middlewareTime,
- controllerTime,
- totalRequestTime,
- });
-
-
- return res.status(200).json({
- success: true,
- data: doc,
- scrape_id: origin?.includes("website") ? jobId : undefined,
- });
+ },
+ {
+ attributes: {
+ "http.method": "POST",
+ "http.route": "/v2/scrape",
+ },
+ kind: SpanKind.SERVER,
+ },
+ );
}
diff --git a/apps/api/src/lib/otel-tracer.ts b/apps/api/src/lib/otel-tracer.ts
new file mode 100644
index 000000000..3ef40ae7b
--- /dev/null
+++ b/apps/api/src/lib/otel-tracer.ts
@@ -0,0 +1,173 @@
+import {
+ trace,
+ context,
+ SpanStatusCode,
+ Span,
+ SpanKind,
+ Attributes,
+ propagation,
+} from "@opentelemetry/api";
+// import { Logger } from 'winston';
+
+const tracer = trace.getTracer("firecrawl-api", "1.0.0");
+
+export { SpanKind };
+
+// Trace context propagation utilities
+export interface SerializedTraceContext {
+ traceParent?: string;
+ traceState?: string;
+}
+
+export function serializeTraceContext(): SerializedTraceContext {
+ const carrier: Record<string, string> = {};
+ propagation.inject(context.active(), carrier);
+
+ return {
+ traceParent: carrier.traceparent,
+ traceState: carrier.tracestate,
+ };
+}
+
+function deserializeTraceContext(serialized: SerializedTraceContext): any {
+ if (!serialized.traceParent) {
+ return context.active();
+ }
+
+ const carrier: Record<string, string> = {
+ traceparent: serialized.traceParent,
+ };
+
+ if (serialized.traceState) {
+ carrier.tracestate = serialized.traceState;
+ }
+
+ return propagation.extract(context.active(), carrier);
+}
+
+// export function withTraceContext<T>(serializedContext: SerializedTraceContext, fn: () => T): T {
+// const ctx = deserializeTraceContext(serializedContext);
+// return context.with(ctx, fn);
+// }
+
+export async function withTraceContextAsync<T>(
+ serializedContext: SerializedTraceContext,
+ fn: () => Promise<T>,
+): Promise<T> {
+ const ctx = deserializeTraceContext(serializedContext);
+ return context.with(ctx, fn);
+}
+
+interface SpanOptions {
+ attributes?: Attributes;
+ kind?: SpanKind;
+}
+
+function startSpan(name: string, options?: SpanOptions): Span {
+ return tracer.startSpan(name, {
+ attributes: options?.attributes,
+ kind: options?.kind || SpanKind.INTERNAL,
+ });
+}
+
+export async function withSpan<T>(
+ name: string,
+ fn: (span: Span) => Promise<T>,
+ options?: SpanOptions,
+): Promise<T> {
+ const span = startSpan(name, options);
+
+ try {
+ const result = await context.with(
+ trace.setSpan(context.active(), span),
+ () => fn(span),
+ );
+ span.setStatus({ code: SpanStatusCode.OK });
+ return result;
+ } catch (error) {
+ span.setStatus({
+ code: SpanStatusCode.ERROR,
+ message: error instanceof Error ? error.message : String(error),
+ });
+
+ if (error instanceof Error) {
+ span.recordException(error);
+ }
+
+ throw error;
+ } finally {
+ span.end();
+ }
+}
+
+// export function withSpanSync<T>(
+// name: string,
+// fn: (span: Span) => T,
+// options?: SpanOptions
+// ): T {
+// const span = startSpan(name, options);
+
+// try {
+// const result = context.with(
+// trace.setSpan(context.active(), span),
+// () => fn(span)
+// );
+// span.setStatus({ code: SpanStatusCode.OK });
+// return result;
+// } catch (error) {
+// span.setStatus({
+// code: SpanStatusCode.ERROR,
+// message: error instanceof Error ? error.message : String(error),
+// });
+
+// if (error instanceof Error) {
+// span.recordException(error);
+// }
+
+// throw error;
+// } finally {
+// span.end();
+// }
+// }
+
+export function setSpanAttributes(span: Span, attributes: Attributes): void {
+ Object.entries(attributes).forEach(([key, value]) => {
+ if (value !== undefined && value !== null) {
+ span.setAttribute(key, value);
+ }
+ });
+}
+
+// export function setSpanError(span: Span, error: Error | unknown, logger?: Logger): void {
+// const errorMessage = error instanceof Error ? error.message : String(error);
+
+// span.setStatus({
+// code: SpanStatusCode.ERROR,
+// message: errorMessage,
+// });
+
+// if (error instanceof Error) {
+// span.recordException(error);
+// span.setAttributes({
+// 'error.type': error.constructor.name,
+// 'error.message': error.message,
+// 'error.stack': error.stack,
+// });
+// }
+
+// if (logger) {
+// logger.error('Span error recorded', { error, spanName: span.spanContext().traceId });
+// }
+// }
+
+// export function getActiveSpan(): Span | undefined {
+// return trace.getActiveSpan();
+// }
+
+// export function createChildSpan(name: string, parentSpan?: Span, options?: SpanOptions): Span {
+// const ctx = parentSpan
+// ? trace.setSpan(context.active(), parentSpan)
+// : context.active();
+
+// return context.with(ctx, () => startSpan(name, options));
+// }
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 116768348..b8d575112 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -32,6 +32,7 @@ import { hasFormatOfType } from "../../../../lib/format-utils";
import { Action } from "../../../../controllers/v1/types";
import { AbortManagerThrownError } from "../../lib/abortManager";
import { youtubePostprocessor } from "../../postprocessors/youtube";
+import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
// This function does not take `Meta` on purpose. It may not access any
// meta values to construct the request -- that must be done by the
@@ -49,407 +50,457 @@ async function performFireEngineScrape<
abort?: AbortSignal,
production = true,
): Promise<FireEngineCheckStatusSuccess> {
- const scrape = await fireEngineScrape(
- meta,
- logger.child({ method: "fireEngineScrape" }),
- request,
- mock,
- abort,
- production,
- );
-
- let status: FireEngineCheckStatusSuccess | undefined = undefined;
- if ((scrape as any).processing) {
- const errorLimit = 3;
- let errors: any[] = [];
-
- while (status === undefined) {
- if (errors.length >= errorLimit) {
- logger.error("Error limit hit.", { errors });
- fireEngineDelete(
- logger.child({
- method: "performFireEngineScrape/fireEngineDelete",
- afterErrors: errors,
- }),
- (scrape as any).jobId,
- mock,
- undefined,
- production,
- );
- throw new Error("Error limit hit. See e.cause.errors for errors.", {
- cause: { errors },
- });
- }
+ return withSpan("engine.fire-engine.perform_scrape", async span => {
+ const startTime = Date.now();
+ let pollCount = 0;
+
+ setSpanAttributes(span, {
+ "fire-engine.url": request.url,
+ "fire-engine.priority": request.priority,
+ "fire-engine.wait": (request as any).wait,
+ "fire-engine.screenshot": (request as any).screenshot,
+ "fire-engine.fullpage": (request as any).fullPage,
+ "fire-engine.proxy": (request as any).proxy,
+ "fire-engine.mobile": (request as any).mobile,
+ "fire-engine.skip_tls": (request as any).skipTlsVerification,
+ "fire-engine.production": production,
+ });
+ const scrape = await fireEngineScrape(
+ meta,
+ logger.child({ method: "fireEngineScrape" }),
+ request,
+ mock,
+ abort,
+ production,
+ );
+
+ let status: FireEngineCheckStatusSuccess | undefined = undefined;
+ if ((scrape as any).processing) {
+ const errorLimit = 3;
+ let errors: any[] = [];
- meta.abort.throwIfAborted();
-
- try {
- status = await fireEngineCheckStatus(
- meta,
- logger.child({ method: "fireEngineCheckStatus" }),
- (scrape as any).jobId,
- mock,
- abort,
- production,
- );
- } catch (error) {
- if (error instanceof StillProcessingError) {
- // nop
- } else if (
- error instanceof EngineError ||
- error instanceof SiteError ||
- error instanceof SSLError ||
- error instanceof DNSResolutionError ||
- error instanceof ActionError ||
- error instanceof UnsupportedFileError ||
- error instanceof FEPageLoadFailed ||
- error instanceof ProxySelectionError
- ) {
+ while (status === undefined) {
+ if (errors.length >= errorLimit) {
+ logger.error("Error limit hit.", { errors });
fireEngineDelete(
logger.child({
method: "performFireEngineScrape/fireEngineDelete",
- afterError: error,
+ afterErrors: errors,
}),
(scrape as any).jobId,
mock,
undefined,
production,
);
- logger.debug("Fire-engine scrape job failed.", {
- error,
- jobId: (scrape as any).jobId,
+ throw new Error("Error limit hit. See e.cause.errors for errors.", {
+ cause: { errors },
});
- throw error;
- } else if (error instanceof AbortManagerThrownError) {
- fireEngineDelete(
- logger.child({
- method: "performFireEngineScrape/fireEngineDelete",
- afterError: error,
- }),
+ }
+
+ meta.abort.throwIfAborted();
+
+ try {
+ pollCount++;
+ status = await fireEngineCheckStatus(
+ meta,
+ logger.child({ method: "fireEngineCheckStatus" }),
(scrape as any).jobId,
mock,
- undefined,
+ abort,
production,
);
- throw error;
- } else {
- errors.push(error);
- logger.debug(
- `An unexpeceted error occurred while calling checkStatus. Error counter is now at ${errors.length}.`,
- { error, jobId: (scrape as any).jobId },
- );
- Sentry.captureException(error);
+ } catch (error) {
+ if (error instanceof StillProcessingError) {
+ // nop
+ } else if (
+ error instanceof EngineError ||
+ error instanceof SiteError ||
+ error instanceof SSLError ||
+ error instanceof DNSResolutionError ||
+ error instanceof ActionError ||
+ error instanceof UnsupportedFileError ||
+ error instanceof FEPageLoadFailed ||
+ error instanceof ProxySelectionError
+ ) {
+ fireEngineDelete(
+ logger.child({
+ method: "performFireEngineScrape/fireEngineDelete",
+ afterError: error,
+ }),
+ (scrape as any).jobId,
+ mock,
+ undefined,
+ production,
+ );
+ logger.debug("Fire-engine scrape job failed.", {
+ error,
+ jobId: (scrape as any).jobId,
+ });
+ throw error;
+ } else if (error instanceof AbortManagerThrownError) {
+ fireEngineDelete(
+ logger.child({
+ method: "performFireEngineScrape/fireEngineDelete",
+ afterError: error,
+ }),
+ (scrape as any).jobId,
+ mock,
+ undefined,
+ production,
+ );
+ throw error;
+ } else {
+ errors.push(error);
+ logger.debug(
+ `An unexpeceted error occurred while calling checkStatus. Error counter is now at ${errors.length}.`,
+ { error, jobId: (scrape as any).jobId },
+ );
+ Sentry.captureException(error);
+ }
}
}
+
+ await new Promise(resolve => setTimeout(resolve, 500));
+ } else {
+ status = scrape as FireEngineCheckStatusSuccess;
}
- await new Promise(resolve => setTimeout(resolve, 500));
- } else {
- status = scrape as FireEngineCheckStatusSuccess;
- }
+ await specialtyScrapeCheck(
+ logger.child({
+ method: "performFireEngineScrape/specialtyScrapeCheck",
+ }),
+ status.responseHeaders,
+ status,
+ );
- await specialtyScrapeCheck(
- logger.child({
- method: "performFireEngineScrape/specialtyScrapeCheck",
- }),
- status.responseHeaders,
- status,
- );
-
- const contentType =
- (Object.entries(status.responseHeaders ?? {}).find(
- x => x[0].toLowerCase() === "content-type",
- ) ?? [])[1] ?? "";
-
- if (contentType.includes("application/json")) {
- status.content = await getInnerJson(status.content);
- }
+ const contentType =
+ (Object.entries(status.responseHeaders ?? {}).find(
+ x => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? "";
- if (status.file) {
- const content = status.file.content;
- delete status.file;
- status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag
- }
+ if (contentType.includes("application/json")) {
+ status.content = await getInnerJson(status.content);
+ }
- fireEngineDelete(
- logger.child({
- method: "performFireEngineScrape/fireEngineDelete",
- }),
- (scrape as any).jobId,
- mock,
- undefined,
- production,
- );
-
- return status;
+ if (status.file) {
+ const content = status.file.content;
+ delete status.file;
+ status.content = Buffer.from(content, "base64").toString("utf8"); // TODO: handle other encodings via Content-Type tag
+ }
+
+ fireEngineDelete(
+ logger.child({
+ method: "performFireEngineScrape/fireEngineDelete",
+ }),
+ (scrape as any).jobId,
+ mock,
+ undefined,
+ production,
+ );
+
+ setSpanAttributes(span, {
+ "fire-engine.poll_count": pollCount,
+ "fire-engine.duration_ms": Date.now() - startTime,
+ "fire-engine.status_code": status.pageStatusCode,
+ "fire-engine.content_length": status.content?.length,
+ "fire-engine.has_screenshot": !!status.screenshot,
+ "fire-engine.has_pdf": !!(status as any).pdf,
+ "fire-engine.job_id": (scrape as any).jobId,
+ });
+
+ return status;
+ });
}
export async function scrapeURLWithFireEngineChromeCDP(
meta: Meta,
): Promise<EngineScrapeResult> {
- const actions: Action[] = [
- // Transform waitFor option into an action (unsupported by chrome-cdp)
- ...(meta.options.waitFor !== 0
- ? [
- {
- type: "wait" as const,
- milliseconds:
- meta.options.waitFor > 30000 ? 30000 : meta.options.waitFor,
- },
- ]
- : []),
-
- // Include specified actions
- ...(meta.options.actions ?? []),
-
- // Transform screenshot format into an action (unsupported by chrome-cdp)
- ...(hasFormatOfType(meta.options.formats, "screenshot")
- ? [
- {
- type: "screenshot" as const,
- fullPage:
- hasFormatOfType(meta.options.formats, "screenshot")?.fullPage ||
- false,
- ...(hasFormatOfType(meta.options.formats, "screenshot")?.viewport
- ? {
- viewport: hasFormatOfType(meta.options.formats, "screenshot")!
- .viewport,
- }
- : {}),
- },
- ]
- : []),
- ];
-
- const totalWait = actions.reduce(
- (a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),
- 0,
- );
-
- const request: FireEngineScrapeRequestCommon &
- FireEngineScrapeRequestChromeCDP = {
- url: meta.rewrittenUrl ?? meta.url,
- engine: "chrome-cdp",
- instantReturn: false,
- skipTlsVerification: meta.options.skipTlsVerification,
- headers: meta.options.headers,
- ...(actions.length > 0
- ? {
- actions,
- }
- : {}),
- priority: meta.internalOptions.priority,
- geolocation: meta.options.location,
- mobile: meta.options.mobile,
- timeout: meta.abort.scrapeTimeout() ?? 300000,
- disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,
- mobileProxy: meta.featureFlags.has("stealthProxy"),
- saveScrapeResultToGCS:
- !meta.internalOptions.zeroDataRetention &&
- meta.internalOptions.saveScrapeResultToGCS,
- zeroDataRetention: meta.internalOptions.zeroDataRetention,
- ...(youtubePostprocessor.shouldRun(
- meta,
- new URL(meta.rewrittenUrl ?? meta.url),
- )
- ? { blockMedia: false }
- : {}),
- };
-
- let response = await performFireEngineScrape(
- meta,
- meta.logger.child({
- method: "scrapeURLWithFireEngineChromeCDP/callFireEngine",
- request,
- }),
- request,
- meta.mock,
- meta.abort.asSignal(),
- true,
- );
-
- if (hasFormatOfType(meta.options.formats, "screenshot")) {
- // meta.logger.debug(
- // "Transforming screenshots from actions into screenshot field",
- // { screenshots: response.screenshots },
- // );
- if (response.screenshots) {
- response.screenshot = response.screenshots.slice(-1)[0];
- response.screenshots = response.screenshots.slice(0, -1);
- }
- // meta.logger.debug("Screenshot transformation done", {
- // screenshots: response.screenshots,
- // screenshot: response.screenshot,
- // });
- }
-
- if (!response.url) {
- meta.logger.warn("Fire-engine did not return the response's URL", {
- response,
- sourceURL: meta.url,
+ return withSpan("engine.fire-engine.chrome-cdp", async span => {
+ setSpanAttributes(span, {
+ "engine.type": "fire-engine-chrome-cdp",
+ "engine.url": meta.url,
+ "engine.team_id": meta.internalOptions.teamId,
});
- }
+ const actions: Action[] = [
+ // Transform waitFor option into an action (unsupported by chrome-cdp)
+ ...(meta.options.waitFor !== 0
+ ? [
+ {
+ type: "wait" as const,
+ milliseconds:
+ meta.options.waitFor > 30000 ? 30000 : meta.options.waitFor,
+ },
+ ]
+ : []),
+
+ // Include specified actions
+ ...(meta.options.actions ?? []),
+
+ // Transform screenshot format into an action (unsupported by chrome-cdp)
+ ...(hasFormatOfType(meta.options.formats, "screenshot")
+ ? [
+ {
+ type: "screenshot" as const,
+ fullPage:
+ hasFormatOfType(meta.options.formats, "screenshot")?.fullPage ||
+ false,
+ ...(hasFormatOfType(meta.options.formats, "screenshot")?.viewport
+ ? {
+ viewport: hasFormatOfType(
+ meta.options.formats,
+ "screenshot",
+ )!.viewport,
+ }
+ : {}),
+ },
+ ]
+ : []),
+ ];
+
+ const totalWait = actions.reduce(
+ (a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),
+ 0,
+ );
- return {
- url: response.url ?? meta.url,
+ const request: FireEngineScrapeRequestCommon &
+ FireEngineScrapeRequestChromeCDP = {
+ url: meta.rewrittenUrl ?? meta.url,
+ engine: "chrome-cdp",
+ instantReturn: false,
+ skipTlsVerification: meta.options.skipTlsVerification,
+ headers: meta.options.headers,
+ ...(actions.length > 0
+ ? {
+ actions,
+ }
+ : {}),
+ priority: meta.internalOptions.priority,
+ geolocation: meta.options.location,
+ mobile: meta.options.mobile,
+ timeout: meta.abort.scrapeTimeout() ?? 300000,
+ disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,
+ mobileProxy: meta.featureFlags.has("stealthProxy"),
+ saveScrapeResultToGCS:
+ !meta.internalOptions.zeroDataRetention &&
+ meta.internalOptions.saveScrapeResultToGCS,
+ zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ ...(youtubePostprocessor.shouldRun(
+ meta,
+ new URL(meta.rewrittenUrl ?? meta.url),
+ )
+ ? { blockMedia: false }
+ : {}),
+ };
+
+ let response = await performFireEngineScrape(
+ meta,
+ meta.logger.child({
+ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine",
+ request,
+ }),
+ request,
+ meta.mock,
+ meta.abort.asSignal(),
+ true,
+ );
- html: response.content,
- error: response.pageError,
- statusCode: response.pageStatusCode,
+ if (hasFormatOfType(meta.options.formats, "screenshot")) {
+ // meta.logger.debug(
+ // "Transforming screenshots from actions into screenshot field",
+ // { screenshots: response.screenshots },
+ // );
+ if (response.screenshots) {
+ response.screenshot = response.screenshots.slice(-1)[0];
+ response.screenshots = response.screenshots.slice(0, -1);
+ }
+ // meta.logger.debug("Screenshot transformation done", {
+ // screenshots: response.screenshots,
+ // screenshot: response.screenshot,
+ // });
+ }
- contentType:
- (Object.entries(response.responseHeaders ?? {}).find(
- x => x[0].toLowerCase() === "content-type",
- ) ?? [])[1] ?? undefined,
-
- screenshot: response.screenshot,
- ...(actions.length > 0
- ? {
- actions: {
- screenshots: response.screenshots ?? [],
- scrapes: response.actionContent ?? [],
- javascriptReturns: (response.actionResults ?? [])
- .filter(x => x.type === "executeJavascript")
- .map(x =>
- JSON.parse((x.result as any as { return: string }).return),
- ),
- pdfs: (response.actionResults ?? [])
- .filter(x => x.type === "pdf")
- .map(x => x.result.link),
- },
- }
- : {}),
+ if (!response.url) {
+ meta.logger.warn("Fire-engine did not return the response's URL", {
+ response,
+ sourceURL: meta.url,
+ });
+ }
- proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- youtubeTranscriptContent: response.youtubeTranscriptContent,
- };
+ return {
+ url: response.url ?? meta.url,
+
+ html: response.content,
+ error: response.pageError,
+ statusCode: response.pageStatusCode,
+
+ contentType:
+ (Object.entries(response.responseHeaders ?? {}).find(
+ x => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined,
+
+ screenshot: response.screenshot,
+ ...(actions.length > 0
+ ? {
+ actions: {
+ screenshots: response.screenshots ?? [],
+ scrapes: response.actionContent ?? [],
+ javascriptReturns: (response.actionResults ?? [])
+ .filter(x => x.type === "executeJavascript")
+ .map(x =>
+ JSON.parse((x.result as any as { return: string }).return),
+ ),
+ pdfs: (response.actionResults ?? [])
+ .filter(x => x.type === "pdf")
+ .map(x => x.result.link),
+ },
+ }
+ : {}),
+
+ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
+ youtubeTranscriptContent: response.youtubeTranscriptContent,
+ };
+ });
}
export async function scrapeURLWithFireEnginePlaywright(
meta: Meta,
): Promise<EngineScrapeResult> {
- const totalWait = meta.options.waitFor;
-
- const request: FireEngineScrapeRequestCommon &
- FireEngineScrapeRequestPlaywright = {
- url: meta.rewrittenUrl ?? meta.url,
- engine: "playwright",
- instantReturn: false,
-
- headers: meta.options.headers,
- priority: meta.internalOptions.priority,
- screenshot:
- hasFormatOfType(meta.options.formats, "screenshot") !== undefined,
- fullPageScreenshot: hasFormatOfType(meta.options.formats, "screenshot")
- ?.fullPage,
- wait: meta.options.waitFor,
- geolocation: meta.options.location,
- blockAds: meta.options.blockAds,
- mobileProxy: meta.featureFlags.has("stealthProxy"),
-
- timeout: meta.abort.scrapeTimeout() ?? 300000,
- saveScrapeResultToGCS:
- !meta.internalOptions.zeroDataRetention &&
- meta.internalOptions.saveScrapeResultToGCS,
- zeroDataRetention: meta.internalOptions.zeroDataRetention,
- };
-
- let response = await performFireEngineScrape(
- meta,
- meta.logger.child({
- method: "scrapeURLWithFireEnginePlaywright/callFireEngine",
- request,
- }),
- request,
- meta.mock,
- meta.abort.asSignal(),
- );
-
- if (!response.url) {
- meta.logger.warn("Fire-engine did not return the response's URL", {
- response,
- sourceURL: meta.url,
+ return withSpan("engine.fire-engine.playwright", async span => {
+ setSpanAttributes(span, {
+ "engine.type": "fire-engine-playwright",
+ "engine.url": meta.url,
+ "engine.team_id": meta.internalOptions.teamId,
});
- }
+ const totalWait = meta.options.waitFor;
+
+ const request: FireEngineScrapeRequestCommon &
+ FireEngineScrapeRequestPlaywright = {
+ url: meta.rewrittenUrl ?? meta.url,
+ engine: "playwright",
+ instantReturn: false,
+
+ headers: meta.options.headers,
+ priority: meta.internalOptions.priority,
+ screenshot:
+ hasFormatOfType(meta.options.formats, "screenshot") !== undefined,
+ fullPageScreenshot: hasFormatOfType(meta.options.formats, "screenshot")
+ ?.fullPage,
+ wait: meta.options.waitFor,
+ geolocation: meta.options.location,
+ blockAds: meta.options.blockAds,
+ mobileProxy: meta.featureFlags.has("stealthProxy"),
+
+ timeout: meta.abort.scrapeTimeout() ?? 300000,
+ saveScrapeResultToGCS:
+ !meta.internalOptions.zeroDataRetention &&
+ meta.internalOptions.saveScrapeResultToGCS,
+ zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ };
+
+ let response = await performFireEngineScrape(
+ meta,
+ meta.logger.child({
+ method: "scrapeURLWithFireEnginePlaywright/callFireEngine",
+ request,
+ }),
+ request,
+ meta.mock,
+ meta.abort.asSignal(),
+ );
+
+ if (!response.url) {
+ meta.logger.warn("Fire-engine did not return the response's URL", {
+ response,
+ sourceURL: meta.url,
+ });
+ }
- return {
- url: response.url ?? meta.url,
+ return {
+ url: response.url ?? meta.url,
- html: response.content,
- error: response.pageError,
- statusCode: response.pageStatusCode,
+ html: response.content,
+ error: response.pageError,
+ statusCode: response.pageStatusCode,
- contentType:
- (Object.entries(response.responseHeaders ?? {}).find(
- x => x[0].toLowerCase() === "content-type",
- ) ?? [])[1] ?? undefined,
+ contentType:
+ (Object.entries(response.responseHeaders ?? {}).find(
+ x => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined,
- ...(response.screenshots !== undefined && response.screenshots.length > 0
- ? {
- screenshot: response.screenshots[0],
- }
- : {}),
+ ...(response.screenshots !== undefined && response.screenshots.length > 0
+ ? {
+ screenshot: response.screenshots[0],
+ }
+ : {}),
- proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- };
+ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
+ };
+ });
}
export async function scrapeURLWithFireEngineTLSClient(
meta: Meta,
): Promise<EngineScrapeResult> {
- const request: FireEngineScrapeRequestCommon &
- FireEngineScrapeRequestTLSClient = {
- url: meta.rewrittenUrl ?? meta.url,
- engine: "tlsclient",
- instantReturn: false,
-
- headers: meta.options.headers,
- priority: meta.internalOptions.priority,
-
- atsv: meta.internalOptions.atsv,
- geolocation: meta.options.location,
- disableJsDom: meta.internalOptions.v0DisableJsDom,
- mobileProxy: meta.featureFlags.has("stealthProxy"),
-
- timeout: meta.abort.scrapeTimeout() ?? 300000,
- saveScrapeResultToGCS:
- !meta.internalOptions.zeroDataRetention &&
- meta.internalOptions.saveScrapeResultToGCS,
- zeroDataRetention: meta.internalOptions.zeroDataRetention,
- };
-
- let response = await performFireEngineScrape(
- meta,
- meta.logger.child({
- method: "scrapeURLWithFireEngineTLSClient/callFireEngine",
- request,
- }),
- request,
- meta.mock,
- meta.abort.asSignal(),
- );
-
- if (!response.url) {
- meta.logger.warn("Fire-engine did not return the response's URL", {
- response,
- sourceURL: meta.url,
+ return withSpan("engine.fire-engine.tlsclient", async span => {
+ setSpanAttributes(span, {
+ "engine.type": "fire-engine-tlsclient",
+ "engine.url": meta.url,
+ "engine.team_id": meta.internalOptions.teamId,
});
- }
+ const request: FireEngineScrapeRequestCommon &
+ FireEngineScrapeRequestTLSClient = {
+ url: meta.rewrittenUrl ?? meta.url,
+ engine: "tlsclient",
+ instantReturn: false,
+
+ headers: meta.options.headers,
+ priority: meta.internalOptions.priority,
+
+ atsv: meta.internalOptions.atsv,
+ geolocation: meta.options.location,
+ disableJsDom: meta.internalOptions.v0DisableJsDom,
+ mobileProxy: meta.featureFlags.has("stealthProxy"),
+
+ timeout: meta.abort.scrapeTimeout() ?? 300000,
+ saveScrapeResultToGCS:
+ !meta.internalOptions.zeroDataRetention &&
+ meta.internalOptions.saveScrapeResultToGCS,
+ zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ };
+
+ let response = await performFireEngineScrape(
+ meta,
+ meta.logger.child({
+ method: "scrapeURLWithFireEngineTLSClient/callFireEngine",
+ request,
+ }),
+ request,
+ meta.mock,
+ meta.abort.asSignal(),
+ );
- return {
- url: response.url ?? meta.url,
+ if (!response.url) {
+ meta.logger.warn("Fire-engine did not return the response's URL", {
+ response,
+ sourceURL: meta.url,
+ });
+ }
- html: response.content,
- error: response.pageError,
- statusCode: response.pageStatusCode,
+ return {
+ url: response.url ?? meta.url,
- contentType:
- (Object.entries(response.responseHeaders ?? {}).find(
- x => x[0].toLowerCase() === "content-type",
- ) ?? [])[1] ?? undefined,
+ html: response.content,
+ error: response.pageError,
+ statusCode: response.pageStatusCode,
+
+ contentType:
+ (Object.entries(response.responseHeaders ?? {}).find(
+ x => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined,
- proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
- };
+ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
+ };
+ });
}
export function fireEngineMaxReasonableTime(
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index e733c1d74..7326d038a 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1,5 +1,6 @@
import { Logger } from "winston";
import * as Sentry from "@sentry/node";
+import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
import {
type Document,
@@ -383,321 +384,366 @@ class WrappedEngineError extends Error {
}
async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
- meta.logger.info(
- `Scraping URL ${JSON.stringify(meta.rewrittenUrl ?? meta.url)}...`,
- );
-
- if (meta.internalOptions.zeroDataRetention) {
- if (meta.featureFlags.has("screenshot")) {
- throw new ZDRViolationError("screenshot");
- }
-
- if (meta.featureFlags.has("screenshot@fullScreen")) {
- throw new ZDRViolationError("screenshot@fullScreen");
- }
+ return withSpan("scrape.engine_loop", async span => {
+ meta.logger.info(
+ `Scraping URL ${JSON.stringify(meta.rewrittenUrl ?? meta.url)}...`,
+ );
- if (
- meta.options.actions &&
- meta.options.actions.find(x => x.type === "screenshot")
- ) {
- throw new ZDRViolationError("screenshot action");
- }
+ setSpanAttributes(span, {
+ "engine.url": meta.rewrittenUrl ?? meta.url,
+ "engine.features": Array.from(meta.featureFlags).join(","),
+ });
- if (
- meta.options.actions &&
- meta.options.actions.find(x => x.type === "pdf")
- ) {
- throw new ZDRViolationError("pdf action");
- }
- }
+ if (meta.internalOptions.zeroDataRetention) {
+ if (meta.featureFlags.has("screenshot")) {
+ throw new ZDRViolationError("screenshot");
+ }
- // TODO: handle sitemap data, see WebScraper/index.ts:280
- // TODO: ScrapeEvents
+ if (meta.featureFlags.has("screenshot@fullScreen")) {
+ throw new ZDRViolationError("screenshot@fullScreen");
+ }
- const fallbackList = buildFallbackList(meta);
+ if (
+ meta.options.actions &&
+ meta.options.actions.find(x => x.type === "screenshot")
+ ) {
+ throw new ZDRViolationError("screenshot action");
+ }
- const snipeAbortController = new AbortController();
- const snipeAbort: AbortInstance = {
- signal: snipeAbortController.signal,
- tier: "engine",
- throwable() {
- return new EngineSnipedError();
- },
- };
+ if (
+ meta.options.actions &&
+ meta.options.actions.find(x => x.type === "pdf")
+ ) {
+ throw new ZDRViolationError("pdf action");
+ }
+ }
- type EngineBundlePromise = {
- engine: Engine;
- unsupportedFeatures: Set<FeatureFlag>;
- promise: Promise<EngineScrapeResultWithContext>;
- };
+ // TODO: handle sitemap data, see WebScraper/index.ts:280
+ // TODO: ScrapeEvents
- const remainingEngines = [...fallbackList];
- let enginePromises: EngineBundlePromise[] = [];
+ const fallbackList = buildFallbackList(meta);
- meta.abort.throwIfAborted();
+ setSpanAttributes(span, {
+ "engine.fallback_list": fallbackList.map(f => f.engine).join(","),
+ });
- let result: EngineScrapeResultWithContext | null = null;
+ const snipeAbortController = new AbortController();
+ const snipeAbort: AbortInstance = {
+ signal: snipeAbortController.signal,
+ tier: "engine",
+ throwable() {
+ return new EngineSnipedError();
+ },
+ };
- while (remainingEngines.length > 0) {
- const { engine, unsupportedFeatures } = remainingEngines.shift()!;
+ type EngineBundlePromise = {
+ engine: Engine;
+ unsupportedFeatures: Set<FeatureFlag>;
+ promise: Promise<EngineScrapeResultWithContext>;
+ };
- const waitUntilWaterfall =
- getEngineMaxReasonableTime(meta, engine) +
- parseInt(process.env.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS || "0", 10);
+ const remainingEngines = [...fallbackList];
+ let enginePromises: EngineBundlePromise[] = [];
+ const enginesAttempted: string[] = [];
+
+ meta.abort.throwIfAborted();
+
+ let result: EngineScrapeResultWithContext | null = null;
+
+ while (remainingEngines.length > 0) {
+ const { engine, unsupportedFeatures } = remainingEngines.shift()!;
+ enginesAttempted.push(engine);
+
+ const waitUntilWaterfall =
+ getEngineMaxReasonableTime(meta, engine) +
+ parseInt(process.env.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS || "0", 10);
+
+ if (
+ !isFinite(waitUntilWaterfall) ||
+ isNaN(waitUntilWaterfall) ||
+ waitUntilWaterfall <= 0
+ ) {
+ meta.logger.warn("Invalid waitUntilWaterfall value", {
+ waitUntilWaterfall,
+ timeout: meta.options.timeout,
+ actions: !!meta.options.actions,
+ hasJson: !!meta.options.formats?.find(x => x.type === "json"),
+ remainingEngines: remainingEngines.length,
+ });
+ }
- if (
- !isFinite(waitUntilWaterfall) ||
- isNaN(waitUntilWaterfall) ||
- waitUntilWaterfall <= 0
- ) {
- meta.logger.warn("Invalid waitUntilWaterfall value", {
+ meta.logger.info("Scraping via " + engine + "...", {
waitUntilWaterfall,
- timeout: meta.options.timeout,
- actions: !!meta.options.actions,
- hasJson: !!meta.options.formats?.find(x => x.type === "json"),
- remainingEngines: remainingEngines.length,
});
- }
- meta.logger.info("Scraping via " + engine + "...", {
- waitUntilWaterfall,
- });
+ enginePromises.push({
+ engine,
+ unsupportedFeatures,
+ promise: (async () => {
+ try {
+ return {
+ engine,
+ unsupportedFeatures,
+ result: await scrapeURLLoopIter(meta, engine, snipeAbort),
+ };
+ } catch (error) {
+ throw new WrappedEngineError(engine, error);
+ }
+ })(),
+ });
- enginePromises.push({
- engine,
- unsupportedFeatures,
- promise: (async () => {
+ while (true) {
try {
- return {
- engine,
- unsupportedFeatures,
- result: await scrapeURLLoopIter(meta, engine, snipeAbort),
- };
+ result = await Promise.race([
+ ...enginePromises.map(x => x.promise),
+ ...(remainingEngines.length > 0
+ ? [
+ new Promise<EngineScrapeResultWithContext>((_, reject) => {
+ setTimeout(() => {
+ reject(new WaterfallNextEngineSignal());
+ }, waitUntilWaterfall);
+ }),
+ ]
+ : []),
+ new Promise<EngineScrapeResultWithContext>((_, reject) => {
+ setTimeout(() => {
+ try {
+ meta.abort.throwIfAborted();
+
+ // Fallback error if above doesn't throw
+ const usingDefaultTimeout =
+ meta.abort.scrapeTimeout() === undefined;
+ throw new ScrapeJobTimeoutError(
+ usingDefaultTimeout
+ ? "Scrape timed out due to maximum length of 5 minutes"
+ : "Scrape timed out",
+ );
+ } catch (error) {
+ reject(error);
+ }
+ }, meta.abort.scrapeTimeout() ?? 300000);
+ }),
+ ]);
+ break;
} catch (error) {
- throw new WrappedEngineError(engine, error);
- }
- })(),
- });
-
- while (true) {
- try {
- result = await Promise.race([
- ...enginePromises.map(x => x.promise),
- ...(remainingEngines.length > 0
- ? [
- new Promise<EngineScrapeResultWithContext>((_, reject) => {
- setTimeout(() => {
- reject(new WaterfallNextEngineSignal());
- }, waitUntilWaterfall);
- }),
- ]
- : []),
- new Promise<EngineScrapeResultWithContext>((_, reject) => {
- setTimeout(() => {
- try {
- meta.abort.throwIfAborted();
-
- // Fallback error if above doesn't throw
- const usingDefaultTimeout =
- meta.abort.scrapeTimeout() === undefined;
- throw new ScrapeJobTimeoutError(
- usingDefaultTimeout
- ? "Scrape timed out due to maximum length of 5 minutes"
- : "Scrape timed out",
- );
- } catch (error) {
- reject(error);
- }
- }, meta.abort.scrapeTimeout() ?? 300000);
- }),
- ]);
- break;
- } catch (error) {
- if (error instanceof WrappedEngineError) {
- if (error.error instanceof EngineError) {
- meta.logger.warn(
- "Engine " + error.engine + " could not scrape the page.",
- {
+ if (error instanceof WrappedEngineError) {
+ if (error.error instanceof EngineError) {
+ meta.logger.warn(
+ "Engine " + error.engine + " could not scrape the page.",
+ {
+ error: error.error,
+ },
+ );
+ } else if (error.error instanceof IndexMissError) {
+ meta.logger.warn(
+ "Engine " +
+ error.engine +
+ " could not find the page in the index.",
+ {
+ error: error.error,
+ },
+ );
+ } else if (
+ error.error instanceof AddFeatureError ||
+ error.error instanceof RemoveFeatureError ||
+ error.error instanceof SiteError ||
+ error.error instanceof SSLError ||
+ error.error instanceof DNSResolutionError ||
+ error.error instanceof ActionError ||
+ error.error instanceof UnsupportedFileError ||
+ error.error instanceof PDFAntibotError ||
+ error.error instanceof PDFInsufficientTimeError ||
+ error.error instanceof ProxySelectionError
+ ) {
+ throw error.error;
+ } else if (error.error instanceof LLMRefusalError) {
+ meta.logger.warn("LLM refusal encountered", {
error: error.error,
- },
- );
- } else if (error.error instanceof IndexMissError) {
- meta.logger.warn(
- "Engine " +
- error.engine +
- " could not find the page in the index.",
- {
+ });
+ throw error.error;
+ } else if (error.error instanceof FEPageLoadFailed) {
+ // This is the internal timeout bug on f-e and should be treated as an EngineError.
+ meta.logger.warn("FEPageLoadFailed encountered", {
error: error.error,
- },
- );
- } else if (
- error.error instanceof AddFeatureError ||
- error.error instanceof RemoveFeatureError ||
- error.error instanceof SiteError ||
- error.error instanceof SSLError ||
- error.error instanceof DNSResolutionError ||
- error.error instanceof ActionError ||
- error.error instanceof UnsupportedFileError ||
- error.error instanceof PDFAntibotError ||
- error.error instanceof PDFInsufficientTimeError ||
- error.error instanceof ProxySelectionError
- ) {
- throw error.error;
- } else if (error.error instanceof LLMRefusalError) {
- meta.logger.warn("LLM refusal encountered", { error: error.error });
- throw error.error;
- } else if (error.error instanceof FEPageLoadFailed) {
- // This is the internal timeout bug on f-e and should be treated as an EngineError.
- meta.logger.warn("FEPageLoadFailed encountered", {
- error: error.error,
- });
- } else if (error.error instanceof AbortManagerThrownError) {
- if (error.error.tier === "engine") {
+ });
+ } else if (error.error instanceof AbortManagerThrownError) {
+ if (error.error.tier === "engine") {
+ meta.logger.warn(
+ "Engine " + error.engine + " timed out while scraping.",
+ { error: error.error },
+ );
+ } else {
+ throw error.error;
+ }
+ } else {
meta.logger.warn(
- "Engine " + error.engine + " timed out while scraping.",
- { error: error.error },
+ "An unexpected error happened while scraping with " +
+ error.engine +
+ ".",
+ { error },
);
- } else {
- throw error.error;
}
- } else {
- meta.logger.warn(
- "An unexpected error happened while scraping with " +
- error.engine +
- ".",
- { error },
+
+ // Filter out the failed engine
+ enginePromises = enginePromises.filter(
+ x => x.engine !== error.engine,
);
- }
- // Filter out the failed engine
- enginePromises = enginePromises.filter(
- x => x.engine !== error.engine,
- );
+ // If we don't have any engines waterfalled, let's waterfall the next engine
+ if (enginePromises.length === 0) {
+ break;
+ }
- // If we don't have any engines waterfalled, let's waterfall the next engine
- if (enginePromises.length === 0) {
+ // Otherwise, just keep racing
+ } else if (
+ error instanceof AddFeatureError ||
+ error instanceof RemoveFeatureError
+ ) {
+ throw error;
+ } else if (error instanceof WaterfallNextEngineSignal) {
+ // It's time to waterfall the next engine
break;
- }
+ } else if (error instanceof AbortManagerThrownError) {
+ if (error.tier === "engine") {
+ meta.logger.warn(
+ "Engine-scoped timeout error received here. Weird!",
+ { error },
+ );
+ }
- // Otherwise, just keep racing
- } else if (
- error instanceof AddFeatureError ||
- error instanceof RemoveFeatureError
- ) {
- throw error;
- } else if (error instanceof WaterfallNextEngineSignal) {
- // It's time to waterfall the next engine
- break;
- } else if (error instanceof AbortManagerThrownError) {
- if (error.tier === "engine") {
- meta.logger.warn(
- "Engine-scoped timeout error received here. Weird!",
- { error },
- );
+ throw error;
+ } else {
+ meta.logger.warn("Unexpected error while racing engines", {
+ error,
+ });
+ throw error;
}
-
- throw error;
- } else {
- meta.logger.warn("Unexpected error while racing engines", { error });
- throw error;
}
}
+
+ if (result === null) {
+ meta.logger.info("Waterfalling to next engine...", {
+ waitUntilWaterfall,
+ });
+ } else {
+ break;
+ }
}
+ snipeAbortController.abort();
+
if (result === null) {
- meta.logger.info("Waterfalling to next engine...", {
- waitUntilWaterfall,
+ setSpanAttributes(span, {
+ "engine.no_engines_left": true,
+ "engine.engines_attempted": enginesAttempted.join(","),
});
- } else {
- break;
+ throw new NoEnginesLeftError(fallbackList.map(x => x.engine));
}
- }
-
- snipeAbortController.abort();
- if (result === null) {
- throw new NoEnginesLeftError(fallbackList.map(x => x.engine));
- }
+ // Set winner engine attributes
+ setSpanAttributes(span, {
+ "engine.winner": result.engine,
+ "engine.engines_attempted": enginesAttempted.join(","),
+ "engine.unsupported_features":
+ result.unsupportedFeatures.size > 0
+ ? Array.from(result.unsupportedFeatures).join(",")
+ : undefined,
+ });
- let engineResult: EngineScrapeResult = result.result;
-
- for (const postprocessor of postprocessors) {
- if (
- postprocessor.shouldRun(
- meta,
- new URL(engineResult.url),
- engineResult.postprocessorsUsed,
- )
- ) {
- meta.logger.info("Running postprocessor " + postprocessor.name);
- try {
- engineResult = await postprocessor.run(
- {
- ...meta,
- logger: meta.logger.child({
- method: "postprocessors/" + postprocessor.name,
- }),
- },
- engineResult,
- );
- } catch (error) {
- meta.logger.warn("Failed to run postprocessor " + postprocessor.name, {
- error,
- });
+ meta.winnerEngine = result.engine;
+ let engineResult: EngineScrapeResult = result.result;
+
+ for (const postprocessor of postprocessors) {
+ if (
+ postprocessor.shouldRun(
+ meta,
+ new URL(engineResult.url),
+ engineResult.postprocessorsUsed,
+ )
+ ) {
+ meta.logger.info("Running postprocessor " + postprocessor.name);
+ try {
+ engineResult = await postprocessor.run(
+ {
+ ...meta,
+ logger: meta.logger.child({
+ method: "postprocessors/" + postprocessor.name,
+ }),
+ },
+ engineResult,
+ );
+ } catch (error) {
+ meta.logger.warn(
+ "Failed to run postprocessor " + postprocessor.name,
+ {
+ error,
+ },
+ );
+ }
}
}
- }
- let document: Document = {
- markdown: engineResult.markdown,
- rawHtml: engineResult.html,
- screenshot: engineResult.screenshot,
- actions: engineResult.actions,
- metadata: {
- sourceURL: meta.internalOptions.unnormalizedSourceURL ?? meta.url,
- url: engineResult.url,
- statusCode: engineResult.statusCode,
- error: engineResult.error,
- numPages: engineResult.pdfMetadata?.numPages,
- ...(engineResult.pdfMetadata?.title
- ? { title: engineResult.pdfMetadata.title }
- : {}),
- contentType: engineResult.contentType,
- proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",
- ...(fallbackList.find(x =>
- ["index", "index;documents"].includes(x.engine),
- )
- ? engineResult.cacheInfo
- ? {
- cacheState: "hit",
- cachedAt: engineResult.cacheInfo.created_at.toISOString(),
- }
- : {
- cacheState: "miss",
- }
- : {}),
- postprocessorsUsed: engineResult.postprocessorsUsed,
- },
- };
+ let document: Document = {
+ markdown: engineResult.markdown,
+ rawHtml: engineResult.html,
+ screenshot: engineResult.screenshot,
+ actions: engineResult.actions,
+ metadata: {
+ sourceURL: meta.internalOptions.unnormalizedSourceURL ?? meta.url,
+ url: engineResult.url,
+ statusCode: engineResult.statusCode,
+ error: engineResult.error,
+ numPages: engineResult.pdfMetadata?.numPages,
+ ...(engineResult.pdfMetadata?.title
+ ? { title: engineResult.pdfMetadata.title }
+ : {}),
+ contentType: engineResult.contentType,
+ proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",
+ ...(fallbackList.find(x =>
+ ["index", "index;documents"].includes(x.engine),
+ )
+ ? engineResult.cacheInfo
+ ? {
+ cacheState: "hit",
+ cachedAt: engineResult.cacheInfo.created_at.toISOString(),
+ }
+ : {
+ cacheState: "miss",
+ }
+ : {}),
+ postprocessorsUsed: engineResult.postprocessorsUsed,
+ },
+ };
- if (result.unsupportedFeatures.size > 0) {
- const warning = `The engine used does not support the following features: ${[...result.unsupportedFeatures].join(", ")} -- your scrape may be partial.`;
- meta.logger.warn(warning, {
- engine: result.engine,
- unsupportedFeatures: result.unsupportedFeatures,
- });
- document.warning =
- document.warning !== undefined
- ? document.warning + " " + warning
- : warning;
- }
+ if (result.unsupportedFeatures.size > 0) {
+ const warning = `The engine used does not support the following features: ${[...result.unsupportedFeatures].join(", ")} -- your scrape may be partial.`;
+ meta.logger.warn(warning, {
+ engine: result.engine,
+ unsupportedFeatures: result.unsupportedFeatures,
+ });
+ document.warning =
+ document.warning !== undefined
+ ? document.warning + " " + warning
+ : warning;
+ }
- document = await executeTransformers(meta, document);
+ document = await executeTransformers(meta, document);
- return {
- success: true,
- document,
- };
+ // Set final span attributes
+ setSpanAttributes(span, {
+ "engine.final_status_code": document.metadata.statusCode,
+ "engine.final_url": document.metadata.url,
+ "engine.content_type": document.metadata.contentType,
+ "engine.proxy_used": document.metadata.proxyUsed,
+ "engine.cache_state": document.metadata.cacheState,
+ "engine.postprocessors_used": engineResult.postprocessorsUsed?.join(","),
+ });
+
+ return {
+ success: true,
+ document,
+ };
+ });
}
export async function scrapeURL(
@@ -707,242 +753,321 @@ export async function scrapeURL(
internalOptions: InternalOptions,
costTracking: CostTracking,
): Promise<ScrapeUrlResponse> {
- const meta = await buildMetaObject(
- id,
- url,
- options,
- internalOptions,
- costTracking,
- );
+ return withSpan("scrape.pipeline", async span => {
+ const meta = await buildMetaObject(
+ id,
+ url,
+ options,
+ internalOptions,
+ costTracking,
+ );
- const startTime = Date.now();
+ const startTime = Date.now();
+
+ // Set initial span attributes
+ setSpanAttributes(span, {
+ "scrape.id": id,
+ "scrape.url": url,
+ "scrape.team_id": internalOptions.teamId,
+ "scrape.crawl_id": internalOptions.crawlId,
+ "scrape.zero_data_retention": internalOptions.zeroDataRetention,
+ "scrape.force_engine": Array.isArray(internalOptions.forceEngine)
+ ? internalOptions.forceEngine.join(",")
+ : internalOptions.forceEngine,
+ "scrape.features": Array.from(meta.featureFlags).join(","),
+ });
- meta.logger.info("scrapeURL entered");
+ meta.logger.info("scrapeURL entered");
- if (meta.rewrittenUrl) {
- meta.logger.info("Rewriting URL");
- }
+ if (meta.rewrittenUrl) {
+ meta.logger.info("Rewriting URL");
+ setSpanAttributes(span, {
+ "scrape.rewritten_url": meta.rewrittenUrl,
+ });
+ }
- if (internalOptions.teamFlags?.checkRobotsOnScrape) {
- const urlToCheck = meta.rewrittenUrl || meta.url;
- meta.logger.info("Checking robots.txt", { url: urlToCheck });
-
- const urlObj = new URL(urlToCheck);
- const isRobotsTxtPath = urlObj.pathname === "/robots.txt";
-
- if (!isRobotsTxtPath) {
- try {
- const { content: robotsTxt } = await fetchRobotsTxt(
- {
- url: urlToCheck,
- zeroDataRetention: internalOptions.zeroDataRetention || false,
- location: options.location,
- },
- id,
- meta.logger,
- meta.abort.asSignal(),
- );
+ if (internalOptions.teamFlags?.checkRobotsOnScrape) {
+ await withSpan("scrape.robots_check", async robotsSpan => {
+ const urlToCheck = meta.rewrittenUrl || meta.url;
+ meta.logger.info("Checking robots.txt", { url: urlToCheck });
+
+ const urlObj = new URL(urlToCheck);
+ const isRobotsTxtPath = urlObj.pathname === "/robots.txt";
- const checker = createRobotsChecker(urlToCheck, robotsTxt);
- const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);
+ setSpanAttributes(robotsSpan, {
+ "robots.url": urlToCheck,
+ "robots.is_robots_txt_path": isRobotsTxtPath,
+ });
- if (!isAllowed) {
- meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });
+ if (!isRobotsTxtPath) {
+ try {
+ const { content: robotsTxt } = await fetchRobotsTxt(
+ {
+ url: urlToCheck,
+ zeroDataRetention: internalOptions.zeroDataRetention || false,
+ location: options.location,
+ },
+ id,
+ meta.logger,
+ meta.abort.asSignal(),
+ );
+
+ const checker = createRobotsChecker(urlToCheck, robotsTxt);
+ const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);
+
+ setSpanAttributes(robotsSpan, {
+ "robots.allowed": isAllowed,
+ });
+
+ if (!isAllowed) {
+ meta.logger.info("URL blocked by robots.txt", {
+ url: urlToCheck,
+ });
+ setSpanAttributes(span, {
+ "scrape.blocked_by_robots": true,
+ });
+ throw new Error("URL blocked by robots.txt");
+ }
+ } catch (error) {
+ if (error.message === "URL blocked by robots.txt") {
+ throw error;
+ }
+ meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {
+ error,
+ url: urlToCheck,
+ });
+ setSpanAttributes(robotsSpan, {
+ "robots.fetch_failed": true,
+ });
+ }
+ }
+ }).catch(error => {
+ if (error.message === "URL blocked by robots.txt") {
return {
success: false,
- error: new Error("URL blocked by robots.txt"),
+ error,
};
}
- } catch (error) {
- meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {
- error,
- url: urlToCheck,
- });
- }
+ throw error;
+ });
}
- }
- meta.logger.info("Pre-recording frequency");
-
- const shouldRecordFrequency =
- useIndex &&
- meta.options.storeInCache &&
- !meta.internalOptions.zeroDataRetention &&
- internalOptions.teamId !== process.env.PRECRAWL_TEAM_ID;
- if (shouldRecordFrequency) {
- (async () => {
- try {
- meta.logger.info("Recording frequency");
- const normalizedURL = normalizeURLForIndex(meta.url);
- const urlHash = hashURL(normalizedURL);
-
- let { data, error } = await index_supabase_service
- .from("index")
- .select("id, created_at, status")
- .eq("url_hash", urlHash)
- .order("created_at", { ascending: false })
- .limit(1);
-
- if (error) {
- meta.logger.warn("Failed to get age data", { error });
- }
+ meta.logger.info("Pre-recording frequency");
- const age = data?.[0]
- ? Date.now() - new Date(data[0].created_at).getTime()
- : -1;
+ const shouldRecordFrequency =
+ useIndex &&
+ meta.options.storeInCache &&
+ !meta.internalOptions.zeroDataRetention &&
+ internalOptions.teamId !== process.env.PRECRAWL_TEAM_ID;
+ if (shouldRecordFrequency) {
+ (async () => {
+ try {
+ meta.logger.info("Recording frequency");
+ const normalizedURL = normalizeURLForIndex(meta.url);
+ const urlHash = hashURL(normalizedURL);
+
+ let { data, error } = await index_supabase_service
+ .from("index")
+ .select("id, created_at, status")
+ .eq("url_hash", urlHash)
+ .order("created_at", { ascending: false })
+ .limit(1);
+
+ if (error) {
+ meta.logger.warn("Failed to get age data", { error });
+ }
- const fakeDomain = meta.options.__experimental_omceDomain;
- const domainSplits = generateDomainSplits(
- new URL(normalizeURLForIndex(meta.url)).hostname,
- fakeDomain,
- );
- const domainHash = hashURL(domainSplits.slice(-1)[0]);
-
- const out = {
- domain_hash: domainHash,
- url: meta.url,
- age2: age,
- };
-
- await addIndexRFInsertJob(out);
- meta.logger.info("Recorded frequency", { out });
- } catch (error) {
- meta.logger.warn("Failed to record frequency", { error });
- }
- })();
- } else {
- meta.logger.info("Not recording frequency", {
- useIndex,
- storeInCache: meta.options.storeInCache,
- zeroDataRetention: meta.internalOptions.zeroDataRetention,
- });
- }
+ const age = data?.[0]
+ ? Date.now() - new Date(data[0].created_at).getTime()
+ : -1;
- try {
- let result: ScrapeUrlResponse;
- while (true) {
- try {
- result = await scrapeURLLoop(meta);
- break;
- } catch (error) {
- if (
- error instanceof AddFeatureError &&
- (meta.internalOptions.forceEngine === undefined ||
- Array.isArray(meta.internalOptions.forceEngine))
- ) {
- meta.logger.debug(
- "More feature flags requested by scraper: adding " +
- error.featureFlags.join(", "),
- { error, existingFlags: meta.featureFlags },
+ const fakeDomain = meta.options.__experimental_omceDomain;
+ const domainSplits = generateDomainSplits(
+ new URL(normalizeURLForIndex(meta.url)).hostname,
+ fakeDomain,
);
- meta.featureFlags = new Set(
- [...meta.featureFlags].concat(error.featureFlags),
- );
- if (error.pdfPrefetch) {
- meta.pdfPrefetch = error.pdfPrefetch;
- }
- } else if (
- error instanceof RemoveFeatureError &&
- (meta.internalOptions.forceEngine === undefined ||
- Array.isArray(meta.internalOptions.forceEngine))
- ) {
- meta.logger.debug(
- "Incorrect feature flags reported by scraper: removing " +
- error.featureFlags.join(","),
- { error, existingFlags: meta.featureFlags },
- );
- meta.featureFlags = new Set(
- [...meta.featureFlags].filter(x => !error.featureFlags.includes(x)),
- );
- } else if (
- error instanceof PDFAntibotError &&
- meta.internalOptions.forceEngine === undefined
- ) {
- if (meta.pdfPrefetch !== undefined) {
- meta.logger.error(
- "PDF was prefetched and still blocked by antibot, failing",
+ const domainHash = hashURL(domainSplits.slice(-1)[0]);
+
+ const out = {
+ domain_hash: domainHash,
+ url: meta.url,
+ age2: age,
+ };
+
+ await addIndexRFInsertJob(out);
+ meta.logger.info("Recorded frequency", { out });
+ } catch (error) {
+ meta.logger.warn("Failed to record frequency", { error });
+ }
+ })();
+ } else {
+ meta.logger.info("Not recording frequency", {
+ useIndex,
+ storeInCache: meta.options.storeInCache,
+ zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ });
+ }
+
+ try {
+ let result: ScrapeUrlResponse;
+ while (true) {
+ try {
+ result = await scrapeURLLoop(meta);
+ break;
+ } catch (error) {
+ if (
+ error instanceof AddFeatureError &&
+ (meta.internalOptions.forceEngine === undefined ||
+ Array.isArray(meta.internalOptions.forceEngine))
+ ) {
+ meta.logger.debug(
+ "More feature flags requested by scraper: adding " +
+ error.featureFlags.join(", "),
+ { error, existingFlags: meta.featureFlags },
);
- throw error;
- } else {
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].concat(error.featureFlags),
+ );
+ if (error.pdfPrefetch) {
+ meta.pdfPrefetch = error.pdfPrefetch;
+ }
+ } else if (
+ error instanceof RemoveFeatureError &&
+ (meta.internalOptions.forceEngine === undefined ||
+ Array.isArray(meta.internalOptions.forceEngine))
+ ) {
meta.logger.debug(
- "PDF was blocked by anti-bot, prefetching with chrome-cdp",
+ "Incorrect feature flags reported by scraper: removing " +
+ error.featureFlags.join(","),
+ { error, existingFlags: meta.featureFlags },
);
meta.featureFlags = new Set(
- [...meta.featureFlags].filter(x => x !== "pdf"),
+ [...meta.featureFlags].filter(
+ x => !error.featureFlags.includes(x),
+ ),
);
+ } else if (
+ error instanceof PDFAntibotError &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ if (meta.pdfPrefetch !== undefined) {
+ meta.logger.error(
+ "PDF was prefetched and still blocked by antibot, failing",
+ );
+ throw error;
+ } else {
+ meta.logger.debug(
+ "PDF was blocked by anti-bot, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "pdf"),
+ );
+ }
+ } else {
+ throw error;
}
- } else {
- throw error;
}
}
- }
- meta.logger.debug("scrapeURL metrics", {
- module: "scrapeURL/metrics",
- timeTaken: Date.now() - startTime,
- maxAgeValid: (meta.options.maxAge ?? 0) > 0,
- shouldUseIndex: shouldUseIndex(meta),
- success: result.success,
- indexHit: result.success && result.document.metadata.cacheState === "hit",
- });
+ meta.logger.debug("scrapeURL metrics", {
+ module: "scrapeURL/metrics",
+ timeTaken: Date.now() - startTime,
+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,
+ shouldUseIndex: shouldUseIndex(meta),
+ success: result.success,
+ indexHit:
+ result.success && result.document.metadata.cacheState === "hit",
+ });
- return result;
- } catch (error) {
- // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {
- // throw new FEPageLoadFailed();
- // } else
- meta.logger.debug("scrapeURL metrics", {
- module: "scrapeURL/metrics",
- timeTaken: Date.now() - startTime,
- maxAgeValid: (meta.options.maxAge ?? 0) > 0,
- shouldUseIndex: shouldUseIndex(meta),
- success: false,
- indexHit: false,
- });
+ setSpanAttributes(span, {
+ "scrape.success": true,
+ "scrape.duration_ms": Date.now() - startTime,
+ "scrape.index_hit":
+ result.success && result.document.metadata.cacheState === "hit",
+ });
- if (error instanceof NoEnginesLeftError) {
- meta.logger.warn("scrapeURL: All scraping engines failed!", { error });
- } else if (error instanceof LLMRefusalError) {
- meta.logger.warn("scrapeURL: LLM refused to extract content", { error });
- } else if (
- error instanceof Error &&
- error.message.includes("Invalid schema for response_format")
- ) {
- // TODO: seperate into custom error
- meta.logger.warn("scrapeURL: LLM schema error", { error });
- // TODO: results?
- } else if (error instanceof SiteError) {
- meta.logger.warn("scrapeURL: Site failed to load in browser", { error });
- } else if (error instanceof SSLError) {
- meta.logger.warn("scrapeURL: SSL error", { error });
- } else if (error instanceof ActionError) {
- meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });
- } else if (error instanceof UnsupportedFileError) {
- meta.logger.warn("scrapeURL: Tried to scrape unsupported file", {
- error,
+ return result;
+ } catch (error) {
+ // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {
+ // throw new FEPageLoadFailed();
+ // } else
+ meta.logger.debug("scrapeURL metrics", {
+ module: "scrapeURL/metrics",
+ timeTaken: Date.now() - startTime,
+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,
+ shouldUseIndex: shouldUseIndex(meta),
+ success: false,
+ indexHit: false,
});
- } else if (error instanceof PDFInsufficientTimeError) {
- meta.logger.warn("scrapeURL: Insufficient time to process PDF", {
- error,
+
+ // Set error attributes on span
+ let errorType = "unknown";
+ if (error instanceof NoEnginesLeftError) {
+ errorType = "NoEnginesLeftError";
+ meta.logger.warn("scrapeURL: All scraping engines failed!", { error });
+ } else if (error instanceof LLMRefusalError) {
+ errorType = "LLMRefusalError";
+ meta.logger.warn("scrapeURL: LLM refused to extract content", {
+ error,
+ });
+ } else if (
+ error instanceof Error &&
+ error.message.includes("Invalid schema for response_format")
+ ) {
+ errorType = "LLMSchemaError";
+ // TODO: seperate into custom error
+ meta.logger.warn("scrapeURL: LLM schema error", { error });
+ // TODO: results?
+ } else if (error instanceof SiteError) {
+ errorType = "SiteError";
+ meta.logger.warn("scrapeURL: Site failed to load in browser", {
+ error,
+ });
+ } else if (error instanceof SSLError) {
+ errorType = "SSLError";
+ meta.logger.warn("scrapeURL: SSL error", { error });
+ } else if (error instanceof ActionError) {
+ errorType = "ActionError";
+ meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });
+ } else if (error instanceof UnsupportedFileError) {
+ errorType = "UnsupportedFileError";
+ meta.logger.warn("scrapeURL: Tried to scrape unsupported file", {
+ error,
+ });
+ } else if (error instanceof PDFInsufficientTimeError) {
+ errorType = "PDFInsufficientTimeError";
+ meta.logger.warn("scrapeURL: Insufficient time to process PDF", {
+ error,
+ });
+ } else if (error instanceof PDFPrefetchFailed) {
+ errorType = "PDFPrefetchFailed";
+ meta.logger.warn(
+ "scrapeURL: Failed to prefetch PDF that is protected by anti-bot",
+ { error },
+ );
+ } else if (error instanceof ProxySelectionError) {
+ errorType = "ProxySelectionError";
+ meta.logger.warn("scrapeURL: Proxy selection error", { error });
+ } else if (error instanceof AbortManagerThrownError) {
+ errorType = "AbortManagerThrownError";
+ throw error.inner;
+ } else {
+ Sentry.captureException(error);
+ meta.logger.error("scrapeURL: Unexpected error happened", { error });
+ // TODO: results?
+ }
+
+ setSpanAttributes(span, {
+ "scrape.success": false,
+ "scrape.error": error instanceof Error ? error.message : String(error),
+ "scrape.error_type": errorType,
+ "scrape.duration_ms": Date.now() - startTime,
});
- } else if (error instanceof PDFPrefetchFailed) {
- meta.logger.warn(
- "scrapeURL: Failed to prefetch PDF that is protected by anti-bot",
- { error },
- );
- } else if (error instanceof ProxySelectionError) {
- meta.logger.warn("scrapeURL: Proxy selection error", { error });
- } else if (error instanceof AbortManagerThrownError) {
- throw error.inner;
- } else {
- Sentry.captureException(error);
- meta.logger.error("scrapeURL: Unexpected error happened", { error });
- // TODO: results?
- }
- return {
- success: false,
- error,
- };
- }
+ return {
+ success: false,
+ error,
+ };
+ }
+ });
}
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index da0bda268..194252f70 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -21,6 +21,7 @@ import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";
import { deserializeTransportableError } from "../lib/error-serde";
import { abTestJob } from "./ab-test";
import { NuQJob, scrapeQueue } from "./worker/nuq";
+import { serializeTraceContext } from "../lib/otel-tracer";
/**
* Checks if a job is a crawl or batch scrape based on its options
@@ -189,8 +190,15 @@ export async function addScrapeJob(
priority: number = 0,
directToBullMQ: boolean = false,
): Promise<NuQJob<ScrapeJobData> | null> {
+ // Capture trace context to propagate to worker
+ const traceContext = serializeTraceContext();
+ const optionsWithTrace: ScrapeJobData = {
+ ...webScraperOptions,
+ traceContext,
+ };
+
return await addScrapeJobRaw(
- webScraperOptions,
+ optionsWithTrace,
jobId,
priority,
directToBullMQ,
@@ -206,6 +214,9 @@ export async function addScrapeJobs(
) {
if (jobs.length === 0) return true;
+ // Capture trace context for all jobs
+ const traceContext = serializeTraceContext();
+
const jobsByTeam = new Map<
string,
{
@@ -349,7 +360,7 @@ export async function addScrapeJobs(
addToCQ.map(async job => {
const size = JSON.stringify(job.data).length;
await _addScrapeJobToConcurrencyQueue(
- job.data,
+ { ...job.data, traceContext },
job.jobId,
job.priority,
);
@@ -358,7 +369,11 @@ export async function addScrapeJobs(
await Promise.all(
addToBull.map(async job => {
- await _addScrapeJobToBullMQ(job.data, job.jobId, job.priority);
+ await _addScrapeJobToBullMQ(
+ { ...job.data, traceContext },
+ job.jobId,
+ job.priority,
+ );
}),
);
}
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 110b0af77..4365a39e1 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -2,6 +2,7 @@ import { Logger } from "winston";
import { logger } from "../../lib/logger";
import { Client, Pool } from "pg";
import { type ScrapeJobData } from "../../types";
+import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
// === Basics
@@ -145,24 +146,42 @@ class NuQ<JobData = any, JobReturnValue = any> {
id: string,
_logger = logger,
): Promise<NuQJob<JobData, JobReturnValue> | null> {
- const start = Date.now();
- try {
- return this.rowToJob(
- (
- await nuqPool.query(
- `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = $1;`,
- [id],
- )
- ).rows[0],
- );
- } finally {
- _logger.info("nuqGetJob metrics", {
- module: "nuq/metrics",
- method: "nuqGetJob",
- duration: Date.now() - start,
- scrapeId: id,
+ return withSpan("nuq.getJob", async span => {
+ setSpanAttributes(span, {
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
});
- }
+
+ const start = Date.now();
+ try {
+ const result = this.rowToJob(
+ (
+ await nuqPool.query(
+ `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = $1;`,
+ [id],
+ )
+ ).rows[0],
+ );
+
+ setSpanAttributes(span, {
+ "nuq.job_found": result !== null,
+ "nuq.job_status": result?.status,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ _logger.info("nuqGetJob metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetJob",
+ duration,
+ scrapeId: id,
+ });
+ }
+ });
}
public async getJobs(
@@ -296,25 +315,44 @@ class NuQ<JobData = any, JobReturnValue = any> {
data: JobData,
priority: number = 0,
): Promise<NuQJob<JobData, JobReturnValue>> {
- const start = Date.now();
- try {
- return this.rowToJob(
- (
- await nuqPool.query(
- `INSERT INTO ${this.queueName} (id, data, priority) VALUES ($1, $2, $3) RETURNING ${this.jobReturning.join(", ")};`,
- [id, data, priority],
- )
- ).rows[0],
- )!;
- } finally {
- logger.info("nuqAddJob metrics", {
- module: "nuq/metrics",
- method: "nuqAddJob",
- duration: Date.now() - start,
- scrapeId: id,
- zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
+ return withSpan("nuq.addJob", async span => {
+ setSpanAttributes(span, {
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
+ "nuq.priority": priority,
+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,
});
- }
+
+ const start = Date.now();
+ try {
+ const result = this.rowToJob(
+ (
+ await nuqPool.query(
+ `INSERT INTO ${this.queueName} (id, data, priority) VALUES ($1, $2, $3) RETURNING ${this.jobReturning.join(", ")};`,
+ [id, data, priority],
+ )
+ ).rows[0],
+ )!;
+
+ setSpanAttributes(span, {
+ "nuq.job_created": true,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ logger.info("nuqAddJob metrics", {
+ module: "nuq/metrics",
+ method: "nuqAddJob",
+ duration,
+ scrapeId: id,
+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,
+ });
+ }
+ });
}
private readonly nuqWaitMode =
@@ -327,89 +365,107 @@ class NuQ<JobData = any, JobReturnValue = any> {
timeout: number | null,
_logger: Logger = logger,
): Promise<JobReturnValue> {
- const done = new Promise<JobReturnValue>(
- (async (resolve, reject) => {
- if (this.nuqWaitMode === "listen") {
- let timer: NodeJS.Timeout | null = null;
- if (timeout !== null) {
- timer = setTimeout(
- (() => {
- this.removeListener(id, listener);
- reject(new Error("Timed out"));
- }).bind(this),
- timeout,
- );
- }
+ return withSpan("nuq.waitForJob", async span => {
+ setSpanAttributes(span, {
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
+ "nuq.timeout": timeout ?? undefined,
+ "nuq.wait_mode": this.nuqWaitMode,
+ });
- const listener = async function (_msg: "completed" | "failed") {
- if (timer) clearTimeout(timer);
- const job = await this.getJob(id, _logger);
- if (!job) {
- reject(new Error("Job raced out while waiting for it"));
- } else {
- if (job.status === "completed") {
- resolve(job.returnvalue!);
- } else {
- reject(new Error(job.failedReason!));
- }
+ const startTime = Date.now();
+
+ const done = new Promise<JobReturnValue>(
+ (async (resolve, reject) => {
+ if (this.nuqWaitMode === "listen") {
+ let timer: NodeJS.Timeout | null = null;
+ if (timeout !== null) {
+ timer = setTimeout(
+ (() => {
+ this.removeListener(id, listener);
+ reject(new Error("Timed out"));
+ }).bind(this),
+ timeout,
+ );
}
- }.bind(this);
-
- try {
- await this.addListener(id, listener);
- } catch (e) {
- reject(e);
- }
- try {
- const job = await this.getJob(id, _logger);
- if (job && ["completed", "failed"].includes(job.status)) {
- this.removeListener(id, listener);
+ const listener = async function (_msg: "completed" | "failed") {
if (timer) clearTimeout(timer);
- if (job.status === "completed") {
- resolve(job.returnvalue!);
+ const job = await this.getJob(id, _logger);
+ if (!job) {
+ reject(new Error("Job raced out while waiting for it"));
} else {
- reject(new Error(job.failedReason!));
+ if (job.status === "completed") {
+ resolve(job.returnvalue!);
+ } else {
+ reject(new Error(job.failedReason!));
+ }
}
- return;
+ }.bind(this);
+
+ try {
+ await this.addListener(id, listener);
+ } catch (e) {
+ reject(e);
}
- } catch (e) {
- _logger.warn("nuqGetJob ensure check failed", {
- module: "nuq",
- method: "nuqWaitForJob",
- error: e,
- scrapeId: id,
- });
- }
- } else {
- const timeoutAt = timeout !== null ? Date.now() + timeout : null;
- const poll = async function poll() {
+
try {
const job = await this.getJob(id, _logger);
if (job && ["completed", "failed"].includes(job.status)) {
+ this.removeListener(id, listener);
+ if (timer) clearTimeout(timer);
if (job.status === "completed") {
- return resolve(job.returnvalue!);
+ resolve(job.returnvalue!);
} else {
- return reject(new Error(job.failedReason!));
+ reject(new Error(job.failedReason!));
}
+ return;
}
} catch (e) {
- return reject(e);
+ _logger.warn("nuqGetJob ensure check failed", {
+ module: "nuq",
+ method: "nuqWaitForJob",
+ error: e,
+ scrapeId: id,
+ });
}
+ } else {
+ const timeoutAt = timeout !== null ? Date.now() + timeout : null;
+ const poll = async function poll() {
+ try {
+ const job = await this.getJob(id, _logger);
+ if (job && ["completed", "failed"].includes(job.status)) {
+ if (job.status === "completed") {
+ return resolve(job.returnvalue!);
+ } else {
+ return reject(new Error(job.failedReason!));
+ }
+ }
+ } catch (e) {
+ return reject(e);
+ }
- if (timeoutAt && Date.now() > timeoutAt) {
- return reject(new Error("Timed out"));
- }
+ if (timeoutAt && Date.now() > timeoutAt) {
+ return reject(new Error("Timed out"));
+ }
- setTimeout(poll.bind(this), 500);
- }.bind(this);
+ setTimeout(poll.bind(this), 500);
+ }.bind(this);
- poll();
- }
- }).bind(this),
- );
+ poll();
+ }
+ }).bind(this),
+ );
+
+ const result = await done;
+
+ setSpanAttributes(span, {
+ "nuq.wait_duration_ms": Date.now() - startTime,
+ "nuq.wait_success": true,
+ });
- return done;
+ return result;
+ });
}
// === Consumer
@@ -468,27 +524,43 @@ class NuQ<JobData = any, JobReturnValue = any> {
returnvalue: any | null,
_logger: Logger = logger,
): Promise<boolean> {
- const start = Date.now();
- try {
- return (
- (
- await nuqPool.query(
- `
- WITH updated AS (UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id)
- SELECT pg_notify('${this.queueName}', (id::text || '|completed')) FROM updated;
- `,
- [id, lock, returnvalue],
- )
- ).rowCount !== 0
- );
- } finally {
- _logger.info("nuqJobFinish metrics", {
- module: "nuq/metrics",
- method: "nuqJobFinish",
- duration: Date.now() - start,
- scrapeId: id,
+ return withSpan("nuq.jobFinish", async span => {
+ setSpanAttributes(span, {
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
});
- }
+
+ const start = Date.now();
+ try {
+ const result =
+ (
+ await nuqPool.query(
+ `
+ WITH updated AS (UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id)
+ SELECT pg_notify('${this.queueName}', (id::text || '|completed')) FROM updated;
+ `,
+ [id, lock, returnvalue],
+ )
+ ).rowCount !== 0;
+
+ setSpanAttributes(span, {
+ "nuq.job_finished": result,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ _logger.info("nuqJobFinish metrics", {
+ module: "nuq/metrics",
+ method: "nuqJobFinish",
+ duration,
+ scrapeId: id,
+ });
+ }
+ });
}
public async jobFail(
@@ -497,27 +569,44 @@ class NuQ<JobData = any, JobReturnValue = any> {
failedReason: string,
_logger: Logger = logger,
): Promise<boolean> {
- const start = Date.now();
- try {
- return (
- (
- await nuqPool.query(
- `
- WITH updated AS (UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id)
- SELECT pg_notify('${this.queueName}', (id::text || '|failed')) FROM updated;
- `,
- [id, lock, failedReason],
- )
- ).rowCount !== 0
- );
- } finally {
- _logger.info("nuqJobFail metrics", {
- module: "nuq/metrics",
- method: "nuqJobFail",
- duration: Date.now() - start,
- scrapeId: id,
+ return withSpan("nuq.jobFail", async span => {
+ setSpanAttributes(span, {
+ "nuq.queue_name": this.queueName,
+ "nuq.job_id": id,
+ "nuq.failed_reason": failedReason,
});
- }
+
+ const start = Date.now();
+ try {
+ const result =
+ (
+ await nuqPool.query(
+ `
+ WITH updated AS (UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id)
+ SELECT pg_notify('${this.queueName}', (id::text || '|failed')) FROM updated;
+ `,
+ [id, lock, failedReason],
+ )
+ ).rowCount !== 0;
+
+ setSpanAttributes(span, {
+ "nuq.job_failed": result,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ _logger.info("nuqJobFail metrics", {
+ module: "nuq/metrics",
+ method: "nuqJobFail",
+ duration,
+ scrapeId: id,
+ });
+ }
+ });
}
// === Metrics
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 17bcd4bea..674929d76 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -66,6 +66,11 @@ import {
} from "../../types";
import { scrapeSitemap } from "../../scraper/crawler/sitemap";
import { shutdownOtel } from "../../otel";
+import {
+ withTraceContextAsync,
+ withSpan,
+ setSpanAttributes,
+} from "../../lib/otel-tracer";
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -1055,6 +1060,27 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {
zeroDataRetention: job.data?.zeroDataRetention ?? false,
});
+ // Restore trace context if available and execute within span
+ if (job.data.traceContext) {
+ return withTraceContextAsync(job.data.traceContext, () =>
+ withSpan("worker.scrape.process", async span => {
+ setSpanAttributes(span, {
+ "worker.job_id": job.id,
+ "worker.mode": job.data.mode,
+ "worker.team_id": job.data.team_id,
+ "worker.crawl_id": job.data.crawl_id || "none",
+ "worker.url": job.data.mode === "single_urls" ? job.data.url : "n/a",
+ });
+
+ return processJobWithTracing(job, logger);
+ }),
+ );
+ } else {
+ return processJobWithTracing(job, logger);
+ }
+};
+
+async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
try {
try {
let extendLockInterval: NodeJS.Timeout | null = null;
@@ -1135,7 +1161,7 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {
throw new Error(serializeTransportableError(new UnknownError(error)));
}
}
-};
+}
const exitHandler = () => {
shutdownOtel().finally(() => {
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 0690ec1e2..86f96b6b9 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -10,11 +10,13 @@ import { ExtractorOptions, Document } from "./lib/entities";
import { InternalOptions } from "./scraper/scrapeURL";
import type { CostTracking } from "./lib/cost-tracking";
import { webhookSchema } from "./services/webhook/schema";
+import { SerializedTraceContext } from "./lib/otel-tracer";
type ScrapeJobCommon = {
concurrencyLimited?: boolean;
team_id: string;
zeroDataRetention: boolean;
+ traceContext?: SerializedTraceContext;
};
export type ScrapeJobData = ScrapeJobCommon &