Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): route data layer scrapes (#3834)
Abimael Martellc5a9bd76765c
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.tsindex 9d1a3ee5b..594d55cde 100644--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts@@ -5,7 +5,9 @@*/describe("Audio format engine routing (buildFallbackList)", () => {- let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines").buildFallbackList;+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;+ let clearDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").clearDataLayerCapabilitiesForTest;+ let setDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").setDataLayerCapabilitiesForTest;const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;const originalIndexUrl = process.env.INDEX_DATABASE_URL;@@ -20,6 +22,14 @@ describe("Audio format engine routing (buildFallbackList)", () => {({ buildFallbackList } = await import("../../../scraper/scrapeURL/engines/index.js"));+ ({+ clearDataLayerCapabilitiesForTest,+ setDataLayerCapabilitiesForTest,+ } = await import("../../../lib/data-layer.js"));+ });++ afterEach(() => {+ clearDataLayerCapabilitiesForTest();});afterAll(() => {@@ -109,4 +119,21 @@ describe("Audio format engine routing (buildFallbackList)", () => {expect(engines).toContain("fire-engine;chrome-cdp");});++ it("does not route agent index-only requests through the data layer", async () => {+ setDataLayerCapabilitiesForTest({+ domains: ["profiles.example"],+ });++ const meta = buildStubMeta([]);+ meta.url = "https://profiles.example/person/example-person";+ meta.options.formats = [{ type: "markdown" }];+ meta.internalOptions.agentIndexOnly = true;+ meta.internalOptions.teamFlags = { enrichBeta: true };++ const fallback = await buildFallbackList(meta);+ const engines = fallback.map(f => f.engine);++ expect(engines).toEqual(["index", "index;documents"]);+ });});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 3d3165600..c1d1758e9 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1319,6 +1319,7 @@ export type TeamFlags = {searchFeedbackOptOut?: boolean;researchBeta?: boolean;highlightsBeta?: boolean;+ enrichBeta?: boolean;// routes the team's new queue work to the FoundationDB backendnuqFdb?: boolean;} | null;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex f51a10a73..2111b486d 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1548,6 +1548,7 @@ export type TeamFlags = {researchBeta?: boolean;highlightsBeta?: boolean;menuBeta?: boolean;+ enrichBeta?: boolean;} | null;interface RequestWithMaybeACUC<diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 364659d3e..1163a09e7 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -38,6 +38,7 @@ import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forciimport responseTime from "response-time";import { shutdownWebhookQueue } from "./services/webhook";import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";+import { warmDataLayerCapabilities } from "./lib/data-layer";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -119,8 +120,9 @@ async function startServer(port = DEFAULT_PORT) {try {await initializeBlocklist();initializeEngineForcing();+ await warmDataLayerCapabilities();} catch (error) {- logger.error("Failed to initialize blocklist and engine forcing", {+ logger.error("Failed to initialize API startup dependencies", {error,});throw error;diff --git a/apps/api/src/lib/data-layer.test.ts b/apps/api/src/lib/data-layer.test.tsnew file mode 100644index 000000000..9a0baaf20--- /dev/null+++ b/apps/api/src/lib/data-layer.test.ts@@ -0,0 +1,231 @@+import { fetch } from "undici";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++import { config } from "../config";+import {+ canUseDataLayerForRequest,+ clearDataLayerCapabilitiesForTest,+ getDataLayerRequestLogContext,+ getDataLayerResponseLogContext,+ getDataLayerSuccessCredits,+ isDataLayerSupportedUrl,+ isSuccessfulDataLayerStatusCode,+ isSupportedDataLayerFormatRequest,+ setDataLayerCapabilitiesForTest,+ warmDataLayerCapabilities,+} from "./data-layer";++vi.mock("undici", () => ({+ fetch: vi.fn(),+}));++const originalConfig = {+ FIRE_ENGINE_BETA_URL: config.FIRE_ENGINE_BETA_URL,+};++describe("data layer routing", () => {+ beforeEach(() => {+ vi.mocked(fetch).mockReset();+ config.FIRE_ENGINE_BETA_URL = "https://fire-engine.example";+ setDataLayerCapabilitiesForTest({+ domains: ["profiles.example"],+ baseDomains: ["network.example"],+ });+ });++ afterEach(() => {+ config.FIRE_ENGINE_BETA_URL = originalConfig.FIRE_ENGINE_BETA_URL;+ clearDataLayerCapabilitiesForTest();+ });++ it("detects URLs using Fire Engine capabilities", async () => {+ await expect(+ isDataLayerSupportedUrl(+ "https://profiles.example/person/example-person/details/experience/?trk=foo",+ ),+ ).resolves.toBe(true);+ await expect(+ isDataLayerSupportedUrl("https://www.network.example/any/path"),+ ).resolves.toBe(true);+ await expect(+ isDataLayerSupportedUrl("https://other.example/person/example-person"),+ ).resolves.toBe(false);+ await expect(isDataLayerSupportedUrl("not a url")).resolves.toBe(false);+ });++ it("caches failed Fire Engine capabilities lookups briefly", async () => {+ clearDataLayerCapabilitiesForTest();+ vi.mocked(fetch).mockResolvedValue({+ ok: false,+ status: 503,+ } as Awaited<ReturnType<typeof fetch>>);++ await expect(+ isDataLayerSupportedUrl("https://profiles.example/person/example-person"),+ ).resolves.toBe(false);+ await expect(+ isDataLayerSupportedUrl("https://profiles.example/person/example-person"),+ ).resolves.toBe(false);++ expect(fetch).toHaveBeenCalledTimes(1);+ });++ it("warms capabilities before request-time URL checks", async () => {+ clearDataLayerCapabilitiesForTest();+ vi.mocked(fetch).mockResolvedValue({+ ok: true,+ status: 200,+ json: async () => ({+ ttlSeconds: 300,+ domains: ["profiles.example"],+ }),+ } as Awaited<ReturnType<typeof fetch>>);++ await warmDataLayerCapabilities();+ await expect(+ isDataLayerSupportedUrl("https://profiles.example/person/example-person"),+ ).resolves.toBe(true);++ expect(fetch).toHaveBeenCalledTimes(1);+ });++ it("builds a compact request log context", () => {+ expect(+ getDataLayerRequestLogContext(+ "https://profiles.example/person/example-person/details/experience/?trk=foo",+ ),+ ).toEqual({+ url: "https://profiles.example/person/example-person/details/experience/?trk=foo",+ host: "profiles.example",+ pathPrefix: "person",+ });++ expect(getDataLayerRequestLogContext("not a url")).toBeUndefined();+ });++ it("extracts response cache metadata for logs", () => {+ expect(+ getDataLayerResponseLogContext({+ cacheState: "hit",+ cachedAt: "2026-06-21T10:00:00.000Z",+ cacheAgeMs: 1000,+ request_id: "req_123",+ extra: "ignored",+ }),+ ).toEqual({+ cacheState: "hit",+ cachedAt: "2026-06-21T10:00:00.000Z",+ cacheAgeMs: 1000,+ providerRequestId: "req_123",+ });++ expect(getDataLayerResponseLogContext(null)).toEqual({});+ });++ it("accepts only formats that Fire Engine can return directly", () => {+ expect(isSupportedDataLayerFormatRequest(undefined)).toBe(true);+ expect(isSupportedDataLayerFormatRequest([{ type: "markdown" }])).toBe(+ true,+ );+ expect(isSupportedDataLayerFormatRequest(["json"])).toBe(true);+ expect(+ isSupportedDataLayerFormatRequest([+ { type: "markdown" },+ { type: "json" },+ ]),+ ).toBe(true);+ expect(isSupportedDataLayerFormatRequest([{ type: "html" }])).toBe(false);+ expect(isSupportedDataLayerFormatRequest([])).toBe(false);+ });++ it("allows eligible requests through the blocklist when the team flag is enabled", async () => {+ await expect(+ canUseDataLayerForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: { enrichBeta: true },+ }),+ ).resolves.toBe(true);++ await expect(+ canUseDataLayerForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "json" }],+ actions: [{ type: "wait" }],+ flags: { enrichBeta: true },+ }),+ ).resolves.toBe(false);++ await expect(+ canUseDataLayerForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "json" }],+ zeroDataRetention: true,+ flags: { enrichBeta: true },+ }),+ ).resolves.toBe(false);+ });++ it("does not bypass unless the team flag is enabled", async () => {+ await expect(+ canUseDataLayerForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ }),+ ).resolves.toBe(false);++ await expect(+ canUseDataLayerForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: { enrichBeta: false },+ }),+ ).resolves.toBe(false);+ });++ it("does not bypass unless Fire Engine is configured", async () => {+ config.FIRE_ENGINE_BETA_URL = undefined;++ await expect(+ canUseDataLayerForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: { enrichBeta: true },+ }),+ ).resolves.toBe(false);+ });++ it("returns 15 credits only for successful handled responses", () => {+ expect(isSuccessfulDataLayerStatusCode(200)).toBe(true);+ expect(isSuccessfulDataLayerStatusCode(204)).toBe(true);+ expect(isSuccessfulDataLayerStatusCode(304)).toBe(true);+ expect(isSuccessfulDataLayerStatusCode(404)).toBe(false);++ expect(+ getDataLayerSuccessCredits({+ dataLayer: { handled: true, integrationId: "example" },+ statusCode: 200,+ }),+ ).toBe(15);++ expect(+ getDataLayerSuccessCredits({+ dataLayer: { handled: true, integrationId: "example" },+ statusCode: 304,+ }),+ ).toBe(15);++ expect(+ getDataLayerSuccessCredits({+ dataLayer: { handled: true, integrationId: "example" },+ statusCode: 404,+ }),+ ).toBeNull();++ expect(+ getDataLayerSuccessCredits({+ statusCode: 200,+ }),+ ).toBeNull();+ });+});diff --git a/apps/api/src/lib/data-layer.ts b/apps/api/src/lib/data-layer.tsnew file mode 100644index 000000000..30a7e010c--- /dev/null+++ b/apps/api/src/lib/data-layer.ts@@ -0,0 +1,341 @@+import { fetch } from "undici";+import { z } from "zod";++import { config } from "../config";+import type { FormatObject } from "../controllers/v2/types";+import { logger as rootLogger } from "./logger";++type RouteInput = {+ url: string;+ formats?: FormatObject[] | unknown[];+ actions?: unknown[];+ headers?: Record<string, unknown>;+ waitFor?: number;+ mobile?: boolean;+ location?: unknown;+ proxy?: unknown;+ blockAds?: boolean;+ zeroDataRetention?: boolean;+ lockdown?: boolean;+ flags?: { enrichBeta?: boolean } | null;+};++export type DataLayerScrapeMetadata = {+ handled: true;+ integrationId?: string;+};++const SUPPORTED_FORMATS = new Set(["markdown", "json", "deterministicJson"]);+const DATA_LAYER_SUCCESS_CREDITS = 15;++const DATA_LAYER_CAPABILITIES_PATH = "/v1/data-layer/capabilities";+const DATA_LAYER_CAPABILITIES_TIMEOUT_MS = 2_000;+const DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS = 30_000;++const dataLayerCapabilitiesSchema = z+ .object({+ version: z.number().optional(),+ ttlSeconds: z.number().positive().optional(),+ domains: z.string().array().optional(),+ baseDomains: z.string().array().optional(),+ })+ .passthrough();++type DataLayerCapabilities = {+ domains: Set<string>;+ baseDomains: Set<string>;+ ttlMs: number;+};++let cachedCapabilities:+ | {+ expiresAt: number;+ value: DataLayerCapabilities | null;+ }+ | undefined;+let capabilitiesRequest: Promise<DataLayerCapabilities | null> | undefined;++function normalizeHost(host: string): string {+ return host.trim().toLowerCase().replace(/\.$/, "");+}++function normalizeCapabilities(+ raw: z.infer<typeof dataLayerCapabilitiesSchema>,+): DataLayerCapabilities {+ const ttlMs =+ typeof raw.ttlSeconds === "number" && Number.isFinite(raw.ttlSeconds)+ ? raw.ttlSeconds * 1000+ : DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS;++ return {+ domains: new Set((raw.domains ?? []).map(normalizeHost)),+ baseDomains: new Set((raw.baseDomains ?? []).map(normalizeHost)),+ ttlMs: Math.max(1_000, ttlMs),+ };+}++function getFireEngineDataLayerUrl(): string | null {+ if (!config.FIRE_ENGINE_BETA_URL) {+ return null;+ }++ return `${config.FIRE_ENGINE_BETA_URL.replace(/\/+$/, "")}${DATA_LAYER_CAPABILITIES_PATH}`;+}++async function fetchDataLayerCapabilities(): Promise<DataLayerCapabilities | null> {+ const url = getFireEngineDataLayerUrl();+ if (!url) {+ return null;+ }++ try {+ const response = await fetch(url, {+ method: "GET",+ signal: AbortSignal.timeout(DATA_LAYER_CAPABILITIES_TIMEOUT_MS),+ });++ if (!response.ok) {+ rootLogger.warn("Data layer capabilities request failed", {+ statusCode: response.status,+ });+ return null;+ }++ const parsed = dataLayerCapabilitiesSchema.parse(await response.json());+ return normalizeCapabilities(parsed);+ } catch (error) {+ rootLogger.warn("Data layer capabilities request errored", { error });+ return null;+ }+}++async function getDataLayerCapabilities(): Promise<DataLayerCapabilities | null> {+ if (cachedCapabilities && cachedCapabilities.expiresAt > Date.now()) {+ return cachedCapabilities.value;+ }++ if (!capabilitiesRequest) {+ capabilitiesRequest = fetchDataLayerCapabilities().finally(() => {+ capabilitiesRequest = undefined;+ });+ }++ const capabilities = await capabilitiesRequest;+ cachedCapabilities = {+ value: capabilities,+ expiresAt:+ Date.now() ++ (capabilities?.ttlMs ?? DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS),+ };++ return capabilities;+}++export async function warmDataLayerCapabilities(): Promise<void> {+ if (!config.FIRE_ENGINE_BETA_URL) {+ return;+ }++ const capabilities = await getDataLayerCapabilities();+ rootLogger.info("Data layer capabilities warmup completed", {+ available: capabilities !== null,+ domains: capabilities?.domains.size ?? 0,+ baseDomains: capabilities?.baseDomains.size ?? 0,+ });+}++function dataLayerCapabilitiesMatchUrl(+ capabilities: DataLayerCapabilities,+ inputUrl: string,+): boolean {+ let parsed: URL;+ try {+ parsed = new URL(inputUrl);+ } catch {+ return false;+ }++ const host = normalizeHost(parsed.hostname);+ if (capabilities.domains.has(host)) {+ return true;+ }++ for (const baseDomain of capabilities.baseDomains) {+ if (host === baseDomain || host.endsWith(`.${baseDomain}`)) {+ return true;+ }+ }++ return false;+}++export async function isDataLayerSupportedUrl(+ inputUrl: string,+): Promise<boolean> {+ const capabilities = await getDataLayerCapabilities();+ return (+ capabilities !== null &&+ dataLayerCapabilitiesMatchUrl(capabilities, inputUrl)+ );+}++export function getDataLayerRequestLogContext(inputUrl: string):+ | {+ url: string;+ host: string;+ pathPrefix: string | null;+ }+ | undefined {+ let parsed: URL;+ try {+ parsed = new URL(inputUrl);+ } catch {+ return undefined;+ }++ return {+ url: parsed.href,+ host: parsed.hostname.toLowerCase(),+ pathPrefix:+ parsed.pathname+ .split("/")+ .map(part => part.trim())+ .filter(part => part.length > 0)[0] ?? null,+ };+}++export function getDataLayerResponseLogContext(meta: unknown): {+ cacheState?: string;+ cachedAt?: string;+ cacheAgeMs?: number;+ providerRequestId?: string;+} {+ if (typeof meta !== "object" || meta === null) {+ return {};+ }++ const record = meta as Record<string, unknown>;+ const requestId = record.request_id ?? record.requestId;++ return {+ ...(typeof record.cacheState === "string"+ ? { cacheState: record.cacheState }+ : {}),+ ...(typeof record.cachedAt === "string"+ ? { cachedAt: record.cachedAt }+ : {}),+ ...(typeof record.cacheAgeMs === "number"+ ? { cacheAgeMs: record.cacheAgeMs }+ : {}),+ ...(typeof requestId === "string" ? { providerRequestId: requestId } : {}),+ };+}++export function isSuccessfulDataLayerStatusCode(statusCode: number): boolean {+ return (statusCode >= 200 && statusCode < 300) || statusCode === 304;+}++export function isSupportedDataLayerFormatRequest(+ formats?: FormatObject[] | unknown[],+): boolean {+ if (formats === undefined) {+ return true;+ }++ if (!Array.isArray(formats) || formats.length === 0) {+ return false;+ }++ return formats.every(format => {+ const type =+ typeof format === "string"+ ? format+ : typeof format === "object" && format !== null && "type" in format+ ? (format as { type?: unknown }).type+ : undefined;++ return typeof type === "string" && SUPPORTED_FORMATS.has(type);+ });+}++export async function canUseDataLayerForRequest(+ input: RouteInput,+): Promise<boolean> {+ if (input.flags?.enrichBeta !== true) {+ return false;+ }++ if (!config.FIRE_ENGINE_BETA_URL) {+ return false;+ }++ if (!input.url) {+ return false;+ }++ if (input.zeroDataRetention || input.lockdown) {+ return false;+ }++ if (Array.isArray(input.actions) && input.actions.length > 0) {+ return false;+ }++ if (input.headers && Object.keys(input.headers).length > 0) {+ return false;+ }++ if (input.waitFor !== undefined && input.waitFor !== 0) {+ return false;+ }++ if (input.mobile || input.location || input.blockAds === false) {+ return false;+ }++ if (input.proxy === "stealth" || input.proxy === "enhanced") {+ return false;+ }++ if (!isSupportedDataLayerFormatRequest(input.formats)) {+ return false;+ }++ return isDataLayerSupportedUrl(input.url);+}++export function getDataLayerSuccessCredits(input: {+ dataLayer?: DataLayerScrapeMetadata;+ statusCode?: number | null;+}): number | null {+ if (input.dataLayer?.handled !== true) {+ return null;+ }++ const statusCode = input.statusCode;+ if (+ statusCode === undefined ||+ statusCode === null ||+ !isSuccessfulDataLayerStatusCode(statusCode)+ ) {+ return null;+ }++ return DATA_LAYER_SUCCESS_CREDITS;+}++export function setDataLayerCapabilitiesForTest(input: {+ domains?: string[];+ baseDomains?: string[];+ ttlSeconds?: number;+}) {+ cachedCapabilities = {+ value: normalizeCapabilities(input),+ expiresAt: Date.now() + (input.ttlSeconds ?? 300) * 1000,+ };+}++export function clearDataLayerCapabilitiesForTest() {+ cachedCapabilities = undefined;+ capabilitiesRequest = undefined;+}diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex 70361b069..cdf49cb52 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -1,6 +1,33 @@import { calculateCreditsToBeBilled } from "./scrape-billing";describe("calculateCreditsToBeBilled", () => {+ it("bills handled data layer successes at 15 credits", async () => {+ const credits = await calculateCreditsToBeBilled(+ {+ formats: [{ type: "markdown" }],+ } as any,+ {+ teamId: "team-id",+ },+ {+ metadata: {+ statusCode: 200,+ url: "https://profiles.example/in/example-person",+ proxyUsed: "basic",+ },+ } as any,+ {+ totalCost: 0,+ } as any,+ {} as any,+ undefined,+ undefined,+ { handled: true },+ );++ expect(credits).toBe(15);+ });+it("bills X/Twitter scrapes at 30 credits", async () => {const credits = await calculateCreditsToBeBilled({diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 2de00540a..23b334c84 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -10,6 +10,10 @@ import { hasFormatOfType } from "./format-utils";import { TransportableError } from "./error";import { FeatureFlag } from "../scraper/scrapeURL/engines";import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";+import {+ DataLayerScrapeMetadata,+ getDataLayerSuccessCredits,+} from "./data-layer";const creditsPerPDFPage = 1;const stealthProxyCostBonus = 4;@@ -28,6 +32,7 @@ export async function calculateCreditsToBeBilled(flags: TeamFlags,error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,+ dataLayer?: DataLayerScrapeMetadata,) {const costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON> =costTracking instanceof CostTracking ? costTracking.toJSON() : costTracking;@@ -55,6 +60,14 @@ export async function calculateCreditsToBeBilled(return creditsToBeBilled;}+ const dataLayerCredits = getDataLayerSuccessCredits({+ dataLayer,+ statusCode: document.metadata?.statusCode,+ });+ if (dataLayerCredits !== null) {+ return dataLayerCredits;+ }+let creditsToBeBilled = 1; // Assuming 1 credit per documentif (options.lockdown) {diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex f3c02ab1d..e5e7700ab 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -29,6 +29,8 @@ import {CREDITS_FEATURE_ID,} from "../services/autumn/autumn.service";import { getTeamBalance } from "../services/autumn/usage";+import { canUseDataLayerForRequest } from "../lib/data-layer";+import { getScrapeZDR } from "../lib/zdr-helpers";export function checkCreditsMiddleware(_minimum?: number,@@ -291,21 +293,45 @@ export function blocklistMiddleware(res: Response,next: NextFunction,) {- if (- typeof req.body.url === "string" &&- isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {- team_id: req.acuc?.team_id ?? null,- origin: typeof req.body.origin === "string" ? req.body.origin : null,- })- ) {- if (!res.headersSent) {- return res.status(403).json({- success: false,- error: UNSUPPORTED_SITE_MESSAGE,- });+ (async () => {+ const zeroDataRetention =+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body?.zeroDataRetention === true ||+ req.body?.lockdown === true;+ const canUseDataLayer =+ typeof req.body.url === "string" &&+ (await canUseDataLayerForRequest({+ url: req.body.url,+ formats: req.body.formats,+ actions: req.body.actions,+ headers: req.body.headers,+ waitFor: req.body.waitFor,+ mobile: req.body.mobile,+ location: req.body.location,+ proxy: req.body.proxy,+ blockAds: req.body.blockAds,+ zeroDataRetention,+ lockdown: req.body.lockdown,+ flags: req.acuc?.flags ?? null,+ }));++ if (+ typeof req.body.url === "string" &&+ !canUseDataLayer &&+ isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {+ team_id: req.acuc?.team_id ?? null,+ origin: typeof req.body.origin === "string" ? req.body.origin : null,+ })+ ) {+ if (!res.headersSent) {+ return res.status(403).json({+ success: false,+ error: UNSUPPORTED_SITE_MESSAGE,+ });+ }}- }- next();+ next();+ })().catch(err => next(err));}export function countryCheck(diff --git a/apps/api/src/scraper/scrapeURL/engines/data-layer.ts b/apps/api/src/scraper/scrapeURL/engines/data-layer.tsnew file mode 100644index 000000000..53275c2e8--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/data-layer.ts@@ -0,0 +1,183 @@+import { z } from "zod";++import { Meta } from "..";+import { EngineScrapeResult } from ".";+import {+ getDataLayerRequestLogContext,+ getDataLayerResponseLogContext,+ isSuccessfulDataLayerStatusCode,+} from "../../../lib/data-layer";+import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";+import { robustFetch } from "../lib/fetch";+import { EngineError } from "../error";+import { fireEngineURL } from "./fire-engine/scrape";++const dataLayerResultSchema = z+ .object({+ content: z.string().optional(),+ json: z.unknown().optional(),+ url: z.string().optional(),+ pageStatusCode: z.number().optional(),+ pageError: z.string().optional(),+ responseHeaders: z.record(z.string(), z.string()).optional(),+ meta: z.record(z.string(), z.unknown()).optional(),+ error: z.string().optional(),+ })+ .passthrough();++const dataLayerScrapeResponseSchema = z.union([+ z+ .object({+ handled: z.literal(false),+ })+ .passthrough(),+ z+ .object({+ handled: z.literal(true),+ statusCode: z.number(),+ integrationId: z.string().optional(),+ credits: z.number().optional(),+ result: dataLayerResultSchema.optional(),+ })+ .passthrough(),+]);++function getContentType(headers?: Record<string, string>): string {+ return (+ Object.entries(headers ?? {}).find(+ ([key]) => key.toLowerCase() === "content-type",+ )?.[1] ?? ""+ );+}++export function dataLayerMaxReasonableTime(meta: Meta): number {+ return meta.options.timeout ?? 60_000;+}++export async function scrapeURLWithDataLayer(+ meta: Meta,+): Promise<EngineScrapeResult> {+ return withSpan("engine.data-layer.scrape", async span => {+ const startTime = Date.now();+ const url = meta.rewrittenUrl ?? meta.url;+ const requestLogContext = getDataLayerRequestLogContext(url);+ const logger = meta.logger.child({ method: "scrapeURLWithDataLayer" });++ setSpanAttributes(span, {+ "engine.type": "data-layer",+ "engine.url": url,+ "engine.team_id": meta.internalOptions.teamId,+ });++ logger.info("Data layer scrape started", {+ ...requestLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ maxAge: meta.options.maxAge,+ });++ try {+ const response = await robustFetch({+ url: `${fireEngineURL}/v1/data-layer/scrape`,+ method: "POST",+ body: {+ url,+ formats: meta.options.formats,+ maxAge: meta.options.maxAge,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ source: "firecrawl",+ },+ logger: logger.child({ method: "dataLayerScrape/robustFetch" }),+ tryCount: 2,+ ignoreFailureStatus: true,+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ schema: dataLayerScrapeResponseSchema,+ });++ if (!response.handled) {+ logger.info("Data layer scrape was not handled", {+ ...requestLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ durationMs: Date.now() - startTime,+ });+ throw new EngineError("Data layer did not handle URL");+ }++ const responseLogContext = getDataLayerResponseLogContext(+ response.result?.meta,+ );+ const pageStatusCode =+ response.result?.pageStatusCode ?? response.statusCode;++ if (+ !isSuccessfulDataLayerStatusCode(response.statusCode) ||+ !isSuccessfulDataLayerStatusCode(pageStatusCode) ||+ !response.result+ ) {+ logger.warn("Data layer scrape failed", {+ ...requestLogContext,+ ...responseLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ integrationId: response.integrationId,+ statusCode: response.statusCode,+ pageStatusCode,+ durationMs: Date.now() - startTime,+ });+ throw new EngineError("Data layer request failed");+ }++ const contentType = getContentType(response.result.responseHeaders);++ logger.info("Data layer scrape completed", {+ ...requestLogContext,+ ...responseLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ integrationId: response.integrationId,+ statusCode: response.statusCode,+ durationMs: Date.now() - startTime,+ });++ setSpanAttributes(span, {+ "data-layer.integration_id": response.integrationId,+ "data-layer.status_code": response.statusCode,+ "data-layer.page_status_code": pageStatusCode,+ "data-layer.cache_state": responseLogContext.cacheState,+ "data-layer.cache_age_ms": responseLogContext.cacheAgeMs,+ "data-layer.duration_ms": Date.now() - startTime,+ });++ return {+ url: response.result.url ?? url,+ html: response.result.content ?? "",+ markdown: contentType.includes("text/markdown")+ ? response.result.content+ : undefined,+ json: response.result.json,+ error: response.result.pageError,+ statusCode: pageStatusCode,+ contentType,+ proxyUsed: "basic",+ dataLayer: {+ handled: true,+ integrationId: response.integrationId,+ },+ };+ } catch (error) {+ logger.warn("Data layer scrape errored", {+ ...requestLogContext,+ scrapeId: meta.id,+ teamId: meta.internalOptions.teamId,+ durationMs: Date.now() - startTime,+ errorMessage: error instanceof Error ? error.message : String(error),+ error,+ });+ throw error;+ }+ });+}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 7ff643f9c..f27516283 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -33,6 +33,7 @@ const successSchema = z.object({// timeTaken: z.number(),content: z.string(),+ json: z.unknown().optional(),url: z.string().optional(),pageStatusCode: z.number(),@@ -40,6 +41,7 @@ const successSchema = z.object({// TODO: this needs to be non-optional, might need fixes on f-e side to ensure reliabilityresponseHeaders: z.record(z.string(), z.string()).optional(),+ meta: z.record(z.string(), z.unknown()).optional(),// timeTakenCookie: z.number().optional(),// timeTakenRequest: z.number().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 24fe4b81f..c967e4014 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -362,6 +362,7 @@ export async function scrapeURLWithFireEngineChromeCDP(timeout: meta.abort.scrapeTimeout() ?? 300000,disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,mobileProxy: meta.featureFlags.has("stealthProxy"),+ maxAge: meta.options.maxAge,saveScrapeResultToGCS:!meta.internalOptions.zeroDataRetention &&meta.internalOptions.saveScrapeResultToGCS,@@ -437,18 +438,23 @@ export async function scrapeURLWithFireEngineChromeCDP(const audioCookies = (response.actionResults ?? []).filter(x => x.type === "getCookies").flatMap(x => x.result.cookies);+ const contentType =+ (Object.entries(response.responseHeaders ?? {}).find(+ x => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined;return {url: response.url ?? meta.url,html: response.content,+ markdown: contentType?.includes("text/markdown")+ ? response.content+ : undefined,+ json: response.json,error: response.pageError,statusCode: response.pageStatusCode,- contentType:- (Object.entries(response.responseHeaders ?? {}).find(- x => x[0].toLowerCase() === "content-type",- ) ?? [])[1] ?? undefined,+ contentType,screenshot,...(actions.length > 0@@ -499,6 +505,7 @@ export async function scrapeURLWithFireEngineTLSClient(mobileProxy: meta.featureFlags.has("stealthProxy"),timeout: meta.abort.scrapeTimeout() ?? 300000,+ maxAge: meta.options.maxAge,saveScrapeResultToGCS:!meta.internalOptions.zeroDataRetention &&meta.internalOptions.saveScrapeResultToGCS,@@ -522,18 +529,23 @@ export async function scrapeURLWithFireEngineTLSClient(sourceURL: meta.url,});}+ const contentType =+ (Object.entries(response.responseHeaders ?? {}).find(+ x => x[0].toLowerCase() === "content-type",+ ) ?? [])[1] ?? undefined;return {url: response.url ?? meta.url,html: response.content,+ markdown: contentType?.includes("text/markdown")+ ? response.content+ : undefined,+ json: response.json,error: response.pageError,statusCode: response.pageStatusCode,- contentType:- (Object.entries(response.responseHeaders ?? {}).find(- x => x[0].toLowerCase() === "content-type",- ) ?? [])[1] ?? undefined,+ contentType,proxyUsed: response.usedMobileProxy ? "stealth" : "basic",timezone: response.timezone,diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 70a703813..ebf59dbda 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -51,6 +51,7 @@ export type FireEngineScrapeRequestCommon = {mobileProxy?: boolean; // leave it undefined if user doesn't specifytimeout?: number;+ maxAge?: number;saveScrapeResultToGCS?: boolean;zeroDataRetention?: boolean;};@@ -76,6 +77,7 @@ const successSchema = z.object({timeTaken: z.number(),content: z.string(),+ json: z.unknown().optional(),url: z.string().optional(),pageStatusCode: z.number(),@@ -83,6 +85,7 @@ const successSchema = z.object({// TODO: this needs to be non-optional, might need fixes on f-e side to ensure reliabilityresponseHeaders: z.record(z.string(), z.string()).optional(),+ meta: z.record(z.string(), z.unknown()).optional(),// timeTakenCookie: z.number().optional(),// timeTakenRequest: z.number().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 89ace87f3..c487efee7 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -7,6 +7,10 @@ import {scrapeURLWithFireEngineChromeCDP,scrapeURLWithFireEngineTLSClient,} from "./fire-engine";+import {+ dataLayerMaxReasonableTime,+ scrapeURLWithDataLayer,+} from "./data-layer";import { pdfMaxReasonableTime, scrapePDF } from "./pdf";import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";import {@@ -30,8 +34,13 @@ import { getPDFMaxPages } from "../../../controllers/v2/types";import type { PdfMetadata } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";import { BrandingNotSupportedError } from "../error";+import {+ canUseDataLayerForRequest,+ type DataLayerScrapeMetadata,+} from "../../../lib/data-layer";export type Engine =+ | "data-layer"| "fire-engine;chrome-cdp"| "fire-engine(retry);chrome-cdp"| "fire-engine;chrome-cdp;stealth"@@ -131,6 +140,7 @@ export type EngineScrapeResult = {html: string;markdown?: string;+ json?: unknown;statusCode: number;error?: string;@@ -161,11 +171,13 @@ export type EngineScrapeResult = {proxyUsed: "basic" | "stealth";timezone?: string;+ dataLayer?: DataLayerScrapeMetadata;};const engineHandlers: {[E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;} = {+ "data-layer": scrapeURLWithDataLayer,index: scrapeURLWithIndex,"index;documents": scrapeURLWithIndex,"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,@@ -185,6 +197,7 @@ const engineHandlers: {const engineMRTs: {[E in Engine]: (meta: Meta) => number;} = {+ "data-layer": dataLayerMaxReasonableTime,index: indexMaxReasonableTime,"index;documents": indexMaxReasonableTime,"fire-engine;chrome-cdp": meta =>@@ -217,6 +230,27 @@ const engineOptions: {quality: number;};} = {+ "data-layer": {+ features: {+ actions: false,+ waitFor: false,+ screenshot: false,+ "screenshot@fullScreen": false,+ pdf: false,+ document: false,+ audio: false,+ video: false,+ atsv: false,+ location: false,+ mobile: false,+ skipTlsVerification: true,+ useFastMode: true,+ stealthProxy: false,+ branding: false,+ disableAdblock: false,+ },+ quality: 2000,+ },index: {features: {actions: false,@@ -546,6 +580,31 @@ export async function buildFallbackList(meta: Meta): Promise<unsupportedFeatures: Set<FeatureFlag>;}[]> {+ if (+ !meta.internalOptions.agentIndexOnly &&+ (await canUseDataLayerForRequest({+ url: meta.rewrittenUrl ?? meta.url,+ formats: meta.options.formats,+ actions: meta.options.actions,+ headers: meta.options.headers,+ waitFor: meta.options.waitFor,+ mobile: meta.options.mobile,+ location: meta.options.location,+ proxy: meta.options.proxy,+ blockAds: meta.options.blockAds,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,+ lockdown: meta.options.lockdown,+ flags: meta.internalOptions.teamFlags ?? null,+ }))+ ) {+ return [+ {+ engine: "data-layer",+ unsupportedFeatures: new Set(),+ },+ ];+ }+const shouldPrioritizeTlsClient = meta.options.__experimental_engpicker? (await queryEngpickerVerdict(meta.options.__experimental_omceDomain ?? new URL(meta.url).hostname,diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex d35566936..784ab1e63 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -105,6 +105,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {meta.rewrittenUrl ??meta.url,html: document.rawHtml!,+ json: document.json,statusCode: document.metadata.statusCode,error: document.metadata.error,screenshot: document.screenshot,@@ -547,6 +548,7 @@ export async function scrapeURLWithIndex(return {url: doc.url,html: doc.html,+ json: doc.json,statusCode: doc.statusCode,error: doc.error,screenshot: doc.screenshot,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 0c9fc5aac..207747925 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -87,12 +87,14 @@ import { writeFile } from "node:fs/promises";import path from "node:path";import { tmpdir } from "node:os";import { randomUUID } from "node:crypto";+import type { DataLayerScrapeMetadata } from "../../lib/data-layer";export type ScrapeUrlResponse =| {success: true;document: Document;unsupportedFeatures?: Set<FeatureFlag>;+ dataLayer?: DataLayerScrapeMetadata;}| {success: false;@@ -552,6 +554,8 @@ async function scrapeURLLoopIter(},);checkMarkdown = engineResult.html?.trim() ?? "";+ } else if (engineResult.markdown?.trim()) {+ checkMarkdown = engineResult.markdown.trim();} else {const requestId = meta.id || meta.internalOptions.crawlId;const zeroDataRetention = meta.internalOptions.zeroDataRetention;@@ -978,6 +982,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {let document: Document = {markdown: engineResult.markdown,rawHtml: engineResult.html,+ json: engineResult.json,screenshot: engineResult.screenshot,actions: engineResult.actions,branding: engineResult.branding,@@ -1038,6 +1043,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {success: true,document,unsupportedFeatures: result.unsupportedFeatures,+ dataLayer: engineResult.dataLayer,};});}diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.test.ts b/apps/api/src/scraper/scrapeURL/transformers/index.test.tsnew file mode 100644index 000000000..1e87844b0--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/index.test.ts@@ -0,0 +1,104 @@+import type { MockedFunction } from "vitest";+import { executeTransformers } from ".";+import { performLLMExtract } from "./llmExtract";++vi.mock("../../../services/index", () => ({+ useIndex: false,+ useSearchIndex: false,+}));++vi.mock("./llmExtract", async importOriginal => {+ const actual = await importOriginal<typeof import("./llmExtract")>();++ return {+ ...actual,+ performLLMExtract: vi.fn(actual.performLLMExtract),+ };+});++const mockedPerformLLMExtract = performLLMExtract as MockedFunction<+ typeof performLLMExtract+>;++function logger() {+ const log = {+ child: vi.fn(),+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ };+ log.child.mockReturnValue(log);+ return log;+}++describe("executeTransformers", () => {+ beforeEach(() => {+ mockedPerformLLMExtract.mockClear();+ });++ it("keeps native JSON and markdown without running LLM JSON extraction", async () => {+ const nativeJson = { id: "person-1", full_name: "Example Person" };+ const nativeMarkdown = "# Example Person";++ const document = await executeTransformers(+ {+ url: "https://www.linkedin.com/in/example",+ options: {+ formats: [{ type: "markdown" }, { type: "json" }],+ onlyMainContent: false,+ },+ internalOptions: {},+ logger: logger(),+ } as any,+ {+ rawHtml:+ "<html><head><title>Ignored</title></head><body></body></html>",+ markdown: nativeMarkdown,+ json: nativeJson,+ metadata: {+ sourceURL: "https://www.linkedin.com/in/example",+ url: "https://www.linkedin.com/in/example",+ statusCode: 200,+ contentType: "text/markdown; charset=utf-8",+ },+ } as any,+ );++ expect(mockedPerformLLMExtract).not.toHaveBeenCalled();+ expect(document.markdown).toBe(nativeMarkdown);+ expect(document.json).toEqual(nativeJson);+ });++ it("maps native JSON to v1 extract compatibility field", async () => {+ const nativeJson = { id: "person-1", full_name: "Example Person" };++ const document = await executeTransformers(+ {+ url: "https://www.linkedin.com/in/example",+ options: {+ formats: [{ type: "markdown" }, { type: "json" }],+ onlyMainContent: false,+ },+ internalOptions: { v1OriginalFormat: "extract" },+ logger: logger(),+ } as any,+ {+ rawHtml:+ "<html><head><title>Ignored</title></head><body></body></html>",+ markdown: "# Example Person",+ json: nativeJson,+ metadata: {+ sourceURL: "https://www.linkedin.com/in/example",+ url: "https://www.linkedin.com/in/example",+ statusCode: 200,+ contentType: "text/markdown; charset=utf-8",+ },+ } as any,+ );++ expect(mockedPerformLLMExtract).not.toHaveBeenCalled();+ expect(document.extract).toEqual(nativeJson);+ expect(document.json).toEqual(nativeJson);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex fb0146123..5d11c12ea 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -118,10 +118,10 @@ async function deriveMarkdownFromHTML(return document;}- // Skip markdown derivation if a postprocessor already set it- if (document.metadata.postprocessorsUsed?.length && document.markdown) {+ // Skip markdown derivation if the engine or a postprocessor already set it.+ if (document.markdown !== undefined) {meta.logger.debug(- "Skipping markdown derivation - postprocessor already set markdown",+ "Skipping markdown derivation - document already has markdown",{ postprocessorsUsed: document.metadata.postprocessorsUsed },);return document;@@ -318,6 +318,30 @@ async function deriveBrandingFromActions(return document;}+async function performLLMExtractUnlessNativeJson(+ meta: Meta,+ document: Document,+): Promise<Document> {+ if (+ document.json !== undefined &&+ hasFormatOfType(meta.options.formats, "json")+ ) {+ if (+ meta.internalOptions.v1OriginalFormat === "extract" &&+ document.extract === undefined+ ) {+ document.extract = document.json;+ }++ meta.logger.debug(+ "Skipping LLM JSON extraction - document already has native JSON",+ );+ return document;+ }++ return performLLMExtract(meta, document);+}+function coerceFieldsToFormats(meta: Meta, document: Document): Document {const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");@@ -599,7 +623,7 @@ const transformerStack: Transformer[] = [fetchMenu,...(useIndex ? [sendDocumentToIndex] : []),...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time search- performLLMExtract,+ performLLMExtractUnlessNativeJson,performDeterministicJson,performSummary,performQuery,diff --git a/apps/api/src/services/index.test.ts b/apps/api/src/services/index.test.tsnew file mode 100644index 000000000..ba7507e7d--- /dev/null+++ b/apps/api/src/services/index.test.ts@@ -0,0 +1,105 @@+import { vi } from "vitest";++const { file, bucket, getSavedPayload, setSavedPayload } = vi.hoisted(() => {+ let savedPayload = "";+ const getSavedPayload = () => savedPayload;+ const setSavedPayload = (value: string) => {+ savedPayload = value;+ };+ const file = vi.fn(() => ({+ save: vi.fn(async (payload: string) => {+ savedPayload = payload;+ }),+ download: vi.fn(async () => [Buffer.from(savedPayload)]),+ }));+ const bucket = vi.fn(() => ({ file }));++ return { file, bucket, getSavedPayload, setSavedPayload };+});++vi.mock("../config", () => ({+ config: {+ GCS_INDEX_BUCKET_NAME: "index-bucket",+ GCS_MEDIA_BUCKET_NAME: "media-bucket",+ },+}));++vi.mock("../lib/gcs-jobs", () => ({+ storage: { bucket },+}));++vi.mock("../lib/otel-tracer", () => ({+ withSpan: vi.fn(async (_name, fn) => fn({})),+ setSpanAttributes: vi.fn(),+}));++vi.mock("../db/connection", () => ({+ dbIndex: {},+}));++vi.mock("../db/rpc", () => ({+ insertOmceJobIfNeeded: vi.fn(),+ queryIndexAtSplitLevel: vi.fn(),+ queryIndexAtDomainSplitLevel: vi.fn(),+ queryOmceSignatures: vi.fn(),+ queryEngpickerVerdict: vi.fn(),+ queryIndexAtSplitLevelWithMeta: vi.fn(),+ queryIndexAtDomainSplitLevelWithMeta: vi.fn(),+ queryDomainPriority: vi.fn(),+}));++vi.mock("./redis", () => ({+ redisEvictConnection: {},+}));++vi.mock("./index-cache", () => ({+ deriveIndexVariantKey: vi.fn(),+ upsertCachedIndexEntries: vi.fn(),+ useIndexCache: false,+}));++vi.mock("../db/schema", () => ({}));++import { getIndexFromGCS, saveIndexToGCS } from "./index";++describe("index GCS documents", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ setSavedPayload("");+ });++ it("persists native JSON payloads with cached index documents", async () => {+ await saveIndexToGCS("idx-1", {+ url: "https://www.example.com/profile",+ html: "# Example",+ json: { id: "person-1", full_name: "Example Person" },+ statusCode: 200,+ contentType: "text/markdown; charset=utf-8",+ proxyUsed: "basic",+ });++ expect(file).toHaveBeenCalledWith("idx-1.json");+ expect(JSON.parse(getSavedPayload()).json).toEqual({+ id: "person-1",+ full_name: "Example Person",+ });+ });++ it("reads native JSON payloads from cached index documents", async () => {+ setSavedPayload(+ JSON.stringify({+ url: "https://www.example.com/profile",+ html: "# Example",+ json: { id: "person-1", full_name: "Example Person" },+ statusCode: 200,+ }),+ );++ const doc = await getIndexFromGCS("idx-1.json");++ expect(doc?.json).toEqual({+ id: "person-1",+ full_name: "Example Person",+ });+ });+});diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 793ce4db9..9e090d819 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -118,6 +118,7 @@ export async function saveIndexToGCS(doc: {url: string;html: string;+ json?: unknown;statusCode: number;error?: string;screenshot?: string;diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex fa2366f59..0fd9eee1b 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -21,6 +21,7 @@ import Express from "express";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";+import { warmDataLayerCapabilities } from "../lib/data-layer";import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";import { finishCrawlSuper } from "./worker/crawl-logic";import { getCrawl } from "../lib/crawl-redis";@@ -465,6 +466,7 @@ app.listen(workerPort, (error?: Error) => {});initializeEngineForcing();+ await warmDataLayerCapabilities();if (config.USE_DB_AUTHENTICATION && !config.DISABLE_MONITORING) {monitorSchedulerInterval = setInterval(() => {diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.tsindex 63dfd6a91..b58df257f 100644--- a/apps/api/src/services/worker/nuq-worker-runner.ts+++ b/apps/api/src/services/worker/nuq-worker-runner.ts@@ -7,6 +7,7 @@ import { register } from "prom-client";import Express from "express";import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";+import { warmDataLayerCapabilities } from "../../lib/data-layer";export type WorkerQueue = {getJobToProcess(logger?: any): Promise<NuQJob<any, any> | null>;@@ -58,6 +59,7 @@ export async function runNuqWorker(options: {try {await initializeBlocklist();initializeEngineForcing();+ await warmDataLayerCapabilities();await options.beforeStart?.();} catch (error) {_logger.error("Failed to initialize NuQ worker", {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 5b5ff8b62..6bb671d65 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -91,6 +91,7 @@ import {recordMonitorScrapeFailure,recordMonitorScrapeSuccess,} from "../monitoring/results";+import type { DataLayerScrapeMetadata } from "../../lib/data-layer";configDotenv();@@ -110,6 +111,7 @@ async function billScrapeJob(flags: TeamFlags,error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,+ dataLayer?: DataLayerScrapeMetadata,) {let creditsToBeBilled: number | null = null;const billing = resolveBillingMetadata({@@ -137,6 +139,7 @@ async function billScrapeJob(flags,error,unsupportedFeatures,+ dataLayer,);// Charge the keyless free tier's per-IP daily credit budget unless this@@ -555,6 +558,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,undefined,pipeline.unsupportedFeatures,+ pipeline.dataLayer,);doc.metadata.creditsUsed = credits_billed ?? undefined;@@ -645,6 +649,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,undefined,pipeline.unsupportedFeatures,+ pipeline.dataLayer,);doc.metadata.creditsUsed = credits_billed ?? undefined;