Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): route data layer scrapes (#3834)

Abimael Martellc5a9bd76765c
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
index 9d1a3ee5b..594d55cde 100644
--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
@@ -5,7 +5,9 @@
*/
describe("Audio format engine routing (buildFallbackList)", () => {
- let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines").buildFallbackList;
+ let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
+ let clearDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").clearDataLayerCapabilitiesForTest;
+ let setDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").setDataLayerCapabilitiesForTest;
const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;
const originalIndexUrl = process.env.INDEX_DATABASE_URL;
@@ -20,6 +22,14 @@ describe("Audio format engine routing (buildFallbackList)", () => {
({ buildFallbackList } = await import(
"../../../scraper/scrapeURL/engines/index.js"
));
+ ({
+ clearDataLayerCapabilitiesForTest,
+ setDataLayerCapabilitiesForTest,
+ } = await import("../../../lib/data-layer.js"));
+ });
+
+ afterEach(() => {
+ clearDataLayerCapabilitiesForTest();
});
afterAll(() => {
@@ -109,4 +119,21 @@ describe("Audio format engine routing (buildFallbackList)", () => {
expect(engines).toContain("fire-engine;chrome-cdp");
});
+
+ it("does not route agent index-only requests through the data layer", async () => {
+ setDataLayerCapabilitiesForTest({
+ domains: ["profiles.example"],
+ });
+
+ const meta = buildStubMeta([]);
+ meta.url = "https://profiles.example/person/example-person";
+ meta.options.formats = [{ type: "markdown" }];
+ meta.internalOptions.agentIndexOnly = true;
+ meta.internalOptions.teamFlags = { enrichBeta: true };
+
+ const fallback = await buildFallbackList(meta);
+ const engines = fallback.map(f => f.engine);
+
+ expect(engines).toEqual(["index", "index;documents"]);
+ });
});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 3d3165600..c1d1758e9 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1319,6 +1319,7 @@ export type TeamFlags = {
searchFeedbackOptOut?: boolean;
researchBeta?: boolean;
highlightsBeta?: boolean;
+ enrichBeta?: boolean;
// routes the team's new queue work to the FoundationDB backend
nuqFdb?: boolean;
} | null;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index f51a10a73..2111b486d 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1548,6 +1548,7 @@ export type TeamFlags = {
researchBeta?: boolean;
highlightsBeta?: boolean;
menuBeta?: boolean;
+ enrichBeta?: boolean;
} | null;
interface RequestWithMaybeACUC<
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 364659d3e..1163a09e7 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -38,6 +38,7 @@ import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forci
import responseTime from "response-time";
import { shutdownWebhookQueue } from "./services/webhook";
import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";
+import { warmDataLayerCapabilities } from "./lib/data-layer";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
@@ -119,8 +120,9 @@ async function startServer(port = DEFAULT_PORT) {
try {
await initializeBlocklist();
initializeEngineForcing();
+ await warmDataLayerCapabilities();
} catch (error) {
- logger.error("Failed to initialize blocklist and engine forcing", {
+ logger.error("Failed to initialize API startup dependencies", {
error,
});
throw error;
diff --git a/apps/api/src/lib/data-layer.test.ts b/apps/api/src/lib/data-layer.test.ts
new file mode 100644
index 000000000..9a0baaf20
--- /dev/null
+++ b/apps/api/src/lib/data-layer.test.ts
@@ -0,0 +1,231 @@
+import { fetch } from "undici";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { config } from "../config";
+import {
+ canUseDataLayerForRequest,
+ clearDataLayerCapabilitiesForTest,
+ getDataLayerRequestLogContext,
+ getDataLayerResponseLogContext,
+ getDataLayerSuccessCredits,
+ isDataLayerSupportedUrl,
+ isSuccessfulDataLayerStatusCode,
+ isSupportedDataLayerFormatRequest,
+ setDataLayerCapabilitiesForTest,
+ warmDataLayerCapabilities,
+} from "./data-layer";
+
+vi.mock("undici", () => ({
+ fetch: vi.fn(),
+}));
+
+const originalConfig = {
+ FIRE_ENGINE_BETA_URL: config.FIRE_ENGINE_BETA_URL,
+};
+
+describe("data layer routing", () => {
+ beforeEach(() => {
+ vi.mocked(fetch).mockReset();
+ config.FIRE_ENGINE_BETA_URL = "https://fire-engine.example";
+ setDataLayerCapabilitiesForTest({
+ domains: ["profiles.example"],
+ baseDomains: ["network.example"],
+ });
+ });
+
+ afterEach(() => {
+ config.FIRE_ENGINE_BETA_URL = originalConfig.FIRE_ENGINE_BETA_URL;
+ clearDataLayerCapabilitiesForTest();
+ });
+
+ it("detects URLs using Fire Engine capabilities", async () => {
+ await expect(
+ isDataLayerSupportedUrl(
+ "https://profiles.example/person/example-person/details/experience/?trk=foo",
+ ),
+ ).resolves.toBe(true);
+ await expect(
+ isDataLayerSupportedUrl("https://www.network.example/any/path"),
+ ).resolves.toBe(true);
+ await expect(
+ isDataLayerSupportedUrl("https://other.example/person/example-person"),
+ ).resolves.toBe(false);
+ await expect(isDataLayerSupportedUrl("not a url")).resolves.toBe(false);
+ });
+
+ it("caches failed Fire Engine capabilities lookups briefly", async () => {
+ clearDataLayerCapabilitiesForTest();
+ vi.mocked(fetch).mockResolvedValue({
+ ok: false,
+ status: 503,
+ } as Awaited<ReturnType<typeof fetch>>);
+
+ await expect(
+ isDataLayerSupportedUrl("https://profiles.example/person/example-person"),
+ ).resolves.toBe(false);
+ await expect(
+ isDataLayerSupportedUrl("https://profiles.example/person/example-person"),
+ ).resolves.toBe(false);
+
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it("warms capabilities before request-time URL checks", async () => {
+ clearDataLayerCapabilitiesForTest();
+ vi.mocked(fetch).mockResolvedValue({
+ ok: true,
+ status: 200,
+ json: async () => ({
+ ttlSeconds: 300,
+ domains: ["profiles.example"],
+ }),
+ } as Awaited<ReturnType<typeof fetch>>);
+
+ await warmDataLayerCapabilities();
+ await expect(
+ isDataLayerSupportedUrl("https://profiles.example/person/example-person"),
+ ).resolves.toBe(true);
+
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it("builds a compact request log context", () => {
+ expect(
+ getDataLayerRequestLogContext(
+ "https://profiles.example/person/example-person/details/experience/?trk=foo",
+ ),
+ ).toEqual({
+ url: "https://profiles.example/person/example-person/details/experience/?trk=foo",
+ host: "profiles.example",
+ pathPrefix: "person",
+ });
+
+ expect(getDataLayerRequestLogContext("not a url")).toBeUndefined();
+ });
+
+ it("extracts response cache metadata for logs", () => {
+ expect(
+ getDataLayerResponseLogContext({
+ cacheState: "hit",
+ cachedAt: "2026-06-21T10:00:00.000Z",
+ cacheAgeMs: 1000,
+ request_id: "req_123",
+ extra: "ignored",
+ }),
+ ).toEqual({
+ cacheState: "hit",
+ cachedAt: "2026-06-21T10:00:00.000Z",
+ cacheAgeMs: 1000,
+ providerRequestId: "req_123",
+ });
+
+ expect(getDataLayerResponseLogContext(null)).toEqual({});
+ });
+
+ it("accepts only formats that Fire Engine can return directly", () => {
+ expect(isSupportedDataLayerFormatRequest(undefined)).toBe(true);
+ expect(isSupportedDataLayerFormatRequest([{ type: "markdown" }])).toBe(
+ true,
+ );
+ expect(isSupportedDataLayerFormatRequest(["json"])).toBe(true);
+ expect(
+ isSupportedDataLayerFormatRequest([
+ { type: "markdown" },
+ { type: "json" },
+ ]),
+ ).toBe(true);
+ expect(isSupportedDataLayerFormatRequest([{ type: "html" }])).toBe(false);
+ expect(isSupportedDataLayerFormatRequest([])).toBe(false);
+ });
+
+ it("allows eligible requests through the blocklist when the team flag is enabled", async () => {
+ await expect(
+ canUseDataLayerForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: { enrichBeta: true },
+ }),
+ ).resolves.toBe(true);
+
+ await expect(
+ canUseDataLayerForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "json" }],
+ actions: [{ type: "wait" }],
+ flags: { enrichBeta: true },
+ }),
+ ).resolves.toBe(false);
+
+ await expect(
+ canUseDataLayerForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "json" }],
+ zeroDataRetention: true,
+ flags: { enrichBeta: true },
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("does not bypass unless the team flag is enabled", async () => {
+ await expect(
+ canUseDataLayerForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ }),
+ ).resolves.toBe(false);
+
+ await expect(
+ canUseDataLayerForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: { enrichBeta: false },
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("does not bypass unless Fire Engine is configured", async () => {
+ config.FIRE_ENGINE_BETA_URL = undefined;
+
+ await expect(
+ canUseDataLayerForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: { enrichBeta: true },
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("returns 15 credits only for successful handled responses", () => {
+ expect(isSuccessfulDataLayerStatusCode(200)).toBe(true);
+ expect(isSuccessfulDataLayerStatusCode(204)).toBe(true);
+ expect(isSuccessfulDataLayerStatusCode(304)).toBe(true);
+ expect(isSuccessfulDataLayerStatusCode(404)).toBe(false);
+
+ expect(
+ getDataLayerSuccessCredits({
+ dataLayer: { handled: true, integrationId: "example" },
+ statusCode: 200,
+ }),
+ ).toBe(15);
+
+ expect(
+ getDataLayerSuccessCredits({
+ dataLayer: { handled: true, integrationId: "example" },
+ statusCode: 304,
+ }),
+ ).toBe(15);
+
+ expect(
+ getDataLayerSuccessCredits({
+ dataLayer: { handled: true, integrationId: "example" },
+ statusCode: 404,
+ }),
+ ).toBeNull();
+
+ expect(
+ getDataLayerSuccessCredits({
+ statusCode: 200,
+ }),
+ ).toBeNull();
+ });
+});
diff --git a/apps/api/src/lib/data-layer.ts b/apps/api/src/lib/data-layer.ts
new file mode 100644
index 000000000..30a7e010c
--- /dev/null
+++ b/apps/api/src/lib/data-layer.ts
@@ -0,0 +1,341 @@
+import { fetch } from "undici";
+import { z } from "zod";
+
+import { config } from "../config";
+import type { FormatObject } from "../controllers/v2/types";
+import { logger as rootLogger } from "./logger";
+
+type RouteInput = {
+ url: string;
+ formats?: FormatObject[] | unknown[];
+ actions?: unknown[];
+ headers?: Record<string, unknown>;
+ waitFor?: number;
+ mobile?: boolean;
+ location?: unknown;
+ proxy?: unknown;
+ blockAds?: boolean;
+ zeroDataRetention?: boolean;
+ lockdown?: boolean;
+ flags?: { enrichBeta?: boolean } | null;
+};
+
+export type DataLayerScrapeMetadata = {
+ handled: true;
+ integrationId?: string;
+};
+
+const SUPPORTED_FORMATS = new Set(["markdown", "json", "deterministicJson"]);
+const DATA_LAYER_SUCCESS_CREDITS = 15;
+
+const DATA_LAYER_CAPABILITIES_PATH = "/v1/data-layer/capabilities";
+const DATA_LAYER_CAPABILITIES_TIMEOUT_MS = 2_000;
+const DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS = 30_000;
+
+const dataLayerCapabilitiesSchema = z
+ .object({
+ version: z.number().optional(),
+ ttlSeconds: z.number().positive().optional(),
+ domains: z.string().array().optional(),
+ baseDomains: z.string().array().optional(),
+ })
+ .passthrough();
+
+type DataLayerCapabilities = {
+ domains: Set<string>;
+ baseDomains: Set<string>;
+ ttlMs: number;
+};
+
+let cachedCapabilities:
+ | {
+ expiresAt: number;
+ value: DataLayerCapabilities | null;
+ }
+ | undefined;
+let capabilitiesRequest: Promise<DataLayerCapabilities | null> | undefined;
+
+function normalizeHost(host: string): string {
+ return host.trim().toLowerCase().replace(/\.$/, "");
+}
+
+function normalizeCapabilities(
+ raw: z.infer<typeof dataLayerCapabilitiesSchema>,
+): DataLayerCapabilities {
+ const ttlMs =
+ typeof raw.ttlSeconds === "number" && Number.isFinite(raw.ttlSeconds)
+ ? raw.ttlSeconds * 1000
+ : DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS;
+
+ return {
+ domains: new Set((raw.domains ?? []).map(normalizeHost)),
+ baseDomains: new Set((raw.baseDomains ?? []).map(normalizeHost)),
+ ttlMs: Math.max(1_000, ttlMs),
+ };
+}
+
+function getFireEngineDataLayerUrl(): string | null {
+ if (!config.FIRE_ENGINE_BETA_URL) {
+ return null;
+ }
+
+ return `${config.FIRE_ENGINE_BETA_URL.replace(/\/+$/, "")}${DATA_LAYER_CAPABILITIES_PATH}`;
+}
+
+async function fetchDataLayerCapabilities(): Promise<DataLayerCapabilities | null> {
+ const url = getFireEngineDataLayerUrl();
+ if (!url) {
+ return null;
+ }
+
+ try {
+ const response = await fetch(url, {
+ method: "GET",
+ signal: AbortSignal.timeout(DATA_LAYER_CAPABILITIES_TIMEOUT_MS),
+ });
+
+ if (!response.ok) {
+ rootLogger.warn("Data layer capabilities request failed", {
+ statusCode: response.status,
+ });
+ return null;
+ }
+
+ const parsed = dataLayerCapabilitiesSchema.parse(await response.json());
+ return normalizeCapabilities(parsed);
+ } catch (error) {
+ rootLogger.warn("Data layer capabilities request errored", { error });
+ return null;
+ }
+}
+
+async function getDataLayerCapabilities(): Promise<DataLayerCapabilities | null> {
+ if (cachedCapabilities && cachedCapabilities.expiresAt > Date.now()) {
+ return cachedCapabilities.value;
+ }
+
+ if (!capabilitiesRequest) {
+ capabilitiesRequest = fetchDataLayerCapabilities().finally(() => {
+ capabilitiesRequest = undefined;
+ });
+ }
+
+ const capabilities = await capabilitiesRequest;
+ cachedCapabilities = {
+ value: capabilities,
+ expiresAt:
+ Date.now() +
+ (capabilities?.ttlMs ?? DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS),
+ };
+
+ return capabilities;
+}
+
+export async function warmDataLayerCapabilities(): Promise<void> {
+ if (!config.FIRE_ENGINE_BETA_URL) {
+ return;
+ }
+
+ const capabilities = await getDataLayerCapabilities();
+ rootLogger.info("Data layer capabilities warmup completed", {
+ available: capabilities !== null,
+ domains: capabilities?.domains.size ?? 0,
+ baseDomains: capabilities?.baseDomains.size ?? 0,
+ });
+}
+
+function dataLayerCapabilitiesMatchUrl(
+ capabilities: DataLayerCapabilities,
+ inputUrl: string,
+): boolean {
+ let parsed: URL;
+ try {
+ parsed = new URL(inputUrl);
+ } catch {
+ return false;
+ }
+
+ const host = normalizeHost(parsed.hostname);
+ if (capabilities.domains.has(host)) {
+ return true;
+ }
+
+ for (const baseDomain of capabilities.baseDomains) {
+ if (host === baseDomain || host.endsWith(`.${baseDomain}`)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+export async function isDataLayerSupportedUrl(
+ inputUrl: string,
+): Promise<boolean> {
+ const capabilities = await getDataLayerCapabilities();
+ return (
+ capabilities !== null &&
+ dataLayerCapabilitiesMatchUrl(capabilities, inputUrl)
+ );
+}
+
+export function getDataLayerRequestLogContext(inputUrl: string):
+ | {
+ url: string;
+ host: string;
+ pathPrefix: string | null;
+ }
+ | undefined {
+ let parsed: URL;
+ try {
+ parsed = new URL(inputUrl);
+ } catch {
+ return undefined;
+ }
+
+ return {
+ url: parsed.href,
+ host: parsed.hostname.toLowerCase(),
+ pathPrefix:
+ parsed.pathname
+ .split("/")
+ .map(part => part.trim())
+ .filter(part => part.length > 0)[0] ?? null,
+ };
+}
+
+export function getDataLayerResponseLogContext(meta: unknown): {
+ cacheState?: string;
+ cachedAt?: string;
+ cacheAgeMs?: number;
+ providerRequestId?: string;
+} {
+ if (typeof meta !== "object" || meta === null) {
+ return {};
+ }
+
+ const record = meta as Record<string, unknown>;
+ const requestId = record.request_id ?? record.requestId;
+
+ return {
+ ...(typeof record.cacheState === "string"
+ ? { cacheState: record.cacheState }
+ : {}),
+ ...(typeof record.cachedAt === "string"
+ ? { cachedAt: record.cachedAt }
+ : {}),
+ ...(typeof record.cacheAgeMs === "number"
+ ? { cacheAgeMs: record.cacheAgeMs }
+ : {}),
+ ...(typeof requestId === "string" ? { providerRequestId: requestId } : {}),
+ };
+}
+
+export function isSuccessfulDataLayerStatusCode(statusCode: number): boolean {
+ return (statusCode >= 200 && statusCode < 300) || statusCode === 304;
+}
+
+export function isSupportedDataLayerFormatRequest(
+ formats?: FormatObject[] | unknown[],
+): boolean {
+ if (formats === undefined) {
+ return true;
+ }
+
+ if (!Array.isArray(formats) || formats.length === 0) {
+ return false;
+ }
+
+ return formats.every(format => {
+ const type =
+ typeof format === "string"
+ ? format
+ : typeof format === "object" && format !== null && "type" in format
+ ? (format as { type?: unknown }).type
+ : undefined;
+
+ return typeof type === "string" && SUPPORTED_FORMATS.has(type);
+ });
+}
+
+export async function canUseDataLayerForRequest(
+ input: RouteInput,
+): Promise<boolean> {
+ if (input.flags?.enrichBeta !== true) {
+ return false;
+ }
+
+ if (!config.FIRE_ENGINE_BETA_URL) {
+ return false;
+ }
+
+ if (!input.url) {
+ return false;
+ }
+
+ if (input.zeroDataRetention || input.lockdown) {
+ return false;
+ }
+
+ if (Array.isArray(input.actions) && input.actions.length > 0) {
+ return false;
+ }
+
+ if (input.headers && Object.keys(input.headers).length > 0) {
+ return false;
+ }
+
+ if (input.waitFor !== undefined && input.waitFor !== 0) {
+ return false;
+ }
+
+ if (input.mobile || input.location || input.blockAds === false) {
+ return false;
+ }
+
+ if (input.proxy === "stealth" || input.proxy === "enhanced") {
+ return false;
+ }
+
+ if (!isSupportedDataLayerFormatRequest(input.formats)) {
+ return false;
+ }
+
+ return isDataLayerSupportedUrl(input.url);
+}
+
+export function getDataLayerSuccessCredits(input: {
+ dataLayer?: DataLayerScrapeMetadata;
+ statusCode?: number | null;
+}): number | null {
+ if (input.dataLayer?.handled !== true) {
+ return null;
+ }
+
+ const statusCode = input.statusCode;
+ if (
+ statusCode === undefined ||
+ statusCode === null ||
+ !isSuccessfulDataLayerStatusCode(statusCode)
+ ) {
+ return null;
+ }
+
+ return DATA_LAYER_SUCCESS_CREDITS;
+}
+
+export function setDataLayerCapabilitiesForTest(input: {
+ domains?: string[];
+ baseDomains?: string[];
+ ttlSeconds?: number;
+}) {
+ cachedCapabilities = {
+ value: normalizeCapabilities(input),
+ expiresAt: Date.now() + (input.ttlSeconds ?? 300) * 1000,
+ };
+}
+
+export function clearDataLayerCapabilitiesForTest() {
+ cachedCapabilities = undefined;
+ capabilitiesRequest = undefined;
+}
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index 70361b069..cdf49cb52 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -1,6 +1,33 @@
import { calculateCreditsToBeBilled } from "./scrape-billing";
describe("calculateCreditsToBeBilled", () => {
+ it("bills handled data layer successes at 15 credits", async () => {
+ const credits = await calculateCreditsToBeBilled(
+ {
+ formats: [{ type: "markdown" }],
+ } as any,
+ {
+ teamId: "team-id",
+ },
+ {
+ metadata: {
+ statusCode: 200,
+ url: "https://profiles.example/in/example-person",
+ proxyUsed: "basic",
+ },
+ } as any,
+ {
+ totalCost: 0,
+ } as any,
+ {} as any,
+ undefined,
+ undefined,
+ { handled: true },
+ );
+
+ expect(credits).toBe(15);
+ });
+
it("bills X/Twitter scrapes at 30 credits", async () => {
const credits = await calculateCreditsToBeBilled(
{
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 2de00540a..23b334c84 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -10,6 +10,10 @@ import { hasFormatOfType } from "./format-utils";
import { TransportableError } from "./error";
import { FeatureFlag } from "../scraper/scrapeURL/engines";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
+import {
+ DataLayerScrapeMetadata,
+ getDataLayerSuccessCredits,
+} from "./data-layer";
const creditsPerPDFPage = 1;
const stealthProxyCostBonus = 4;
@@ -28,6 +32,7 @@ export async function calculateCreditsToBeBilled(
flags: TeamFlags,
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
+ dataLayer?: DataLayerScrapeMetadata,
) {
const costTrackingJSON: ReturnType<typeof CostTracking.prototype.toJSON> =
costTracking instanceof CostTracking ? costTracking.toJSON() : costTracking;
@@ -55,6 +60,14 @@ export async function calculateCreditsToBeBilled(
return creditsToBeBilled;
}
+ const dataLayerCredits = getDataLayerSuccessCredits({
+ dataLayer,
+ statusCode: document.metadata?.statusCode,
+ });
+ if (dataLayerCredits !== null) {
+ return dataLayerCredits;
+ }
+
let creditsToBeBilled = 1; // Assuming 1 credit per document
if (options.lockdown) {
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index f3c02ab1d..e5e7700ab 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -29,6 +29,8 @@ import {
CREDITS_FEATURE_ID,
} from "../services/autumn/autumn.service";
import { getTeamBalance } from "../services/autumn/usage";
+import { canUseDataLayerForRequest } from "../lib/data-layer";
+import { getScrapeZDR } from "../lib/zdr-helpers";
export function checkCreditsMiddleware(
_minimum?: number,
@@ -291,21 +293,45 @@ export function blocklistMiddleware(
res: Response,
next: NextFunction,
) {
- if (
- typeof req.body.url === "string" &&
- isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {
- team_id: req.acuc?.team_id ?? null,
- origin: typeof req.body.origin === "string" ? req.body.origin : null,
- })
- ) {
- if (!res.headersSent) {
- return res.status(403).json({
- success: false,
- error: UNSUPPORTED_SITE_MESSAGE,
- });
+ (async () => {
+ const zeroDataRetention =
+ getScrapeZDR(req.acuc?.flags) === "forced" ||
+ req.body?.zeroDataRetention === true ||
+ req.body?.lockdown === true;
+ const canUseDataLayer =
+ typeof req.body.url === "string" &&
+ (await canUseDataLayerForRequest({
+ url: req.body.url,
+ formats: req.body.formats,
+ actions: req.body.actions,
+ headers: req.body.headers,
+ waitFor: req.body.waitFor,
+ mobile: req.body.mobile,
+ location: req.body.location,
+ proxy: req.body.proxy,
+ blockAds: req.body.blockAds,
+ zeroDataRetention,
+ lockdown: req.body.lockdown,
+ flags: req.acuc?.flags ?? null,
+ }));
+
+ if (
+ typeof req.body.url === "string" &&
+ !canUseDataLayer &&
+ isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {
+ team_id: req.acuc?.team_id ?? null,
+ origin: typeof req.body.origin === "string" ? req.body.origin : null,
+ })
+ ) {
+ if (!res.headersSent) {
+ return res.status(403).json({
+ success: false,
+ error: UNSUPPORTED_SITE_MESSAGE,
+ });
+ }
}
- }
- next();
+ next();
+ })().catch(err => next(err));
}
export function countryCheck(
diff --git a/apps/api/src/scraper/scrapeURL/engines/data-layer.ts b/apps/api/src/scraper/scrapeURL/engines/data-layer.ts
new file mode 100644
index 000000000..53275c2e8
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/data-layer.ts
@@ -0,0 +1,183 @@
+import { z } from "zod";
+
+import { Meta } from "..";
+import { EngineScrapeResult } from ".";
+import {
+ getDataLayerRequestLogContext,
+ getDataLayerResponseLogContext,
+ isSuccessfulDataLayerStatusCode,
+} from "../../../lib/data-layer";
+import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";
+import { robustFetch } from "../lib/fetch";
+import { EngineError } from "../error";
+import { fireEngineURL } from "./fire-engine/scrape";
+
+const dataLayerResultSchema = z
+ .object({
+ content: z.string().optional(),
+ json: z.unknown().optional(),
+ url: z.string().optional(),
+ pageStatusCode: z.number().optional(),
+ pageError: z.string().optional(),
+ responseHeaders: z.record(z.string(), z.string()).optional(),
+ meta: z.record(z.string(), z.unknown()).optional(),
+ error: z.string().optional(),
+ })
+ .passthrough();
+
+const dataLayerScrapeResponseSchema = z.union([
+ z
+ .object({
+ handled: z.literal(false),
+ })
+ .passthrough(),
+ z
+ .object({
+ handled: z.literal(true),
+ statusCode: z.number(),
+ integrationId: z.string().optional(),
+ credits: z.number().optional(),
+ result: dataLayerResultSchema.optional(),
+ })
+ .passthrough(),
+]);
+
+function getContentType(headers?: Record<string, string>): string {
+ return (
+ Object.entries(headers ?? {}).find(
+ ([key]) => key.toLowerCase() === "content-type",
+ )?.[1] ?? ""
+ );
+}
+
+export function dataLayerMaxReasonableTime(meta: Meta): number {
+ return meta.options.timeout ?? 60_000;
+}
+
+export async function scrapeURLWithDataLayer(
+ meta: Meta,
+): Promise<EngineScrapeResult> {
+ return withSpan("engine.data-layer.scrape", async span => {
+ const startTime = Date.now();
+ const url = meta.rewrittenUrl ?? meta.url;
+ const requestLogContext = getDataLayerRequestLogContext(url);
+ const logger = meta.logger.child({ method: "scrapeURLWithDataLayer" });
+
+ setSpanAttributes(span, {
+ "engine.type": "data-layer",
+ "engine.url": url,
+ "engine.team_id": meta.internalOptions.teamId,
+ });
+
+ logger.info("Data layer scrape started", {
+ ...requestLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ maxAge: meta.options.maxAge,
+ });
+
+ try {
+ const response = await robustFetch({
+ url: `${fireEngineURL}/v1/data-layer/scrape`,
+ method: "POST",
+ body: {
+ url,
+ formats: meta.options.formats,
+ maxAge: meta.options.maxAge,
+ zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ source: "firecrawl",
+ },
+ logger: logger.child({ method: "dataLayerScrape/robustFetch" }),
+ tryCount: 2,
+ ignoreFailureStatus: true,
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ schema: dataLayerScrapeResponseSchema,
+ });
+
+ if (!response.handled) {
+ logger.info("Data layer scrape was not handled", {
+ ...requestLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ durationMs: Date.now() - startTime,
+ });
+ throw new EngineError("Data layer did not handle URL");
+ }
+
+ const responseLogContext = getDataLayerResponseLogContext(
+ response.result?.meta,
+ );
+ const pageStatusCode =
+ response.result?.pageStatusCode ?? response.statusCode;
+
+ if (
+ !isSuccessfulDataLayerStatusCode(response.statusCode) ||
+ !isSuccessfulDataLayerStatusCode(pageStatusCode) ||
+ !response.result
+ ) {
+ logger.warn("Data layer scrape failed", {
+ ...requestLogContext,
+ ...responseLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ integrationId: response.integrationId,
+ statusCode: response.statusCode,
+ pageStatusCode,
+ durationMs: Date.now() - startTime,
+ });
+ throw new EngineError("Data layer request failed");
+ }
+
+ const contentType = getContentType(response.result.responseHeaders);
+
+ logger.info("Data layer scrape completed", {
+ ...requestLogContext,
+ ...responseLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ integrationId: response.integrationId,
+ statusCode: response.statusCode,
+ durationMs: Date.now() - startTime,
+ });
+
+ setSpanAttributes(span, {
+ "data-layer.integration_id": response.integrationId,
+ "data-layer.status_code": response.statusCode,
+ "data-layer.page_status_code": pageStatusCode,
+ "data-layer.cache_state": responseLogContext.cacheState,
+ "data-layer.cache_age_ms": responseLogContext.cacheAgeMs,
+ "data-layer.duration_ms": Date.now() - startTime,
+ });
+
+ return {
+ url: response.result.url ?? url,
+ html: response.result.content ?? "",
+ markdown: contentType.includes("text/markdown")
+ ? response.result.content
+ : undefined,
+ json: response.result.json,
+ error: response.result.pageError,
+ statusCode: pageStatusCode,
+ contentType,
+ proxyUsed: "basic",
+ dataLayer: {
+ handled: true,
+ integrationId: response.integrationId,
+ },
+ };
+ } catch (error) {
+ logger.warn("Data layer scrape errored", {
+ ...requestLogContext,
+ scrapeId: meta.id,
+ teamId: meta.internalOptions.teamId,
+ durationMs: Date.now() - startTime,
+ errorMessage: error instanceof Error ? error.message : String(error),
+ error,
+ });
+ throw error;
+ }
+ });
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 7ff643f9c..f27516283 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -33,6 +33,7 @@ const successSchema = z.object({
// timeTaken: z.number(),
content: z.string(),
+ json: z.unknown().optional(),
url: z.string().optional(),
pageStatusCode: z.number(),
@@ -40,6 +41,7 @@ const successSchema = z.object({
// TODO: this needs to be non-optional, might need fixes on f-e side to ensure reliability
responseHeaders: z.record(z.string(), z.string()).optional(),
+ meta: z.record(z.string(), z.unknown()).optional(),
// timeTakenCookie: z.number().optional(),
// timeTakenRequest: z.number().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 24fe4b81f..c967e4014 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -362,6 +362,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
timeout: meta.abort.scrapeTimeout() ?? 300000,
disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,
mobileProxy: meta.featureFlags.has("stealthProxy"),
+ maxAge: meta.options.maxAge,
saveScrapeResultToGCS:
!meta.internalOptions.zeroDataRetention &&
meta.internalOptions.saveScrapeResultToGCS,
@@ -437,18 +438,23 @@ export async function scrapeURLWithFireEngineChromeCDP(
const audioCookies = (response.actionResults ?? [])
.filter(x => x.type === "getCookies")
.flatMap(x => x.result.cookies);
+ const contentType =
+ (Object.entries(response.responseHeaders ?? {}).find(
+ x => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined;
return {
url: response.url ?? meta.url,
html: response.content,
+ markdown: contentType?.includes("text/markdown")
+ ? response.content
+ : undefined,
+ json: response.json,
error: response.pageError,
statusCode: response.pageStatusCode,
- contentType:
- (Object.entries(response.responseHeaders ?? {}).find(
- x => x[0].toLowerCase() === "content-type",
- ) ?? [])[1] ?? undefined,
+ contentType,
screenshot,
...(actions.length > 0
@@ -499,6 +505,7 @@ export async function scrapeURLWithFireEngineTLSClient(
mobileProxy: meta.featureFlags.has("stealthProxy"),
timeout: meta.abort.scrapeTimeout() ?? 300000,
+ maxAge: meta.options.maxAge,
saveScrapeResultToGCS:
!meta.internalOptions.zeroDataRetention &&
meta.internalOptions.saveScrapeResultToGCS,
@@ -522,18 +529,23 @@ export async function scrapeURLWithFireEngineTLSClient(
sourceURL: meta.url,
});
}
+ const contentType =
+ (Object.entries(response.responseHeaders ?? {}).find(
+ x => x[0].toLowerCase() === "content-type",
+ ) ?? [])[1] ?? undefined;
return {
url: response.url ?? meta.url,
html: response.content,
+ markdown: contentType?.includes("text/markdown")
+ ? response.content
+ : undefined,
+ json: response.json,
error: response.pageError,
statusCode: response.pageStatusCode,
- contentType:
- (Object.entries(response.responseHeaders ?? {}).find(
- x => x[0].toLowerCase() === "content-type",
- ) ?? [])[1] ?? undefined,
+ contentType,
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
timezone: response.timezone,
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 70a703813..ebf59dbda 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -51,6 +51,7 @@ export type FireEngineScrapeRequestCommon = {
mobileProxy?: boolean; // leave it undefined if user doesn't specify
timeout?: number;
+ maxAge?: number;
saveScrapeResultToGCS?: boolean;
zeroDataRetention?: boolean;
};
@@ -76,6 +77,7 @@ const successSchema = z.object({
timeTaken: z.number(),
content: z.string(),
+ json: z.unknown().optional(),
url: z.string().optional(),
pageStatusCode: z.number(),
@@ -83,6 +85,7 @@ const successSchema = z.object({
// TODO: this needs to be non-optional, might need fixes on f-e side to ensure reliability
responseHeaders: z.record(z.string(), z.string()).optional(),
+ meta: z.record(z.string(), z.unknown()).optional(),
// timeTakenCookie: z.number().optional(),
// timeTakenRequest: z.number().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 89ace87f3..c487efee7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -7,6 +7,10 @@ import {
scrapeURLWithFireEngineChromeCDP,
scrapeURLWithFireEngineTLSClient,
} from "./fire-engine";
+import {
+ dataLayerMaxReasonableTime,
+ scrapeURLWithDataLayer,
+} from "./data-layer";
import { pdfMaxReasonableTime, scrapePDF } from "./pdf";
import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";
import {
@@ -30,8 +34,13 @@ import { getPDFMaxPages } from "../../../controllers/v2/types";
import type { PdfMetadata } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
import { BrandingNotSupportedError } from "../error";
+import {
+ canUseDataLayerForRequest,
+ type DataLayerScrapeMetadata,
+} from "../../../lib/data-layer";
export type Engine =
+ | "data-layer"
| "fire-engine;chrome-cdp"
| "fire-engine(retry);chrome-cdp"
| "fire-engine;chrome-cdp;stealth"
@@ -131,6 +140,7 @@ export type EngineScrapeResult = {
html: string;
markdown?: string;
+ json?: unknown;
statusCode: number;
error?: string;
@@ -161,11 +171,13 @@ export type EngineScrapeResult = {
proxyUsed: "basic" | "stealth";
timezone?: string;
+ dataLayer?: DataLayerScrapeMetadata;
};
const engineHandlers: {
[E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;
} = {
+ "data-layer": scrapeURLWithDataLayer,
index: scrapeURLWithIndex,
"index;documents": scrapeURLWithIndex,
"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,
@@ -185,6 +197,7 @@ const engineHandlers: {
const engineMRTs: {
[E in Engine]: (meta: Meta) => number;
} = {
+ "data-layer": dataLayerMaxReasonableTime,
index: indexMaxReasonableTime,
"index;documents": indexMaxReasonableTime,
"fire-engine;chrome-cdp": meta =>
@@ -217,6 +230,27 @@ const engineOptions: {
quality: number;
};
} = {
+ "data-layer": {
+ features: {
+ actions: false,
+ waitFor: false,
+ screenshot: false,
+ "screenshot@fullScreen": false,
+ pdf: false,
+ document: false,
+ audio: false,
+ video: false,
+ atsv: false,
+ location: false,
+ mobile: false,
+ skipTlsVerification: true,
+ useFastMode: true,
+ stealthProxy: false,
+ branding: false,
+ disableAdblock: false,
+ },
+ quality: 2000,
+ },
index: {
features: {
actions: false,
@@ -546,6 +580,31 @@ export async function buildFallbackList(meta: Meta): Promise<
unsupportedFeatures: Set<FeatureFlag>;
}[]
> {
+ if (
+ !meta.internalOptions.agentIndexOnly &&
+ (await canUseDataLayerForRequest({
+ url: meta.rewrittenUrl ?? meta.url,
+ formats: meta.options.formats,
+ actions: meta.options.actions,
+ headers: meta.options.headers,
+ waitFor: meta.options.waitFor,
+ mobile: meta.options.mobile,
+ location: meta.options.location,
+ proxy: meta.options.proxy,
+ blockAds: meta.options.blockAds,
+ zeroDataRetention: meta.internalOptions.zeroDataRetention,
+ lockdown: meta.options.lockdown,
+ flags: meta.internalOptions.teamFlags ?? null,
+ }))
+ ) {
+ return [
+ {
+ engine: "data-layer",
+ unsupportedFeatures: new Set(),
+ },
+ ];
+ }
+
const shouldPrioritizeTlsClient = meta.options.__experimental_engpicker
? (await queryEngpickerVerdict(
meta.options.__experimental_omceDomain ?? new URL(meta.url).hostname,
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index d35566936..784ab1e63 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -105,6 +105,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
meta.rewrittenUrl ??
meta.url,
html: document.rawHtml!,
+ json: document.json,
statusCode: document.metadata.statusCode,
error: document.metadata.error,
screenshot: document.screenshot,
@@ -547,6 +548,7 @@ export async function scrapeURLWithIndex(
return {
url: doc.url,
html: doc.html,
+ json: doc.json,
statusCode: doc.statusCode,
error: doc.error,
screenshot: doc.screenshot,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 0c9fc5aac..207747925 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -87,12 +87,14 @@ import { writeFile } from "node:fs/promises";
import path from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
+import type { DataLayerScrapeMetadata } from "../../lib/data-layer";
export type ScrapeUrlResponse =
| {
success: true;
document: Document;
unsupportedFeatures?: Set<FeatureFlag>;
+ dataLayer?: DataLayerScrapeMetadata;
}
| {
success: false;
@@ -552,6 +554,8 @@ async function scrapeURLLoopIter(
},
);
checkMarkdown = engineResult.html?.trim() ?? "";
+ } else if (engineResult.markdown?.trim()) {
+ checkMarkdown = engineResult.markdown.trim();
} else {
const requestId = meta.id || meta.internalOptions.crawlId;
const zeroDataRetention = meta.internalOptions.zeroDataRetention;
@@ -978,6 +982,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
let document: Document = {
markdown: engineResult.markdown,
rawHtml: engineResult.html,
+ json: engineResult.json,
screenshot: engineResult.screenshot,
actions: engineResult.actions,
branding: engineResult.branding,
@@ -1038,6 +1043,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
success: true,
document,
unsupportedFeatures: result.unsupportedFeatures,
+ dataLayer: engineResult.dataLayer,
};
});
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.test.ts b/apps/api/src/scraper/scrapeURL/transformers/index.test.ts
new file mode 100644
index 000000000..1e87844b0
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.test.ts
@@ -0,0 +1,104 @@
+import type { MockedFunction } from "vitest";
+import { executeTransformers } from ".";
+import { performLLMExtract } from "./llmExtract";
+
+vi.mock("../../../services/index", () => ({
+ useIndex: false,
+ useSearchIndex: false,
+}));
+
+vi.mock("./llmExtract", async importOriginal => {
+ const actual = await importOriginal<typeof import("./llmExtract")>();
+
+ return {
+ ...actual,
+ performLLMExtract: vi.fn(actual.performLLMExtract),
+ };
+});
+
+const mockedPerformLLMExtract = performLLMExtract as MockedFunction<
+ typeof performLLMExtract
+>;
+
+function logger() {
+ const log = {
+ child: vi.fn(),
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ };
+ log.child.mockReturnValue(log);
+ return log;
+}
+
+describe("executeTransformers", () => {
+ beforeEach(() => {
+ mockedPerformLLMExtract.mockClear();
+ });
+
+ it("keeps native JSON and markdown without running LLM JSON extraction", async () => {
+ const nativeJson = { id: "person-1", full_name: "Example Person" };
+ const nativeMarkdown = "# Example Person";
+
+ const document = await executeTransformers(
+ {
+ url: "https://www.linkedin.com/in/example",
+ options: {
+ formats: [{ type: "markdown" }, { type: "json" }],
+ onlyMainContent: false,
+ },
+ internalOptions: {},
+ logger: logger(),
+ } as any,
+ {
+ rawHtml:
+ "<html><head><title>Ignored</title></head><body></body></html>",
+ markdown: nativeMarkdown,
+ json: nativeJson,
+ metadata: {
+ sourceURL: "https://www.linkedin.com/in/example",
+ url: "https://www.linkedin.com/in/example",
+ statusCode: 200,
+ contentType: "text/markdown; charset=utf-8",
+ },
+ } as any,
+ );
+
+ expect(mockedPerformLLMExtract).not.toHaveBeenCalled();
+ expect(document.markdown).toBe(nativeMarkdown);
+ expect(document.json).toEqual(nativeJson);
+ });
+
+ it("maps native JSON to v1 extract compatibility field", async () => {
+ const nativeJson = { id: "person-1", full_name: "Example Person" };
+
+ const document = await executeTransformers(
+ {
+ url: "https://www.linkedin.com/in/example",
+ options: {
+ formats: [{ type: "markdown" }, { type: "json" }],
+ onlyMainContent: false,
+ },
+ internalOptions: { v1OriginalFormat: "extract" },
+ logger: logger(),
+ } as any,
+ {
+ rawHtml:
+ "<html><head><title>Ignored</title></head><body></body></html>",
+ markdown: "# Example Person",
+ json: nativeJson,
+ metadata: {
+ sourceURL: "https://www.linkedin.com/in/example",
+ url: "https://www.linkedin.com/in/example",
+ statusCode: 200,
+ contentType: "text/markdown; charset=utf-8",
+ },
+ } as any,
+ );
+
+ expect(mockedPerformLLMExtract).not.toHaveBeenCalled();
+ expect(document.extract).toEqual(nativeJson);
+ expect(document.json).toEqual(nativeJson);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index fb0146123..5d11c12ea 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -118,10 +118,10 @@ async function deriveMarkdownFromHTML(
return document;
}
- // Skip markdown derivation if a postprocessor already set it
- if (document.metadata.postprocessorsUsed?.length && document.markdown) {
+ // Skip markdown derivation if the engine or a postprocessor already set it.
+ if (document.markdown !== undefined) {
meta.logger.debug(
- "Skipping markdown derivation - postprocessor already set markdown",
+ "Skipping markdown derivation - document already has markdown",
{ postprocessorsUsed: document.metadata.postprocessorsUsed },
);
return document;
@@ -318,6 +318,30 @@ async function deriveBrandingFromActions(
return document;
}
+async function performLLMExtractUnlessNativeJson(
+ meta: Meta,
+ document: Document,
+): Promise<Document> {
+ if (
+ document.json !== undefined &&
+ hasFormatOfType(meta.options.formats, "json")
+ ) {
+ if (
+ meta.internalOptions.v1OriginalFormat === "extract" &&
+ document.extract === undefined
+ ) {
+ document.extract = document.json;
+ }
+
+ meta.logger.debug(
+ "Skipping LLM JSON extraction - document already has native JSON",
+ );
+ return document;
+ }
+
+ return performLLMExtract(meta, document);
+}
+
function coerceFieldsToFormats(meta: Meta, document: Document): Document {
const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown");
const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml");
@@ -599,7 +623,7 @@ const transformerStack: Transformer[] = [
fetchMenu,
...(useIndex ? [sendDocumentToIndex] : []),
...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time search
- performLLMExtract,
+ performLLMExtractUnlessNativeJson,
performDeterministicJson,
performSummary,
performQuery,
diff --git a/apps/api/src/services/index.test.ts b/apps/api/src/services/index.test.ts
new file mode 100644
index 000000000..ba7507e7d
--- /dev/null
+++ b/apps/api/src/services/index.test.ts
@@ -0,0 +1,105 @@
+import { vi } from "vitest";
+
+const { file, bucket, getSavedPayload, setSavedPayload } = vi.hoisted(() => {
+ let savedPayload = "";
+ const getSavedPayload = () => savedPayload;
+ const setSavedPayload = (value: string) => {
+ savedPayload = value;
+ };
+ const file = vi.fn(() => ({
+ save: vi.fn(async (payload: string) => {
+ savedPayload = payload;
+ }),
+ download: vi.fn(async () => [Buffer.from(savedPayload)]),
+ }));
+ const bucket = vi.fn(() => ({ file }));
+
+ return { file, bucket, getSavedPayload, setSavedPayload };
+});
+
+vi.mock("../config", () => ({
+ config: {
+ GCS_INDEX_BUCKET_NAME: "index-bucket",
+ GCS_MEDIA_BUCKET_NAME: "media-bucket",
+ },
+}));
+
+vi.mock("../lib/gcs-jobs", () => ({
+ storage: { bucket },
+}));
+
+vi.mock("../lib/otel-tracer", () => ({
+ withSpan: vi.fn(async (_name, fn) => fn({})),
+ setSpanAttributes: vi.fn(),
+}));
+
+vi.mock("../db/connection", () => ({
+ dbIndex: {},
+}));
+
+vi.mock("../db/rpc", () => ({
+ insertOmceJobIfNeeded: vi.fn(),
+ queryIndexAtSplitLevel: vi.fn(),
+ queryIndexAtDomainSplitLevel: vi.fn(),
+ queryOmceSignatures: vi.fn(),
+ queryEngpickerVerdict: vi.fn(),
+ queryIndexAtSplitLevelWithMeta: vi.fn(),
+ queryIndexAtDomainSplitLevelWithMeta: vi.fn(),
+ queryDomainPriority: vi.fn(),
+}));
+
+vi.mock("./redis", () => ({
+ redisEvictConnection: {},
+}));
+
+vi.mock("./index-cache", () => ({
+ deriveIndexVariantKey: vi.fn(),
+ upsertCachedIndexEntries: vi.fn(),
+ useIndexCache: false,
+}));
+
+vi.mock("../db/schema", () => ({}));
+
+import { getIndexFromGCS, saveIndexToGCS } from "./index";
+
+describe("index GCS documents", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ setSavedPayload("");
+ });
+
+ it("persists native JSON payloads with cached index documents", async () => {
+ await saveIndexToGCS("idx-1", {
+ url: "https://www.example.com/profile",
+ html: "# Example",
+ json: { id: "person-1", full_name: "Example Person" },
+ statusCode: 200,
+ contentType: "text/markdown; charset=utf-8",
+ proxyUsed: "basic",
+ });
+
+ expect(file).toHaveBeenCalledWith("idx-1.json");
+ expect(JSON.parse(getSavedPayload()).json).toEqual({
+ id: "person-1",
+ full_name: "Example Person",
+ });
+ });
+
+ it("reads native JSON payloads from cached index documents", async () => {
+ setSavedPayload(
+ JSON.stringify({
+ url: "https://www.example.com/profile",
+ html: "# Example",
+ json: { id: "person-1", full_name: "Example Person" },
+ statusCode: 200,
+ }),
+ );
+
+ const doc = await getIndexFromGCS("idx-1.json");
+
+ expect(doc?.json).toEqual({
+ id: "person-1",
+ full_name: "Example Person",
+ });
+ });
+});
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 793ce4db9..9e090d819 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -118,6 +118,7 @@ export async function saveIndexToGCS(
doc: {
url: string;
html: string;
+ json?: unknown;
statusCode: number;
error?: string;
screenshot?: string;
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index fa2366f59..0fd9eee1b 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -21,6 +21,7 @@ import Express from "express";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";
+import { warmDataLayerCapabilities } from "../lib/data-layer";
import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
import { finishCrawlSuper } from "./worker/crawl-logic";
import { getCrawl } from "../lib/crawl-redis";
@@ -465,6 +466,7 @@ app.listen(workerPort, (error?: Error) => {
});
initializeEngineForcing();
+ await warmDataLayerCapabilities();
if (config.USE_DB_AUTHENTICATION && !config.DISABLE_MONITORING) {
monitorSchedulerInterval = setInterval(() => {
diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.ts
index 63dfd6a91..b58df257f 100644
--- a/apps/api/src/services/worker/nuq-worker-runner.ts
+++ b/apps/api/src/services/worker/nuq-worker-runner.ts
@@ -7,6 +7,7 @@ import { register } from "prom-client";
import Express from "express";
import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";
+import { warmDataLayerCapabilities } from "../../lib/data-layer";
export type WorkerQueue = {
getJobToProcess(logger?: any): Promise<NuQJob<any, any> | null>;
@@ -58,6 +59,7 @@ export async function runNuqWorker(options: {
try {
await initializeBlocklist();
initializeEngineForcing();
+ await warmDataLayerCapabilities();
await options.beforeStart?.();
} catch (error) {
_logger.error("Failed to initialize NuQ worker", {
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 5b5ff8b62..6bb671d65 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -91,6 +91,7 @@ import {
recordMonitorScrapeFailure,
recordMonitorScrapeSuccess,
} from "../monitoring/results";
+import type { DataLayerScrapeMetadata } from "../../lib/data-layer";
configDotenv();
@@ -110,6 +111,7 @@ async function billScrapeJob(
flags: TeamFlags,
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
+ dataLayer?: DataLayerScrapeMetadata,
) {
let creditsToBeBilled: number | null = null;
const billing = resolveBillingMetadata({
@@ -137,6 +139,7 @@ async function billScrapeJob(
flags,
error,
unsupportedFeatures,
+ dataLayer,
);
// Charge the keyless free tier's per-IP daily credit budget unless this
@@ -555,6 +558,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
undefined,
pipeline.unsupportedFeatures,
+ pipeline.dataLayer,
);
doc.metadata.creditsUsed = credits_billed ?? undefined;
@@ -645,6 +649,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
undefined,
pipeline.unsupportedFeatures,
+ pipeline.dataLayer,
);
doc.metadata.creditsUsed = credits_billed ?? undefined;