Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(api): remove the data-layer scrape path (#4029)
Abimael Martelleb27f0b2095b
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.tsindex 594d55cde..5c2d1cac3 100644--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts@@ -6,8 +6,6 @@describe("Audio format engine routing (buildFallbackList)", () => {let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;- let clearDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").clearDataLayerCapabilitiesForTest;- let setDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").setDataLayerCapabilitiesForTest;const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;const originalIndexUrl = process.env.INDEX_DATABASE_URL;@@ -22,14 +20,6 @@ describe("Audio format engine routing (buildFallbackList)", () => {({ buildFallbackList } = await import("../../../scraper/scrapeURL/engines/index.js"));- ({- clearDataLayerCapabilitiesForTest,- setDataLayerCapabilitiesForTest,- } = await import("../../../lib/data-layer.js"));- });-- afterEach(() => {- clearDataLayerCapabilitiesForTest();});afterAll(() => {@@ -120,20 +110,4 @@ describe("Audio format engine routing (buildFallbackList)", () => {expect(engines).toContain("fire-engine;chrome-cdp");});- it("does not route agent index-only requests through the data layer", async () => {- setDataLayerCapabilitiesForTest({- domains: ["profiles.example"],- });-- const meta = buildStubMeta([]);- meta.url = "https://profiles.example/person/example-person";- meta.options.formats = [{ type: "markdown" }];- meta.internalOptions.agentIndexOnly = true;- meta.internalOptions.teamFlags = { enrichBeta: true };-- const fallback = await buildFallbackList(meta);- const engines = fallback.map(f => f.engine);-- expect(engines).toEqual(["index", "index;documents"]);- });});diff --git a/apps/api/src/lib/data-layer.test.ts b/apps/api/src/lib/data-layer.test.tsdeleted file mode 100644index d2231f4f2..000000000--- a/apps/api/src/lib/data-layer.test.ts+++ /dev/null@@ -1,325 +0,0 @@-import { fetch } from "undici";-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";--import { config } from "../config";-import {- canUseDataLayerForRequest,- clearDataLayerCapabilitiesForTest,- getDataLayerAccessForRequest,- getDataLayerRequestLogContext,- getDataLayerResponseLogContext,- getDataLayerSuccessCredits,- getThirdPartyDataTermsRequiredResponse,- isDataLayerSupportedUrl,- isSuccessfulDataLayerStatusCode,- isSupportedDataLayerFormatRequest,- setDataLayerCapabilitiesForTest,-} from "./data-layer";--vi.mock("undici", () => ({- fetch: vi.fn(),-}));--const originalConfig = {- FIRE_ENGINE_BETA_URL: config.FIRE_ENGINE_BETA_URL,-};-const ENABLED_DATA_LAYER_FLAGS = {- professionalProfileCompanyDataBeta: true,- organizationDataSourceAccess: {- provider: {- status: "enabled",- termsKey: "professional_profile_company_data",- termsVersion: "2026-07-03",- termsAcceptedAt: "2026-07-03T00:00:00.000Z",- enabledAt: "2026-07-03T00:00:00.000Z",- disabledAt: null,- disabledReason: null,- },- },-};--describe("data layer routing", () => {- beforeEach(() => {- vi.mocked(fetch).mockReset();- config.FIRE_ENGINE_BETA_URL = "https://fire-engine.example";- setDataLayerCapabilitiesForTest({- domains: ["profiles.example"],- baseDomains: ["network.example"],- });- });-- afterEach(() => {- config.FIRE_ENGINE_BETA_URL = originalConfig.FIRE_ENGINE_BETA_URL;- clearDataLayerCapabilitiesForTest();- });-- it("detects URLs using Fire Engine capabilities", async () => {- await expect(- isDataLayerSupportedUrl(- "https://profiles.example/person/example-person/details/experience/?trk=foo",- ),- ).resolves.toBe(true);- await expect(- isDataLayerSupportedUrl("https://www.network.example/any/path"),- ).resolves.toBe(true);- await expect(- isDataLayerSupportedUrl("https://other.example/person/example-person"),- ).resolves.toBe(false);- await expect(isDataLayerSupportedUrl("not a url")).resolves.toBe(false);- });-- it("uses route capabilities when Fire Engine advertises path-level support", async () => {- setDataLayerCapabilitiesForTest({- domains: ["profiles.example"],- routes: [- {- domains: ["profiles.example"],- pathPrefixes: ["/person/", "/company/"],- },- ],- });-- await expect(- isDataLayerSupportedUrl("https://profiles.example/person/example-person"),- ).resolves.toBe(true);- await expect(- isDataLayerSupportedUrl("https://profiles.example/company/example"),- ).resolves.toBe(true);- await expect(- isDataLayerSupportedUrl("https://profiles.example/jobs/example"),- ).resolves.toBe(false);- });-- it("caches failed Fire Engine capabilities lookups briefly", async () => {- clearDataLayerCapabilitiesForTest();- vi.mocked(fetch).mockResolvedValue({- ok: false,- status: 503,- } as Awaited<ReturnType<typeof fetch>>);-- await expect(- isDataLayerSupportedUrl("https://profiles.example/person/example-person"),- ).resolves.toBe(false);- await expect(- isDataLayerSupportedUrl("https://profiles.example/person/example-person"),- ).resolves.toBe(false);-- expect(fetch).toHaveBeenCalledTimes(1);- });-- it("builds a compact request log context", () => {- expect(- getDataLayerRequestLogContext(- "https://profiles.example/person/example-person/details/experience/?trk=foo",- ),- ).toEqual({- url: "https://profiles.example/person/example-person/details/experience/?trk=foo",- host: "profiles.example",- pathPrefix: "person",- });-- expect(getDataLayerRequestLogContext("not a url")).toBeUndefined();- });-- it("extracts response cache metadata for logs", () => {- expect(- getDataLayerResponseLogContext({- cacheState: "hit",- cachedAt: "2026-06-21T10:00:00.000Z",- cacheAgeMs: 1000,- request_id: "req_123",- extra: "ignored",- }),- ).toEqual({- cacheState: "hit",- cachedAt: "2026-06-21T10:00:00.000Z",- cacheAgeMs: 1000,- providerRequestId: "req_123",- });-- expect(getDataLayerResponseLogContext(null)).toEqual({});- });-- it("accepts only formats that Fire Engine can return directly", () => {- expect(isSupportedDataLayerFormatRequest(undefined)).toBe(true);- expect(isSupportedDataLayerFormatRequest([{ type: "markdown" }])).toBe(- true,- );- expect(isSupportedDataLayerFormatRequest(["json"])).toBe(true);- expect(- isSupportedDataLayerFormatRequest([- { type: "markdown" },- { type: "json" },- ]),- ).toBe(true);- expect(isSupportedDataLayerFormatRequest([{ type: "html" }])).toBe(false);- expect(isSupportedDataLayerFormatRequest([])).toBe(false);- });-- it("allows eligible requests through the blocklist when the org data source beta flag is enabled", async () => {- await expect(- canUseDataLayerForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- flags: ENABLED_DATA_LAYER_FLAGS,- }),- ).resolves.toBe(true);-- await expect(- canUseDataLayerForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "json" }],- actions: [{ type: "wait" }],- flags: ENABLED_DATA_LAYER_FLAGS,- }),- ).resolves.toBe(false);-- await expect(- canUseDataLayerForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "json" }],- zeroDataRetention: true,- flags: ENABLED_DATA_LAYER_FLAGS,- }),- ).resolves.toBe(false);- });-- it("requires current Third-Party Data terms before routing", async () => {- await expect(- getDataLayerAccessForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- flags: { professionalProfileCompanyDataBeta: true },- }),- ).resolves.toEqual({ allowed: false, termsRequired: true });-- expect(getThirdPartyDataTermsRequiredResponse()).toMatchObject({- success: false,- code: "THIRD_PARTY_DATA_TERMS_REQUIRED",- requiresAction: {- type: "accept_terms",- terms: "professional_profile_company_data",- version: "2026-07-03",- },- });-- await expect(- canUseDataLayerForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- flags: { professionalProfileCompanyDataBeta: true },- }),- ).resolves.toBe(false);-- await expect(- canUseDataLayerForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- flags: { professionalProfileCompanyDataBeta: true },- }),- ).resolves.toBe(false);- });-- it("requires current terms when organization data source access is stale", async () => {- await expect(- getDataLayerAccessForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- flags: {- professionalProfileCompanyDataBeta: true,- organizationDataSourceAccess: {- provider: {- status: "enabled",- termsKey: "professional_profile_company_data",- termsVersion: "2026-07-02",- },- },- },- }),- ).resolves.toEqual({ allowed: false, termsRequired: true });- });-- it("does not route or prompt for terms when organization data source access is disabled", async () => {- await expect(- getDataLayerAccessForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- flags: {- professionalProfileCompanyDataBeta: true,- organizationDataSourceAccess: {- provider: {- status: "disabled",- termsKey: "professional_profile_company_data",- termsVersion: "2026-07-03",- disabledAt: "2026-07-04T00:00:00.000Z",- disabledReason: "customer_disabled",- },- },- },- }),- ).resolves.toEqual({ allowed: false, termsRequired: false });- });-- it("does not bypass unless the org data source beta flag is enabled", async () => {- await expect(- canUseDataLayerForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- }),- ).resolves.toBe(false);-- await expect(- canUseDataLayerForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- flags: { professionalProfileCompanyDataBeta: false },- }),- ).resolves.toBe(false);- });-- it("does not bypass unless Fire Engine is configured", async () => {- config.FIRE_ENGINE_BETA_URL = undefined;-- await expect(- canUseDataLayerForRequest({- url: "https://profiles.example/person/example-person",- formats: [{ type: "markdown" }],- flags: ENABLED_DATA_LAYER_FLAGS,- }),- ).resolves.toBe(false);- });-- it("returns 15 credits only for successful handled responses", () => {- expect(isSuccessfulDataLayerStatusCode(200)).toBe(true);- expect(isSuccessfulDataLayerStatusCode(204)).toBe(true);- expect(isSuccessfulDataLayerStatusCode(304)).toBe(true);- expect(isSuccessfulDataLayerStatusCode(404)).toBe(false);-- expect(- getDataLayerSuccessCredits({- dataLayer: { handled: true, integrationId: "example" },- statusCode: 200,- }),- ).toBe(15);-- expect(- getDataLayerSuccessCredits({- dataLayer: { handled: true, integrationId: "example" },- statusCode: 304,- }),- ).toBe(15);-- expect(- getDataLayerSuccessCredits({- dataLayer: { handled: true, integrationId: "example" },- statusCode: 404,- }),- ).toBeNull();-- expect(- getDataLayerSuccessCredits({- statusCode: 200,- }),- ).toBeNull();- });-});diff --git a/apps/api/src/lib/data-layer.ts b/apps/api/src/lib/data-layer.tsdeleted file mode 100644index eeedce349..000000000--- a/apps/api/src/lib/data-layer.ts+++ /dev/null@@ -1,529 +0,0 @@-import { fetch } from "undici";-import { z } from "zod";--import { config } from "../config";-import type { FormatObject } from "../controllers/v2/types";-import { logger as rootLogger } from "./logger";--type OrganizationDataSourceAccessRecord = {- status?: string | null;- termsKey?: string | null;- termsVersion?: string | null;- termsAcceptedAt?: string | null;- enabledAt?: string | null;- disabledAt?: string | null;- disabledReason?: string | null;-};--type OrganizationDataSourceAccess = Record<- string,- OrganizationDataSourceAccessRecord | null | undefined->;--type RouteInput = {- url: string;- formats?: FormatObject[] | unknown[];- actions?: unknown[];- headers?: Record<string, unknown>;- waitFor?: number;- mobile?: boolean;- location?: unknown;- proxy?: unknown;- blockAds?: boolean;- zeroDataRetention?: boolean;- lockdown?: boolean;- flags?: {- professionalProfileCompanyDataBeta?: boolean;- organizationDataSourceAccess?: OrganizationDataSourceAccess | null;- } | null;-};--export type DataLayerScrapeMetadata = {- handled: true;- integrationId?: string;-};--const SUPPORTED_FORMATS = new Set(["markdown", "json", "deterministicJson"]);-const DATA_LAYER_SUCCESS_CREDITS = 15;-const PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID =- "professional_profile_company_data";-const PROFESSIONAL_PROFILE_COMPANY_DATA_BETA_FLAG =- "professionalProfileCompanyDataBeta";-const THIRD_PARTY_DATA_TERMS_VERSION = "2026-07-03";-const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";-const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE =- "An organization admin must accept the Professional Profile & Company Data terms before this URL can be processed.";--const DATA_LAYER_CAPABILITIES_PATH = "/v1/data-layer/capabilities";-const DATA_LAYER_CAPABILITIES_TIMEOUT_MS = 2_000;-const DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS = 30_000;--const dataLayerCapabilityRouteSchema = z- .object({- domains: z.string().array().optional(),- baseDomains: z.string().array().optional(),- pathPrefixes: z.string().array().optional(),- })- .passthrough();--const dataLayerCapabilitiesSchema = z- .object({- version: z.number().optional(),- ttlSeconds: z.number().positive().optional(),- domains: z.string().array().optional(),- baseDomains: z.string().array().optional(),- routes: dataLayerCapabilityRouteSchema.array().optional(),- })- .passthrough();--type DataLayerCapabilityRoute = {- domains: Set<string>;- baseDomains: Set<string>;- pathPrefixes: string[];-};--type DataLayerCapabilities = {- domains: Set<string>;- baseDomains: Set<string>;- routes: DataLayerCapabilityRoute[];- ttlMs: number;-};--let cachedCapabilities:- | {- expiresAt: number;- value: DataLayerCapabilities | null;- }- | undefined;-let capabilitiesRequest: Promise<DataLayerCapabilities | null> | undefined;--function normalizeHost(host: string): string {- return host.trim().toLowerCase().replace(/\.$/, "");-}--function normalizePathPrefix(prefix: string): string {- const trimmed = prefix.trim();- if (!trimmed) {- return "/";- }-- return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;-}--function normalizeCapabilities(- raw: z.infer<typeof dataLayerCapabilitiesSchema>,-): DataLayerCapabilities {- const ttlMs =- typeof raw.ttlSeconds === "number" && Number.isFinite(raw.ttlSeconds)- ? raw.ttlSeconds * 1000- : DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS;-- return {- domains: new Set((raw.domains ?? []).map(normalizeHost)),- baseDomains: new Set((raw.baseDomains ?? []).map(normalizeHost)),- routes: (raw.routes ?? []).map(route => ({- domains: new Set((route.domains ?? []).map(normalizeHost)),- baseDomains: new Set((route.baseDomains ?? []).map(normalizeHost)),- pathPrefixes: (route.pathPrefixes ?? []).map(normalizePathPrefix),- })),- ttlMs: Math.max(1_000, ttlMs),- };-}--function getFireEngineDataLayerUrl(): string | null {- if (!config.FIRE_ENGINE_BETA_URL) {- return null;- }-- return `${config.FIRE_ENGINE_BETA_URL.replace(/\/+$/, "")}${DATA_LAYER_CAPABILITIES_PATH}`;-}--async function fetchDataLayerCapabilities(): Promise<DataLayerCapabilities | null> {- const url = getFireEngineDataLayerUrl();- if (!url) {- return null;- }-- try {- const response = await fetch(url, {- method: "GET",- signal: AbortSignal.timeout(DATA_LAYER_CAPABILITIES_TIMEOUT_MS),- });-- if (!response.ok) {- rootLogger.warn("Data layer capabilities request failed", {- statusCode: response.status,- });- return null;- }-- const parsed = dataLayerCapabilitiesSchema.parse(await response.json());- return normalizeCapabilities(parsed);- } catch (error) {- rootLogger.warn("Data layer capabilities request errored", { error });- return null;- }-}--async function getDataLayerCapabilities(): Promise<DataLayerCapabilities | null> {- if (cachedCapabilities && cachedCapabilities.expiresAt > Date.now()) {- return cachedCapabilities.value;- }-- if (!capabilitiesRequest) {- capabilitiesRequest = fetchDataLayerCapabilities().finally(() => {- capabilitiesRequest = undefined;- });- }-- const capabilities = await capabilitiesRequest;- cachedCapabilities = {- value: capabilities,- expiresAt:- Date.now() +- (capabilities?.ttlMs ?? DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS),- };-- return capabilities;-}--function dataLayerCapabilitiesMatchUrl(- capabilities: DataLayerCapabilities,- inputUrl: string,-): boolean {- let parsed: URL;- try {- parsed = new URL(inputUrl);- } catch {- return false;- }-- const host = normalizeHost(parsed.hostname);- const pathname = parsed.pathname || "/";-- if (capabilities.routes.length > 0) {- return capabilities.routes.some(route =>- dataLayerRouteMatchesUrl(route, host, pathname),- );- }-- if (capabilities.domains.has(host)) {- return true;- }-- for (const baseDomain of capabilities.baseDomains) {- if (host === baseDomain || host.endsWith(`.${baseDomain}`)) {- return true;- }- }-- return false;-}--function dataLayerRouteMatchesUrl(- route: DataLayerCapabilityRoute,- host: string,- pathname: string,-): boolean {- const hostMatches =- route.domains.has(host) ||- [...route.baseDomains].some(- baseDomain => host === baseDomain || host.endsWith(`.${baseDomain}`),- );-- if (!hostMatches) {- return false;- }-- if (route.pathPrefixes.length === 0) {- return true;- }-- return route.pathPrefixes.some(prefix => pathname.startsWith(prefix));-}--export async function isDataLayerSupportedUrl(- inputUrl: string,-): Promise<boolean> {- const capabilities = await getDataLayerCapabilities();- return (- capabilities !== null &&- dataLayerCapabilitiesMatchUrl(capabilities, inputUrl)- );-}--export function getDataLayerRequestLogContext(inputUrl: string):- | {- url: string;- host: string;- pathPrefix: string | null;- }- | undefined {- let parsed: URL;- try {- parsed = new URL(inputUrl);- } catch {- return undefined;- }-- return {- url: parsed.href,- host: parsed.hostname.toLowerCase(),- pathPrefix:- parsed.pathname- .split("/")- .map(part => part.trim())- .filter(part => part.length > 0)[0] ?? null,- };-}--export function getDataLayerResponseLogContext(meta: unknown): {- cacheState?: string;- cachedAt?: string;- cacheAgeMs?: number;- providerRequestId?: string;-} {- if (typeof meta !== "object" || meta === null) {- return {};- }-- const record = meta as Record<string, unknown>;- const requestId = record.request_id ?? record.requestId;-- return {- ...(typeof record.cacheState === "string"- ? { cacheState: record.cacheState }- : {}),- ...(typeof record.cachedAt === "string"- ? { cachedAt: record.cachedAt }- : {}),- ...(typeof record.cacheAgeMs === "number"- ? { cacheAgeMs: record.cacheAgeMs }- : {}),- ...(typeof requestId === "string" ? { providerRequestId: requestId } : {}),- };-}--export function isSuccessfulDataLayerStatusCode(statusCode: number): boolean {- return (statusCode >= 200 && statusCode < 300) || statusCode === 304;-}--export function isSupportedDataLayerFormatRequest(- formats?: FormatObject[] | unknown[],-): boolean {- if (formats === undefined) {- return true;- }-- if (!Array.isArray(formats) || formats.length === 0) {- return false;- }-- return formats.every(format => {- const type =- typeof format === "string"- ? format- : typeof format === "object" && format !== null && "type" in format- ? (format as { type?: unknown }).type- : undefined;-- return typeof type === "string" && SUPPORTED_FORMATS.has(type);- });-}--function getOrganizationDataSourceAccess(- flags: RouteInput["flags"],-): OrganizationDataSourceAccessRecord | null {- const access = flags?.organizationDataSourceAccess;- if (typeof access !== "object" || access === null) {- return null;- }-- for (const record of Object.values(access)) {- if (- typeof record === "object" &&- record !== null &&- record.termsKey === PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID- ) {- return record;- }- }-- return null;-}--function hasCurrentOrganizationDataSourceTerms(- access: OrganizationDataSourceAccessRecord,- termsKey: string,- version: string,-): boolean {- return access.termsKey === termsKey && access.termsVersion === version;-}--type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled";--function getProfessionalProfileCompanyDataDecision(- flags: RouteInput["flags"],-): DataSourceAccessDecision {- if (config.USE_DB_AUTHENTICATION !== true) {- return "allowed";- }-- const access = getOrganizationDataSourceAccess(flags);- if (access) {- if (access.status !== "enabled") {- return "not_enabled";- }-- return hasCurrentOrganizationDataSourceTerms(- access,- PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID,- THIRD_PARTY_DATA_TERMS_VERSION,- )- ? "allowed"- : "terms_required";- }-- return "terms_required";-}--function isDataLayerEligibleRequest(input: RouteInput): boolean {- if (input.flags?.[PROFESSIONAL_PROFILE_COMPANY_DATA_BETA_FLAG] !== true) {- return false;- }-- if (!config.FIRE_ENGINE_BETA_URL) {- return false;- }-- if (!input.url) {- return false;- }-- if (input.zeroDataRetention || input.lockdown) {- return false;- }-- if (Array.isArray(input.actions) && input.actions.length > 0) {- return false;- }-- if (input.headers && Object.keys(input.headers).length > 0) {- return false;- }-- if (input.waitFor !== undefined && input.waitFor !== 0) {- return false;- }-- if (input.mobile || input.location || input.blockAds === false) {- return false;- }-- if (input.proxy === "stealth" || input.proxy === "enhanced") {- return false;- }-- if (!isSupportedDataLayerFormatRequest(input.formats)) {- return false;- }-- return true;-}--export async function getDataLayerAccessForRequest(input: RouteInput): Promise<- | {- allowed: true;- termsRequired: false;- }- | {- allowed: false;- termsRequired: boolean;- }-> {- if (!isDataLayerEligibleRequest(input)) {- return { allowed: false, termsRequired: false };- }-- const supported = await isDataLayerSupportedUrl(input.url);- if (!supported) {- return { allowed: false, termsRequired: false };- }-- const dataSourceDecision = getProfessionalProfileCompanyDataDecision(- input.flags,- );- if (dataSourceDecision === "terms_required") {- return { allowed: false, termsRequired: true };- }- if (dataSourceDecision !== "allowed") {- return { allowed: false, termsRequired: false };- }-- return { allowed: true, termsRequired: false };-}--export async function canUseDataLayerForRequest(- input: RouteInput,-): Promise<boolean> {- return (await getDataLayerAccessForRequest(input)).allowed;-}--function getThirdPartyDataTermsSettingsUrl(): string {- return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`;-}--export function getThirdPartyDataTermsRequiredResponse() {- return {- success: false,- code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE,- error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE,- requiresAction: {- type: "accept_terms",- terms: PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID,- version: THIRD_PARTY_DATA_TERMS_VERSION,- url: getThirdPartyDataTermsSettingsUrl(),- },- };-}--export function getDataLayerSuccessCredits(input: {- dataLayer?: DataLayerScrapeMetadata;- statusCode?: number | null;-}): number | null {- if (input.dataLayer?.handled !== true) {- return null;- }-- const statusCode = input.statusCode;- if (- statusCode === undefined ||- statusCode === null ||- !isSuccessfulDataLayerStatusCode(statusCode)- ) {- return null;- }-- return DATA_LAYER_SUCCESS_CREDITS;-}--export function setDataLayerCapabilitiesForTest(input: {- domains?: string[];- baseDomains?: string[];- routes?: {- domains?: string[];- baseDomains?: string[];- pathPrefixes?: string[];- }[];- ttlSeconds?: number;-}) {- cachedCapabilities = {- value: normalizeCapabilities(input),- expiresAt: Date.now() + (input.ttlSeconds ?? 300) * 1000,- };-}--export function clearDataLayerCapabilitiesForTest() {- cachedCapabilities = undefined;- capabilitiesRequest = undefined;-}diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex 5103995c6..a6c67caee 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -6,33 +6,6 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";import type { ThreatDecision } from "./threat-protection/types";describe("calculateCreditsToBeBilled", () => {- it("bills handled data layer successes at 15 credits", async () => {- const credits = await calculateCreditsToBeBilled(- {- formats: [{ type: "markdown" }],- } as any,- {- teamId: "team-id",- },- {- metadata: {- statusCode: 200,- url: "https://profiles.example/in/example-person",- proxyUsed: "basic",- },- } as any,- {- totalCost: 0,- } as any,- {} as any,- undefined,- undefined,- { handled: true },- );-- expect(credits).toBe(15);- });-it("bills X/Twitter scrapes at 30 credits", async () => {const credits = await calculateCreditsToBeBilled({@@ -169,7 +142,6 @@ const billWithDecisions = (args: {{} as any,args.error,undefined,- undefined,args.threatDecisions,);diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 7350715ea..b393646d7 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -10,10 +10,6 @@ import { hasFormatOfType } from "./format-utils";import { TransportableError } from "./error";import { FeatureFlag } from "../scraper/scrapeURL/engines";import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";-import {- DataLayerScrapeMetadata,- getDataLayerSuccessCredits,-} from "./data-layer";import type { ThreatDecision } from "./threat-protection/types";import { UnsafeDomainBlockedError } from "./threat-protection/error";@@ -68,7 +64,6 @@ export async function calculateCreditsToBeBilled(flags: TeamFlags,error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,- dataLayer?: DataLayerScrapeMetadata,// Threat protection decisions for this scrape (initial + redirect checks,// in order). Each decision with `providerConsulted` bills a scan fee (+2// per unique scanned URL) on top of the scrape's own cost — on both success@@ -117,14 +112,6 @@ export async function calculateCreditsToBeBilled(return creditsToBeBilled + threatScanCredits;}- const dataLayerCredits = getDataLayerSuccessCredits({- dataLayer,- statusCode: document.metadata?.statusCode,- });- if (dataLayerCredits !== null) {- return dataLayerCredits + threatScanCredits;- }-let creditsToBeBilled = 1; // Assuming 1 credit per documentif (options.lockdown) {diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 88abdd894..5528a9aa8 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -29,11 +29,6 @@ import {CREDITS_FEATURE_ID,} from "../services/autumn/autumn.service";import { getTeamBalance } from "../services/autumn/usage";-import {- getDataLayerAccessForRequest,- getThirdPartyDataTermsRequiredResponse,-} from "../lib/data-layer";-import { getScrapeZDR } from "../lib/zdr-helpers";export function checkCreditsMiddleware(_minimum?: number,@@ -291,38 +286,8 @@ export function blocklistMiddleware(next: NextFunction,) {(async () => {- const zeroDataRetention =- getScrapeZDR(req.acuc?.flags) === "forced" ||- req.body?.zeroDataRetention === true ||- req.body?.lockdown === true;- const dataLayerAccess =- typeof req.body.url === "string" &&- (await getDataLayerAccessForRequest({- url: req.body.url,- formats: req.body.formats,- actions: req.body.actions,- headers: req.body.headers,- waitFor: req.body.waitFor,- mobile: req.body.mobile,- location: req.body.location,- proxy: req.body.proxy,- blockAds: req.body.blockAds,- zeroDataRetention,- lockdown: req.body.lockdown,- flags: req.acuc?.flags ?? null,- }));- const canUseDataLayer =- typeof dataLayerAccess === "object" && dataLayerAccess.allowed;-- if (typeof dataLayerAccess === "object" && dataLayerAccess.termsRequired) {- if (!res.headersSent) {- return res.status(403).json(getThirdPartyDataTermsRequiredResponse());- }- }-if (typeof req.body.url === "string" &&- !canUseDataLayer &&isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {team_id: req.acuc?.team_id ?? null,origin: typeof req.body.origin === "string" ? req.body.origin : null,diff --git a/apps/api/src/scraper/scrapeURL/engines/data-layer.ts b/apps/api/src/scraper/scrapeURL/engines/data-layer.tsdeleted file mode 100644index 53275c2e8..000000000--- a/apps/api/src/scraper/scrapeURL/engines/data-layer.ts+++ /dev/null@@ -1,183 +0,0 @@-import { z } from "zod";--import { Meta } from "..";-import { EngineScrapeResult } from ".";-import {- getDataLayerRequestLogContext,- getDataLayerResponseLogContext,- isSuccessfulDataLayerStatusCode,-} from "../../../lib/data-layer";-import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";-import { robustFetch } from "../lib/fetch";-import { EngineError } from "../error";-import { fireEngineURL } from "./fire-engine/scrape";--const dataLayerResultSchema = z- .object({- content: z.string().optional(),- json: z.unknown().optional(),- url: z.string().optional(),- pageStatusCode: z.number().optional(),- pageError: z.string().optional(),- responseHeaders: z.record(z.string(), z.string()).optional(),- meta: z.record(z.string(), z.unknown()).optional(),- error: z.string().optional(),- })- .passthrough();--const dataLayerScrapeResponseSchema = z.union([- z- .object({- handled: z.literal(false),- })- .passthrough(),- z- .object({- handled: z.literal(true),- statusCode: z.number(),- integrationId: z.string().optional(),- credits: z.number().optional(),- result: dataLayerResultSchema.optional(),- })- .passthrough(),-]);--function getContentType(headers?: Record<string, string>): string {- return (- Object.entries(headers ?? {}).find(- ([key]) => key.toLowerCase() === "content-type",- )?.[1] ?? ""- );-}--export function dataLayerMaxReasonableTime(meta: Meta): number {- return meta.options.timeout ?? 60_000;-}--export async function scrapeURLWithDataLayer(- meta: Meta,-): Promise<EngineScrapeResult> {- return withSpan("engine.data-layer.scrape", async span => {- const startTime = Date.now();- const url = meta.rewrittenUrl ?? meta.url;- const requestLogContext = getDataLayerRequestLogContext(url);- const logger = meta.logger.child({ method: "scrapeURLWithDataLayer" });-- setSpanAttributes(span, {- "engine.type": "data-layer",- "engine.url": url,- "engine.team_id": meta.internalOptions.teamId,- });-- logger.info("Data layer scrape started", {- ...requestLogContext,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- maxAge: meta.options.maxAge,- });-- try {- const response = await robustFetch({- url: `${fireEngineURL}/v1/data-layer/scrape`,- method: "POST",- body: {- url,- formats: meta.options.formats,- maxAge: meta.options.maxAge,- zeroDataRetention: meta.internalOptions.zeroDataRetention,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- source: "firecrawl",- },- logger: logger.child({ method: "dataLayerScrape/robustFetch" }),- tryCount: 2,- ignoreFailureStatus: true,- mock: meta.mock,- abort: meta.abort.asSignal(),- schema: dataLayerScrapeResponseSchema,- });-- if (!response.handled) {- logger.info("Data layer scrape was not handled", {- ...requestLogContext,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- durationMs: Date.now() - startTime,- });- throw new EngineError("Data layer did not handle URL");- }-- const responseLogContext = getDataLayerResponseLogContext(- response.result?.meta,- );- const pageStatusCode =- response.result?.pageStatusCode ?? response.statusCode;-- if (- !isSuccessfulDataLayerStatusCode(response.statusCode) ||- !isSuccessfulDataLayerStatusCode(pageStatusCode) ||- !response.result- ) {- logger.warn("Data layer scrape failed", {- ...requestLogContext,- ...responseLogContext,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- integrationId: response.integrationId,- statusCode: response.statusCode,- pageStatusCode,- durationMs: Date.now() - startTime,- });- throw new EngineError("Data layer request failed");- }-- const contentType = getContentType(response.result.responseHeaders);-- logger.info("Data layer scrape completed", {- ...requestLogContext,- ...responseLogContext,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- integrationId: response.integrationId,- statusCode: response.statusCode,- durationMs: Date.now() - startTime,- });-- setSpanAttributes(span, {- "data-layer.integration_id": response.integrationId,- "data-layer.status_code": response.statusCode,- "data-layer.page_status_code": pageStatusCode,- "data-layer.cache_state": responseLogContext.cacheState,- "data-layer.cache_age_ms": responseLogContext.cacheAgeMs,- "data-layer.duration_ms": Date.now() - startTime,- });-- return {- url: response.result.url ?? url,- html: response.result.content ?? "",- markdown: contentType.includes("text/markdown")- ? response.result.content- : undefined,- json: response.result.json,- error: response.result.pageError,- statusCode: pageStatusCode,- contentType,- proxyUsed: "basic",- dataLayer: {- handled: true,- integrationId: response.integrationId,- },- };- } catch (error) {- logger.warn("Data layer scrape errored", {- ...requestLogContext,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- durationMs: Date.now() - startTime,- errorMessage: error instanceof Error ? error.message : String(error),- error,- });- throw error;- }- });-}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex c487efee7..8a9b2f727 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -7,10 +7,6 @@ import {scrapeURLWithFireEngineChromeCDP,scrapeURLWithFireEngineTLSClient,} from "./fire-engine";-import {- dataLayerMaxReasonableTime,- scrapeURLWithDataLayer,-} from "./data-layer";import { pdfMaxReasonableTime, scrapePDF } from "./pdf";import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";import {@@ -34,13 +30,8 @@ import { getPDFMaxPages } from "../../../controllers/v2/types";import type { PdfMetadata } from "./pdf/types";import { BrandingProfile } from "../../../types/branding";import { BrandingNotSupportedError } from "../error";-import {- canUseDataLayerForRequest,- type DataLayerScrapeMetadata,-} from "../../../lib/data-layer";export type Engine =- | "data-layer"| "fire-engine;chrome-cdp"| "fire-engine(retry);chrome-cdp"| "fire-engine;chrome-cdp;stealth"@@ -171,13 +162,11 @@ export type EngineScrapeResult = {proxyUsed: "basic" | "stealth";timezone?: string;- dataLayer?: DataLayerScrapeMetadata;};const engineHandlers: {[E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;} = {- "data-layer": scrapeURLWithDataLayer,index: scrapeURLWithIndex,"index;documents": scrapeURLWithIndex,"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,@@ -197,7 +186,6 @@ const engineHandlers: {const engineMRTs: {[E in Engine]: (meta: Meta) => number;} = {- "data-layer": dataLayerMaxReasonableTime,index: indexMaxReasonableTime,"index;documents": indexMaxReasonableTime,"fire-engine;chrome-cdp": meta =>@@ -230,27 +218,6 @@ const engineOptions: {quality: number;};} = {- "data-layer": {- features: {- actions: false,- waitFor: false,- screenshot: false,- "screenshot@fullScreen": false,- pdf: false,- document: false,- audio: false,- video: false,- atsv: false,- location: false,- mobile: false,- skipTlsVerification: true,- useFastMode: true,- stealthProxy: false,- branding: false,- disableAdblock: false,- },- quality: 2000,- },index: {features: {actions: false,@@ -580,31 +547,6 @@ export async function buildFallbackList(meta: Meta): Promise<unsupportedFeatures: Set<FeatureFlag>;}[]> {- if (- !meta.internalOptions.agentIndexOnly &&- (await canUseDataLayerForRequest({- url: meta.rewrittenUrl ?? meta.url,- formats: meta.options.formats,- actions: meta.options.actions,- headers: meta.options.headers,- waitFor: meta.options.waitFor,- mobile: meta.options.mobile,- location: meta.options.location,- proxy: meta.options.proxy,- blockAds: meta.options.blockAds,- zeroDataRetention: meta.internalOptions.zeroDataRetention,- lockdown: meta.options.lockdown,- flags: meta.internalOptions.teamFlags ?? null,- }))- ) {- return [- {- engine: "data-layer",- unsupportedFeatures: new Set(),- },- ];- }-const shouldPrioritizeTlsClient = meta.options.__experimental_engpicker? (await queryEngpickerVerdict(meta.options.__experimental_omceDomain ?? new URL(meta.url).hostname,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 1c86edf65..fb1e738f6 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -87,7 +87,6 @@ import { writeFile } from "node:fs/promises";import path from "node:path";import { tmpdir } from "node:os";import { randomUUID } from "node:crypto";-import type { DataLayerScrapeMetadata } from "../../lib/data-layer";import {checkUrl,type ThreatCheckDedup,@@ -102,7 +101,6 @@ export type ScrapeUrlResponse =success: true;document: Document;unsupportedFeatures?: Set<FeatureFlag>;- dataLayer?: DataLayerScrapeMetadata;/*** Threat protection decisions made for this scrape (initial domain* check + any redirect re-checks, in order). Read by the billing layer@@ -1073,7 +1071,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {success: true,document,unsupportedFeatures: result.unsupportedFeatures,- dataLayer: engineResult.dataLayer,};});}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex cac3b23f6..f3d968e02 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -99,7 +99,6 @@ import {recordMonitorScrapeFailure,recordMonitorScrapeSuccess,} from "../monitoring/results";-import type { DataLayerScrapeMetadata } from "../../lib/data-layer";configDotenv();@@ -119,7 +118,6 @@ async function billScrapeJob(flags: TeamFlags,error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,- dataLayer?: DataLayerScrapeMetadata,threatDecisions?: ThreatDecision[],) {let creditsToBeBilled: number | null = null;@@ -148,7 +146,6 @@ async function billScrapeJob(flags,error,unsupportedFeatures,- dataLayer,threatDecisions,);@@ -666,7 +663,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,undefined,pipeline.unsupportedFeatures,- pipeline.dataLayer,pipeline.threatDecisions,);@@ -758,7 +754,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,undefined,pipeline.unsupportedFeatures,- pipeline.dataLayer,pipeline.threatDecisions,);@@ -959,7 +954,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(await getACUCTeam(job.data.team_id))?.flags ?? null,error instanceof Error ? error : null,undefined,- undefined,pipeline?.threatDecisions,);