Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore(api): remove the data-layer scrape path (#4029)

Abimael Martelleb27f0b2095b
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
index 594d55cde..5c2d1cac3 100644
--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
@@ -6,8 +6,6 @@
describe("Audio format engine routing (buildFallbackList)", () => {
let buildFallbackList: typeof import("../../../scraper/scrapeURL/engines/index.js").buildFallbackList;
- let clearDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").clearDataLayerCapabilitiesForTest;
- let setDataLayerCapabilitiesForTest: typeof import("../../../lib/data-layer.js").setDataLayerCapabilitiesForTest;
const originalFireEngineUrl = process.env.FIRE_ENGINE_BETA_URL;
const originalIndexUrl = process.env.INDEX_DATABASE_URL;
@@ -22,14 +20,6 @@ describe("Audio format engine routing (buildFallbackList)", () => {
({ buildFallbackList } = await import(
"../../../scraper/scrapeURL/engines/index.js"
));
- ({
- clearDataLayerCapabilitiesForTest,
- setDataLayerCapabilitiesForTest,
- } = await import("../../../lib/data-layer.js"));
- });
-
- afterEach(() => {
- clearDataLayerCapabilitiesForTest();
});
afterAll(() => {
@@ -120,20 +110,4 @@ describe("Audio format engine routing (buildFallbackList)", () => {
expect(engines).toContain("fire-engine;chrome-cdp");
});
- it("does not route agent index-only requests through the data layer", async () => {
- setDataLayerCapabilitiesForTest({
- domains: ["profiles.example"],
- });
-
- const meta = buildStubMeta([]);
- meta.url = "https://profiles.example/person/example-person";
- meta.options.formats = [{ type: "markdown" }];
- meta.internalOptions.agentIndexOnly = true;
- meta.internalOptions.teamFlags = { enrichBeta: true };
-
- const fallback = await buildFallbackList(meta);
- const engines = fallback.map(f => f.engine);
-
- expect(engines).toEqual(["index", "index;documents"]);
- });
});
diff --git a/apps/api/src/lib/data-layer.test.ts b/apps/api/src/lib/data-layer.test.ts
deleted file mode 100644
index d2231f4f2..000000000
--- a/apps/api/src/lib/data-layer.test.ts
+++ /dev/null
@@ -1,325 +0,0 @@
-import { fetch } from "undici";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-
-import { config } from "../config";
-import {
- canUseDataLayerForRequest,
- clearDataLayerCapabilitiesForTest,
- getDataLayerAccessForRequest,
- getDataLayerRequestLogContext,
- getDataLayerResponseLogContext,
- getDataLayerSuccessCredits,
- getThirdPartyDataTermsRequiredResponse,
- isDataLayerSupportedUrl,
- isSuccessfulDataLayerStatusCode,
- isSupportedDataLayerFormatRequest,
- setDataLayerCapabilitiesForTest,
-} from "./data-layer";
-
-vi.mock("undici", () => ({
- fetch: vi.fn(),
-}));
-
-const originalConfig = {
- FIRE_ENGINE_BETA_URL: config.FIRE_ENGINE_BETA_URL,
-};
-const ENABLED_DATA_LAYER_FLAGS = {
- professionalProfileCompanyDataBeta: true,
- organizationDataSourceAccess: {
- provider: {
- status: "enabled",
- termsKey: "professional_profile_company_data",
- termsVersion: "2026-07-03",
- termsAcceptedAt: "2026-07-03T00:00:00.000Z",
- enabledAt: "2026-07-03T00:00:00.000Z",
- disabledAt: null,
- disabledReason: null,
- },
- },
-};
-
-describe("data layer routing", () => {
- beforeEach(() => {
- vi.mocked(fetch).mockReset();
- config.FIRE_ENGINE_BETA_URL = "https://fire-engine.example";
- setDataLayerCapabilitiesForTest({
- domains: ["profiles.example"],
- baseDomains: ["network.example"],
- });
- });
-
- afterEach(() => {
- config.FIRE_ENGINE_BETA_URL = originalConfig.FIRE_ENGINE_BETA_URL;
- clearDataLayerCapabilitiesForTest();
- });
-
- it("detects URLs using Fire Engine capabilities", async () => {
- await expect(
- isDataLayerSupportedUrl(
- "https://profiles.example/person/example-person/details/experience/?trk=foo",
- ),
- ).resolves.toBe(true);
- await expect(
- isDataLayerSupportedUrl("https://www.network.example/any/path"),
- ).resolves.toBe(true);
- await expect(
- isDataLayerSupportedUrl("https://other.example/person/example-person"),
- ).resolves.toBe(false);
- await expect(isDataLayerSupportedUrl("not a url")).resolves.toBe(false);
- });
-
- it("uses route capabilities when Fire Engine advertises path-level support", async () => {
- setDataLayerCapabilitiesForTest({
- domains: ["profiles.example"],
- routes: [
- {
- domains: ["profiles.example"],
- pathPrefixes: ["/person/", "/company/"],
- },
- ],
- });
-
- await expect(
- isDataLayerSupportedUrl("https://profiles.example/person/example-person"),
- ).resolves.toBe(true);
- await expect(
- isDataLayerSupportedUrl("https://profiles.example/company/example"),
- ).resolves.toBe(true);
- await expect(
- isDataLayerSupportedUrl("https://profiles.example/jobs/example"),
- ).resolves.toBe(false);
- });
-
- it("caches failed Fire Engine capabilities lookups briefly", async () => {
- clearDataLayerCapabilitiesForTest();
- vi.mocked(fetch).mockResolvedValue({
- ok: false,
- status: 503,
- } as Awaited<ReturnType<typeof fetch>>);
-
- await expect(
- isDataLayerSupportedUrl("https://profiles.example/person/example-person"),
- ).resolves.toBe(false);
- await expect(
- isDataLayerSupportedUrl("https://profiles.example/person/example-person"),
- ).resolves.toBe(false);
-
- expect(fetch).toHaveBeenCalledTimes(1);
- });
-
- it("builds a compact request log context", () => {
- expect(
- getDataLayerRequestLogContext(
- "https://profiles.example/person/example-person/details/experience/?trk=foo",
- ),
- ).toEqual({
- url: "https://profiles.example/person/example-person/details/experience/?trk=foo",
- host: "profiles.example",
- pathPrefix: "person",
- });
-
- expect(getDataLayerRequestLogContext("not a url")).toBeUndefined();
- });
-
- it("extracts response cache metadata for logs", () => {
- expect(
- getDataLayerResponseLogContext({
- cacheState: "hit",
- cachedAt: "2026-06-21T10:00:00.000Z",
- cacheAgeMs: 1000,
- request_id: "req_123",
- extra: "ignored",
- }),
- ).toEqual({
- cacheState: "hit",
- cachedAt: "2026-06-21T10:00:00.000Z",
- cacheAgeMs: 1000,
- providerRequestId: "req_123",
- });
-
- expect(getDataLayerResponseLogContext(null)).toEqual({});
- });
-
- it("accepts only formats that Fire Engine can return directly", () => {
- expect(isSupportedDataLayerFormatRequest(undefined)).toBe(true);
- expect(isSupportedDataLayerFormatRequest([{ type: "markdown" }])).toBe(
- true,
- );
- expect(isSupportedDataLayerFormatRequest(["json"])).toBe(true);
- expect(
- isSupportedDataLayerFormatRequest([
- { type: "markdown" },
- { type: "json" },
- ]),
- ).toBe(true);
- expect(isSupportedDataLayerFormatRequest([{ type: "html" }])).toBe(false);
- expect(isSupportedDataLayerFormatRequest([])).toBe(false);
- });
-
- it("allows eligible requests through the blocklist when the org data source beta flag is enabled", async () => {
- await expect(
- canUseDataLayerForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- flags: ENABLED_DATA_LAYER_FLAGS,
- }),
- ).resolves.toBe(true);
-
- await expect(
- canUseDataLayerForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "json" }],
- actions: [{ type: "wait" }],
- flags: ENABLED_DATA_LAYER_FLAGS,
- }),
- ).resolves.toBe(false);
-
- await expect(
- canUseDataLayerForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "json" }],
- zeroDataRetention: true,
- flags: ENABLED_DATA_LAYER_FLAGS,
- }),
- ).resolves.toBe(false);
- });
-
- it("requires current Third-Party Data terms before routing", async () => {
- await expect(
- getDataLayerAccessForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- flags: { professionalProfileCompanyDataBeta: true },
- }),
- ).resolves.toEqual({ allowed: false, termsRequired: true });
-
- expect(getThirdPartyDataTermsRequiredResponse()).toMatchObject({
- success: false,
- code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
- requiresAction: {
- type: "accept_terms",
- terms: "professional_profile_company_data",
- version: "2026-07-03",
- },
- });
-
- await expect(
- canUseDataLayerForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- flags: { professionalProfileCompanyDataBeta: true },
- }),
- ).resolves.toBe(false);
-
- await expect(
- canUseDataLayerForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- flags: { professionalProfileCompanyDataBeta: true },
- }),
- ).resolves.toBe(false);
- });
-
- it("requires current terms when organization data source access is stale", async () => {
- await expect(
- getDataLayerAccessForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- flags: {
- professionalProfileCompanyDataBeta: true,
- organizationDataSourceAccess: {
- provider: {
- status: "enabled",
- termsKey: "professional_profile_company_data",
- termsVersion: "2026-07-02",
- },
- },
- },
- }),
- ).resolves.toEqual({ allowed: false, termsRequired: true });
- });
-
- it("does not route or prompt for terms when organization data source access is disabled", async () => {
- await expect(
- getDataLayerAccessForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- flags: {
- professionalProfileCompanyDataBeta: true,
- organizationDataSourceAccess: {
- provider: {
- status: "disabled",
- termsKey: "professional_profile_company_data",
- termsVersion: "2026-07-03",
- disabledAt: "2026-07-04T00:00:00.000Z",
- disabledReason: "customer_disabled",
- },
- },
- },
- }),
- ).resolves.toEqual({ allowed: false, termsRequired: false });
- });
-
- it("does not bypass unless the org data source beta flag is enabled", async () => {
- await expect(
- canUseDataLayerForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- }),
- ).resolves.toBe(false);
-
- await expect(
- canUseDataLayerForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- flags: { professionalProfileCompanyDataBeta: false },
- }),
- ).resolves.toBe(false);
- });
-
- it("does not bypass unless Fire Engine is configured", async () => {
- config.FIRE_ENGINE_BETA_URL = undefined;
-
- await expect(
- canUseDataLayerForRequest({
- url: "https://profiles.example/person/example-person",
- formats: [{ type: "markdown" }],
- flags: ENABLED_DATA_LAYER_FLAGS,
- }),
- ).resolves.toBe(false);
- });
-
- it("returns 15 credits only for successful handled responses", () => {
- expect(isSuccessfulDataLayerStatusCode(200)).toBe(true);
- expect(isSuccessfulDataLayerStatusCode(204)).toBe(true);
- expect(isSuccessfulDataLayerStatusCode(304)).toBe(true);
- expect(isSuccessfulDataLayerStatusCode(404)).toBe(false);
-
- expect(
- getDataLayerSuccessCredits({
- dataLayer: { handled: true, integrationId: "example" },
- statusCode: 200,
- }),
- ).toBe(15);
-
- expect(
- getDataLayerSuccessCredits({
- dataLayer: { handled: true, integrationId: "example" },
- statusCode: 304,
- }),
- ).toBe(15);
-
- expect(
- getDataLayerSuccessCredits({
- dataLayer: { handled: true, integrationId: "example" },
- statusCode: 404,
- }),
- ).toBeNull();
-
- expect(
- getDataLayerSuccessCredits({
- statusCode: 200,
- }),
- ).toBeNull();
- });
-});
diff --git a/apps/api/src/lib/data-layer.ts b/apps/api/src/lib/data-layer.ts
deleted file mode 100644
index eeedce349..000000000
--- a/apps/api/src/lib/data-layer.ts
+++ /dev/null
@@ -1,529 +0,0 @@
-import { fetch } from "undici";
-import { z } from "zod";
-
-import { config } from "../config";
-import type { FormatObject } from "../controllers/v2/types";
-import { logger as rootLogger } from "./logger";
-
-type OrganizationDataSourceAccessRecord = {
- status?: string | null;
- termsKey?: string | null;
- termsVersion?: string | null;
- termsAcceptedAt?: string | null;
- enabledAt?: string | null;
- disabledAt?: string | null;
- disabledReason?: string | null;
-};
-
-type OrganizationDataSourceAccess = Record<
- string,
- OrganizationDataSourceAccessRecord | null | undefined
->;
-
-type RouteInput = {
- url: string;
- formats?: FormatObject[] | unknown[];
- actions?: unknown[];
- headers?: Record<string, unknown>;
- waitFor?: number;
- mobile?: boolean;
- location?: unknown;
- proxy?: unknown;
- blockAds?: boolean;
- zeroDataRetention?: boolean;
- lockdown?: boolean;
- flags?: {
- professionalProfileCompanyDataBeta?: boolean;
- organizationDataSourceAccess?: OrganizationDataSourceAccess | null;
- } | null;
-};
-
-export type DataLayerScrapeMetadata = {
- handled: true;
- integrationId?: string;
-};
-
-const SUPPORTED_FORMATS = new Set(["markdown", "json", "deterministicJson"]);
-const DATA_LAYER_SUCCESS_CREDITS = 15;
-const PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID =
- "professional_profile_company_data";
-const PROFESSIONAL_PROFILE_COMPANY_DATA_BETA_FLAG =
- "professionalProfileCompanyDataBeta";
-const THIRD_PARTY_DATA_TERMS_VERSION = "2026-07-03";
-const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";
-const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE =
- "An organization admin must accept the Professional Profile & Company Data terms before this URL can be processed.";
-
-const DATA_LAYER_CAPABILITIES_PATH = "/v1/data-layer/capabilities";
-const DATA_LAYER_CAPABILITIES_TIMEOUT_MS = 2_000;
-const DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS = 30_000;
-
-const dataLayerCapabilityRouteSchema = z
- .object({
- domains: z.string().array().optional(),
- baseDomains: z.string().array().optional(),
- pathPrefixes: z.string().array().optional(),
- })
- .passthrough();
-
-const dataLayerCapabilitiesSchema = z
- .object({
- version: z.number().optional(),
- ttlSeconds: z.number().positive().optional(),
- domains: z.string().array().optional(),
- baseDomains: z.string().array().optional(),
- routes: dataLayerCapabilityRouteSchema.array().optional(),
- })
- .passthrough();
-
-type DataLayerCapabilityRoute = {
- domains: Set<string>;
- baseDomains: Set<string>;
- pathPrefixes: string[];
-};
-
-type DataLayerCapabilities = {
- domains: Set<string>;
- baseDomains: Set<string>;
- routes: DataLayerCapabilityRoute[];
- ttlMs: number;
-};
-
-let cachedCapabilities:
- | {
- expiresAt: number;
- value: DataLayerCapabilities | null;
- }
- | undefined;
-let capabilitiesRequest: Promise<DataLayerCapabilities | null> | undefined;
-
-function normalizeHost(host: string): string {
- return host.trim().toLowerCase().replace(/\.$/, "");
-}
-
-function normalizePathPrefix(prefix: string): string {
- const trimmed = prefix.trim();
- if (!trimmed) {
- return "/";
- }
-
- return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
-}
-
-function normalizeCapabilities(
- raw: z.infer<typeof dataLayerCapabilitiesSchema>,
-): DataLayerCapabilities {
- const ttlMs =
- typeof raw.ttlSeconds === "number" && Number.isFinite(raw.ttlSeconds)
- ? raw.ttlSeconds * 1000
- : DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS;
-
- return {
- domains: new Set((raw.domains ?? []).map(normalizeHost)),
- baseDomains: new Set((raw.baseDomains ?? []).map(normalizeHost)),
- routes: (raw.routes ?? []).map(route => ({
- domains: new Set((route.domains ?? []).map(normalizeHost)),
- baseDomains: new Set((route.baseDomains ?? []).map(normalizeHost)),
- pathPrefixes: (route.pathPrefixes ?? []).map(normalizePathPrefix),
- })),
- ttlMs: Math.max(1_000, ttlMs),
- };
-}
-
-function getFireEngineDataLayerUrl(): string | null {
- if (!config.FIRE_ENGINE_BETA_URL) {
- return null;
- }
-
- return `${config.FIRE_ENGINE_BETA_URL.replace(/\/+$/, "")}${DATA_LAYER_CAPABILITIES_PATH}`;
-}
-
-async function fetchDataLayerCapabilities(): Promise<DataLayerCapabilities | null> {
- const url = getFireEngineDataLayerUrl();
- if (!url) {
- return null;
- }
-
- try {
- const response = await fetch(url, {
- method: "GET",
- signal: AbortSignal.timeout(DATA_LAYER_CAPABILITIES_TIMEOUT_MS),
- });
-
- if (!response.ok) {
- rootLogger.warn("Data layer capabilities request failed", {
- statusCode: response.status,
- });
- return null;
- }
-
- const parsed = dataLayerCapabilitiesSchema.parse(await response.json());
- return normalizeCapabilities(parsed);
- } catch (error) {
- rootLogger.warn("Data layer capabilities request errored", { error });
- return null;
- }
-}
-
-async function getDataLayerCapabilities(): Promise<DataLayerCapabilities | null> {
- if (cachedCapabilities && cachedCapabilities.expiresAt > Date.now()) {
- return cachedCapabilities.value;
- }
-
- if (!capabilitiesRequest) {
- capabilitiesRequest = fetchDataLayerCapabilities().finally(() => {
- capabilitiesRequest = undefined;
- });
- }
-
- const capabilities = await capabilitiesRequest;
- cachedCapabilities = {
- value: capabilities,
- expiresAt:
- Date.now() +
- (capabilities?.ttlMs ?? DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS),
- };
-
- return capabilities;
-}
-
-function dataLayerCapabilitiesMatchUrl(
- capabilities: DataLayerCapabilities,
- inputUrl: string,
-): boolean {
- let parsed: URL;
- try {
- parsed = new URL(inputUrl);
- } catch {
- return false;
- }
-
- const host = normalizeHost(parsed.hostname);
- const pathname = parsed.pathname || "/";
-
- if (capabilities.routes.length > 0) {
- return capabilities.routes.some(route =>
- dataLayerRouteMatchesUrl(route, host, pathname),
- );
- }
-
- if (capabilities.domains.has(host)) {
- return true;
- }
-
- for (const baseDomain of capabilities.baseDomains) {
- if (host === baseDomain || host.endsWith(`.${baseDomain}`)) {
- return true;
- }
- }
-
- return false;
-}
-
-function dataLayerRouteMatchesUrl(
- route: DataLayerCapabilityRoute,
- host: string,
- pathname: string,
-): boolean {
- const hostMatches =
- route.domains.has(host) ||
- [...route.baseDomains].some(
- baseDomain => host === baseDomain || host.endsWith(`.${baseDomain}`),
- );
-
- if (!hostMatches) {
- return false;
- }
-
- if (route.pathPrefixes.length === 0) {
- return true;
- }
-
- return route.pathPrefixes.some(prefix => pathname.startsWith(prefix));
-}
-
-export async function isDataLayerSupportedUrl(
- inputUrl: string,
-): Promise<boolean> {
- const capabilities = await getDataLayerCapabilities();
- return (
- capabilities !== null &&
- dataLayerCapabilitiesMatchUrl(capabilities, inputUrl)
- );
-}
-
-export function getDataLayerRequestLogContext(inputUrl: string):
- | {
- url: string;
- host: string;
- pathPrefix: string | null;
- }
- | undefined {
- let parsed: URL;
- try {
- parsed = new URL(inputUrl);
- } catch {
- return undefined;
- }
-
- return {
- url: parsed.href,
- host: parsed.hostname.toLowerCase(),
- pathPrefix:
- parsed.pathname
- .split("/")
- .map(part => part.trim())
- .filter(part => part.length > 0)[0] ?? null,
- };
-}
-
-export function getDataLayerResponseLogContext(meta: unknown): {
- cacheState?: string;
- cachedAt?: string;
- cacheAgeMs?: number;
- providerRequestId?: string;
-} {
- if (typeof meta !== "object" || meta === null) {
- return {};
- }
-
- const record = meta as Record<string, unknown>;
- const requestId = record.request_id ?? record.requestId;
-
- return {
- ...(typeof record.cacheState === "string"
- ? { cacheState: record.cacheState }
- : {}),
- ...(typeof record.cachedAt === "string"
- ? { cachedAt: record.cachedAt }
- : {}),
- ...(typeof record.cacheAgeMs === "number"
- ? { cacheAgeMs: record.cacheAgeMs }
- : {}),
- ...(typeof requestId === "string" ? { providerRequestId: requestId } : {}),
- };
-}
-
-export function isSuccessfulDataLayerStatusCode(statusCode: number): boolean {
- return (statusCode >= 200 && statusCode < 300) || statusCode === 304;
-}
-
-export function isSupportedDataLayerFormatRequest(
- formats?: FormatObject[] | unknown[],
-): boolean {
- if (formats === undefined) {
- return true;
- }
-
- if (!Array.isArray(formats) || formats.length === 0) {
- return false;
- }
-
- return formats.every(format => {
- const type =
- typeof format === "string"
- ? format
- : typeof format === "object" && format !== null && "type" in format
- ? (format as { type?: unknown }).type
- : undefined;
-
- return typeof type === "string" && SUPPORTED_FORMATS.has(type);
- });
-}
-
-function getOrganizationDataSourceAccess(
- flags: RouteInput["flags"],
-): OrganizationDataSourceAccessRecord | null {
- const access = flags?.organizationDataSourceAccess;
- if (typeof access !== "object" || access === null) {
- return null;
- }
-
- for (const record of Object.values(access)) {
- if (
- typeof record === "object" &&
- record !== null &&
- record.termsKey === PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID
- ) {
- return record;
- }
- }
-
- return null;
-}
-
-function hasCurrentOrganizationDataSourceTerms(
- access: OrganizationDataSourceAccessRecord,
- termsKey: string,
- version: string,
-): boolean {
- return access.termsKey === termsKey && access.termsVersion === version;
-}
-
-type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled";
-
-function getProfessionalProfileCompanyDataDecision(
- flags: RouteInput["flags"],
-): DataSourceAccessDecision {
- if (config.USE_DB_AUTHENTICATION !== true) {
- return "allowed";
- }
-
- const access = getOrganizationDataSourceAccess(flags);
- if (access) {
- if (access.status !== "enabled") {
- return "not_enabled";
- }
-
- return hasCurrentOrganizationDataSourceTerms(
- access,
- PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID,
- THIRD_PARTY_DATA_TERMS_VERSION,
- )
- ? "allowed"
- : "terms_required";
- }
-
- return "terms_required";
-}
-
-function isDataLayerEligibleRequest(input: RouteInput): boolean {
- if (input.flags?.[PROFESSIONAL_PROFILE_COMPANY_DATA_BETA_FLAG] !== true) {
- return false;
- }
-
- if (!config.FIRE_ENGINE_BETA_URL) {
- return false;
- }
-
- if (!input.url) {
- return false;
- }
-
- if (input.zeroDataRetention || input.lockdown) {
- return false;
- }
-
- if (Array.isArray(input.actions) && input.actions.length > 0) {
- return false;
- }
-
- if (input.headers && Object.keys(input.headers).length > 0) {
- return false;
- }
-
- if (input.waitFor !== undefined && input.waitFor !== 0) {
- return false;
- }
-
- if (input.mobile || input.location || input.blockAds === false) {
- return false;
- }
-
- if (input.proxy === "stealth" || input.proxy === "enhanced") {
- return false;
- }
-
- if (!isSupportedDataLayerFormatRequest(input.formats)) {
- return false;
- }
-
- return true;
-}
-
-export async function getDataLayerAccessForRequest(input: RouteInput): Promise<
- | {
- allowed: true;
- termsRequired: false;
- }
- | {
- allowed: false;
- termsRequired: boolean;
- }
-> {
- if (!isDataLayerEligibleRequest(input)) {
- return { allowed: false, termsRequired: false };
- }
-
- const supported = await isDataLayerSupportedUrl(input.url);
- if (!supported) {
- return { allowed: false, termsRequired: false };
- }
-
- const dataSourceDecision = getProfessionalProfileCompanyDataDecision(
- input.flags,
- );
- if (dataSourceDecision === "terms_required") {
- return { allowed: false, termsRequired: true };
- }
- if (dataSourceDecision !== "allowed") {
- return { allowed: false, termsRequired: false };
- }
-
- return { allowed: true, termsRequired: false };
-}
-
-export async function canUseDataLayerForRequest(
- input: RouteInput,
-): Promise<boolean> {
- return (await getDataLayerAccessForRequest(input)).allowed;
-}
-
-function getThirdPartyDataTermsSettingsUrl(): string {
- return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`;
-}
-
-export function getThirdPartyDataTermsRequiredResponse() {
- return {
- success: false,
- code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE,
- error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE,
- requiresAction: {
- type: "accept_terms",
- terms: PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID,
- version: THIRD_PARTY_DATA_TERMS_VERSION,
- url: getThirdPartyDataTermsSettingsUrl(),
- },
- };
-}
-
-export function getDataLayerSuccessCredits(input: {
- dataLayer?: DataLayerScrapeMetadata;
- statusCode?: number | null;
-}): number | null {
- if (input.dataLayer?.handled !== true) {
- return null;
- }
-
- const statusCode = input.statusCode;
- if (
- statusCode === undefined ||
- statusCode === null ||
- !isSuccessfulDataLayerStatusCode(statusCode)
- ) {
- return null;
- }
-
- return DATA_LAYER_SUCCESS_CREDITS;
-}
-
-export function setDataLayerCapabilitiesForTest(input: {
- domains?: string[];
- baseDomains?: string[];
- routes?: {
- domains?: string[];
- baseDomains?: string[];
- pathPrefixes?: string[];
- }[];
- ttlSeconds?: number;
-}) {
- cachedCapabilities = {
- value: normalizeCapabilities(input),
- expiresAt: Date.now() + (input.ttlSeconds ?? 300) * 1000,
- };
-}
-
-export function clearDataLayerCapabilitiesForTest() {
- cachedCapabilities = undefined;
- capabilitiesRequest = undefined;
-}
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index 5103995c6..a6c67caee 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -6,33 +6,6 @@ import { UnsafeDomainBlockedError } from "./threat-protection/error";
import type { ThreatDecision } from "./threat-protection/types";
describe("calculateCreditsToBeBilled", () => {
- it("bills handled data layer successes at 15 credits", async () => {
- const credits = await calculateCreditsToBeBilled(
- {
- formats: [{ type: "markdown" }],
- } as any,
- {
- teamId: "team-id",
- },
- {
- metadata: {
- statusCode: 200,
- url: "https://profiles.example/in/example-person",
- proxyUsed: "basic",
- },
- } as any,
- {
- totalCost: 0,
- } as any,
- {} as any,
- undefined,
- undefined,
- { handled: true },
- );
-
- expect(credits).toBe(15);
- });
-
it("bills X/Twitter scrapes at 30 credits", async () => {
const credits = await calculateCreditsToBeBilled(
{
@@ -169,7 +142,6 @@ const billWithDecisions = (args: {
{} as any,
args.error,
undefined,
- undefined,
args.threatDecisions,
);
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 7350715ea..b393646d7 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -10,10 +10,6 @@ import { hasFormatOfType } from "./format-utils";
import { TransportableError } from "./error";
import { FeatureFlag } from "../scraper/scrapeURL/engines";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
-import {
- DataLayerScrapeMetadata,
- getDataLayerSuccessCredits,
-} from "./data-layer";
import type { ThreatDecision } from "./threat-protection/types";
import { UnsafeDomainBlockedError } from "./threat-protection/error";
@@ -68,7 +64,6 @@ export async function calculateCreditsToBeBilled(
flags: TeamFlags,
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
- dataLayer?: DataLayerScrapeMetadata,
// Threat protection decisions for this scrape (initial + redirect checks,
// in order). Each decision with `providerConsulted` bills a scan fee (+2
// per unique scanned URL) on top of the scrape's own cost — on both success
@@ -117,14 +112,6 @@ export async function calculateCreditsToBeBilled(
return creditsToBeBilled + threatScanCredits;
}
- const dataLayerCredits = getDataLayerSuccessCredits({
- dataLayer,
- statusCode: document.metadata?.statusCode,
- });
- if (dataLayerCredits !== null) {
- return dataLayerCredits + threatScanCredits;
- }
-
let creditsToBeBilled = 1; // Assuming 1 credit per document
if (options.lockdown) {
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 88abdd894..5528a9aa8 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -29,11 +29,6 @@ import {
CREDITS_FEATURE_ID,
} from "../services/autumn/autumn.service";
import { getTeamBalance } from "../services/autumn/usage";
-import {
- getDataLayerAccessForRequest,
- getThirdPartyDataTermsRequiredResponse,
-} from "../lib/data-layer";
-import { getScrapeZDR } from "../lib/zdr-helpers";
export function checkCreditsMiddleware(
_minimum?: number,
@@ -291,38 +286,8 @@ export function blocklistMiddleware(
next: NextFunction,
) {
(async () => {
- const zeroDataRetention =
- getScrapeZDR(req.acuc?.flags) === "forced" ||
- req.body?.zeroDataRetention === true ||
- req.body?.lockdown === true;
- const dataLayerAccess =
- typeof req.body.url === "string" &&
- (await getDataLayerAccessForRequest({
- url: req.body.url,
- formats: req.body.formats,
- actions: req.body.actions,
- headers: req.body.headers,
- waitFor: req.body.waitFor,
- mobile: req.body.mobile,
- location: req.body.location,
- proxy: req.body.proxy,
- blockAds: req.body.blockAds,
- zeroDataRetention,
- lockdown: req.body.lockdown,
- flags: req.acuc?.flags ?? null,
- }));
- const canUseDataLayer =
- typeof dataLayerAccess === "object" && dataLayerAccess.allowed;
-
- if (typeof dataLayerAccess === "object" && dataLayerAccess.termsRequired) {
- if (!res.headersSent) {
- return res.status(403).json(getThirdPartyDataTermsRequiredResponse());
- }
- }
-
if (
typeof req.body.url === "string" &&
- !canUseDataLayer &&
isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {
team_id: req.acuc?.team_id ?? null,
origin: typeof req.body.origin === "string" ? req.body.origin : null,
diff --git a/apps/api/src/scraper/scrapeURL/engines/data-layer.ts b/apps/api/src/scraper/scrapeURL/engines/data-layer.ts
deleted file mode 100644
index 53275c2e8..000000000
--- a/apps/api/src/scraper/scrapeURL/engines/data-layer.ts
+++ /dev/null
@@ -1,183 +0,0 @@
-import { z } from "zod";
-
-import { Meta } from "..";
-import { EngineScrapeResult } from ".";
-import {
- getDataLayerRequestLogContext,
- getDataLayerResponseLogContext,
- isSuccessfulDataLayerStatusCode,
-} from "../../../lib/data-layer";
-import { setSpanAttributes, withSpan } from "../../../lib/otel-tracer";
-import { robustFetch } from "../lib/fetch";
-import { EngineError } from "../error";
-import { fireEngineURL } from "./fire-engine/scrape";
-
-const dataLayerResultSchema = z
- .object({
- content: z.string().optional(),
- json: z.unknown().optional(),
- url: z.string().optional(),
- pageStatusCode: z.number().optional(),
- pageError: z.string().optional(),
- responseHeaders: z.record(z.string(), z.string()).optional(),
- meta: z.record(z.string(), z.unknown()).optional(),
- error: z.string().optional(),
- })
- .passthrough();
-
-const dataLayerScrapeResponseSchema = z.union([
- z
- .object({
- handled: z.literal(false),
- })
- .passthrough(),
- z
- .object({
- handled: z.literal(true),
- statusCode: z.number(),
- integrationId: z.string().optional(),
- credits: z.number().optional(),
- result: dataLayerResultSchema.optional(),
- })
- .passthrough(),
-]);
-
-function getContentType(headers?: Record<string, string>): string {
- return (
- Object.entries(headers ?? {}).find(
- ([key]) => key.toLowerCase() === "content-type",
- )?.[1] ?? ""
- );
-}
-
-export function dataLayerMaxReasonableTime(meta: Meta): number {
- return meta.options.timeout ?? 60_000;
-}
-
-export async function scrapeURLWithDataLayer(
- meta: Meta,
-): Promise<EngineScrapeResult> {
- return withSpan("engine.data-layer.scrape", async span => {
- const startTime = Date.now();
- const url = meta.rewrittenUrl ?? meta.url;
- const requestLogContext = getDataLayerRequestLogContext(url);
- const logger = meta.logger.child({ method: "scrapeURLWithDataLayer" });
-
- setSpanAttributes(span, {
- "engine.type": "data-layer",
- "engine.url": url,
- "engine.team_id": meta.internalOptions.teamId,
- });
-
- logger.info("Data layer scrape started", {
- ...requestLogContext,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- maxAge: meta.options.maxAge,
- });
-
- try {
- const response = await robustFetch({
- url: `${fireEngineURL}/v1/data-layer/scrape`,
- method: "POST",
- body: {
- url,
- formats: meta.options.formats,
- maxAge: meta.options.maxAge,
- zeroDataRetention: meta.internalOptions.zeroDataRetention,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- source: "firecrawl",
- },
- logger: logger.child({ method: "dataLayerScrape/robustFetch" }),
- tryCount: 2,
- ignoreFailureStatus: true,
- mock: meta.mock,
- abort: meta.abort.asSignal(),
- schema: dataLayerScrapeResponseSchema,
- });
-
- if (!response.handled) {
- logger.info("Data layer scrape was not handled", {
- ...requestLogContext,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- durationMs: Date.now() - startTime,
- });
- throw new EngineError("Data layer did not handle URL");
- }
-
- const responseLogContext = getDataLayerResponseLogContext(
- response.result?.meta,
- );
- const pageStatusCode =
- response.result?.pageStatusCode ?? response.statusCode;
-
- if (
- !isSuccessfulDataLayerStatusCode(response.statusCode) ||
- !isSuccessfulDataLayerStatusCode(pageStatusCode) ||
- !response.result
- ) {
- logger.warn("Data layer scrape failed", {
- ...requestLogContext,
- ...responseLogContext,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- integrationId: response.integrationId,
- statusCode: response.statusCode,
- pageStatusCode,
- durationMs: Date.now() - startTime,
- });
- throw new EngineError("Data layer request failed");
- }
-
- const contentType = getContentType(response.result.responseHeaders);
-
- logger.info("Data layer scrape completed", {
- ...requestLogContext,
- ...responseLogContext,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- integrationId: response.integrationId,
- statusCode: response.statusCode,
- durationMs: Date.now() - startTime,
- });
-
- setSpanAttributes(span, {
- "data-layer.integration_id": response.integrationId,
- "data-layer.status_code": response.statusCode,
- "data-layer.page_status_code": pageStatusCode,
- "data-layer.cache_state": responseLogContext.cacheState,
- "data-layer.cache_age_ms": responseLogContext.cacheAgeMs,
- "data-layer.duration_ms": Date.now() - startTime,
- });
-
- return {
- url: response.result.url ?? url,
- html: response.result.content ?? "",
- markdown: contentType.includes("text/markdown")
- ? response.result.content
- : undefined,
- json: response.result.json,
- error: response.result.pageError,
- statusCode: pageStatusCode,
- contentType,
- proxyUsed: "basic",
- dataLayer: {
- handled: true,
- integrationId: response.integrationId,
- },
- };
- } catch (error) {
- logger.warn("Data layer scrape errored", {
- ...requestLogContext,
- scrapeId: meta.id,
- teamId: meta.internalOptions.teamId,
- durationMs: Date.now() - startTime,
- errorMessage: error instanceof Error ? error.message : String(error),
- error,
- });
- throw error;
- }
- });
-}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index c487efee7..8a9b2f727 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -7,10 +7,6 @@ import {
scrapeURLWithFireEngineChromeCDP,
scrapeURLWithFireEngineTLSClient,
} from "./fire-engine";
-import {
- dataLayerMaxReasonableTime,
- scrapeURLWithDataLayer,
-} from "./data-layer";
import { pdfMaxReasonableTime, scrapePDF } from "./pdf";
import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";
import {
@@ -34,13 +30,8 @@ import { getPDFMaxPages } from "../../../controllers/v2/types";
import type { PdfMetadata } from "./pdf/types";
import { BrandingProfile } from "../../../types/branding";
import { BrandingNotSupportedError } from "../error";
-import {
- canUseDataLayerForRequest,
- type DataLayerScrapeMetadata,
-} from "../../../lib/data-layer";
export type Engine =
- | "data-layer"
| "fire-engine;chrome-cdp"
| "fire-engine(retry);chrome-cdp"
| "fire-engine;chrome-cdp;stealth"
@@ -171,13 +162,11 @@ export type EngineScrapeResult = {
proxyUsed: "basic" | "stealth";
timezone?: string;
- dataLayer?: DataLayerScrapeMetadata;
};
const engineHandlers: {
[E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;
} = {
- "data-layer": scrapeURLWithDataLayer,
index: scrapeURLWithIndex,
"index;documents": scrapeURLWithIndex,
"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,
@@ -197,7 +186,6 @@ const engineHandlers: {
const engineMRTs: {
[E in Engine]: (meta: Meta) => number;
} = {
- "data-layer": dataLayerMaxReasonableTime,
index: indexMaxReasonableTime,
"index;documents": indexMaxReasonableTime,
"fire-engine;chrome-cdp": meta =>
@@ -230,27 +218,6 @@ const engineOptions: {
quality: number;
};
} = {
- "data-layer": {
- features: {
- actions: false,
- waitFor: false,
- screenshot: false,
- "screenshot@fullScreen": false,
- pdf: false,
- document: false,
- audio: false,
- video: false,
- atsv: false,
- location: false,
- mobile: false,
- skipTlsVerification: true,
- useFastMode: true,
- stealthProxy: false,
- branding: false,
- disableAdblock: false,
- },
- quality: 2000,
- },
index: {
features: {
actions: false,
@@ -580,31 +547,6 @@ export async function buildFallbackList(meta: Meta): Promise<
unsupportedFeatures: Set<FeatureFlag>;
}[]
> {
- if (
- !meta.internalOptions.agentIndexOnly &&
- (await canUseDataLayerForRequest({
- url: meta.rewrittenUrl ?? meta.url,
- formats: meta.options.formats,
- actions: meta.options.actions,
- headers: meta.options.headers,
- waitFor: meta.options.waitFor,
- mobile: meta.options.mobile,
- location: meta.options.location,
- proxy: meta.options.proxy,
- blockAds: meta.options.blockAds,
- zeroDataRetention: meta.internalOptions.zeroDataRetention,
- lockdown: meta.options.lockdown,
- flags: meta.internalOptions.teamFlags ?? null,
- }))
- ) {
- return [
- {
- engine: "data-layer",
- unsupportedFeatures: new Set(),
- },
- ];
- }
-
const shouldPrioritizeTlsClient = meta.options.__experimental_engpicker
? (await queryEngpickerVerdict(
meta.options.__experimental_omceDomain ?? new URL(meta.url).hostname,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 1c86edf65..fb1e738f6 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -87,7 +87,6 @@ import { writeFile } from "node:fs/promises";
import path from "node:path";
import { tmpdir } from "node:os";
import { randomUUID } from "node:crypto";
-import type { DataLayerScrapeMetadata } from "../../lib/data-layer";
import {
checkUrl,
type ThreatCheckDedup,
@@ -102,7 +101,6 @@ export type ScrapeUrlResponse =
success: true;
document: Document;
unsupportedFeatures?: Set<FeatureFlag>;
- dataLayer?: DataLayerScrapeMetadata;
/**
* Threat protection decisions made for this scrape (initial domain
* check + any redirect re-checks, in order). Read by the billing layer
@@ -1073,7 +1071,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
success: true,
document,
unsupportedFeatures: result.unsupportedFeatures,
- dataLayer: engineResult.dataLayer,
};
});
}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index cac3b23f6..f3d968e02 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -99,7 +99,6 @@ import {
recordMonitorScrapeFailure,
recordMonitorScrapeSuccess,
} from "../monitoring/results";
-import type { DataLayerScrapeMetadata } from "../../lib/data-layer";
configDotenv();
@@ -119,7 +118,6 @@ async function billScrapeJob(
flags: TeamFlags,
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
- dataLayer?: DataLayerScrapeMetadata,
threatDecisions?: ThreatDecision[],
) {
let creditsToBeBilled: number | null = null;
@@ -148,7 +146,6 @@ async function billScrapeJob(
flags,
error,
unsupportedFeatures,
- dataLayer,
threatDecisions,
);
@@ -666,7 +663,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
undefined,
pipeline.unsupportedFeatures,
- pipeline.dataLayer,
pipeline.threatDecisions,
);
@@ -758,7 +754,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
undefined,
pipeline.unsupportedFeatures,
- pipeline.dataLayer,
pipeline.threatDecisions,
);
@@ -959,7 +954,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(await getACUCTeam(job.data.team_id))?.flags ?? null,
error instanceof Error ? error : null,
undefined,
- undefined,
pipeline?.threatDecisions,
);