Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): gate data layer access (#3900)
Abimael Martell813642f7f8cd
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 21ab47125..1d8543806 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1334,6 +1334,19 @@ export type TeamFlags = {researchBeta?: boolean;highlightsBeta?: boolean;enrichBeta?: boolean;+ professionalProfileCompanyDataBeta?: boolean;+ organizationDataSourceAccess?: Record<+ string,+ {+ status?: "enabled" | "disabled" | "suspended" | string | null;+ termsKey?: string | null;+ termsVersion?: string | null;+ termsAcceptedAt?: string | null;+ enabledAt?: string | null;+ disabledAt?: string | null;+ disabledReason?: string | null;+ }+ >;// routes the team's new queue work to the FoundationDB backendnuqFdb?: boolean;} | null;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 9fab09851..6ae3cece9 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1563,6 +1563,19 @@ export type TeamFlags = {highlightsBeta?: boolean;menuBeta?: boolean;enrichBeta?: boolean;+ professionalProfileCompanyDataBeta?: boolean;+ organizationDataSourceAccess?: Record<+ string,+ {+ status?: "enabled" | "disabled" | "suspended" | string | null;+ termsKey?: string | null;+ termsVersion?: string | null;+ termsAcceptedAt?: string | null;+ enabledAt?: string | null;+ disabledAt?: string | null;+ disabledReason?: string | null;+ }+ >;} | null;interface RequestWithMaybeACUC<diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.tsindex 3328e107c..a8c578f61 100644--- a/apps/api/src/db/rpc.ts+++ b/apps/api/src/db/rpc.ts@@ -34,7 +34,7 @@ export async function authCreditUsageChunk(): Promise<AuthCreditUsageChunkRow[]> {const rows = await execRows<AuthCreditUsageChunkRow>(database,- sql`select * from auth_credit_usage_chunk_50(input_key => ${input_key})`,+ sql`select * from auth_credit_usage_chunk_51(input_key => ${input_key})`,);// api_key_id is a bigint column, so the pg driver hands it back as a string.for (const row of rows) {@@ -51,7 +51,7 @@ export function authCreditUsageChunkFromTeam(): Promise<AuthCreditUsageChunkRow[]> {return execRows(database,- sql`select * from auth_credit_usage_chunk_50_from_team(input_team => ${input_team})`,+ sql`select * from auth_credit_usage_chunk_51_from_team(input_team => ${input_team})`,);}diff --git a/apps/api/src/lib/data-layer.test.ts b/apps/api/src/lib/data-layer.test.tsindex ca9a16193..d2231f4f2 100644--- a/apps/api/src/lib/data-layer.test.ts+++ b/apps/api/src/lib/data-layer.test.ts@@ -5,9 +5,11 @@ import { config } from "../config";import {canUseDataLayerForRequest,clearDataLayerCapabilitiesForTest,+ getDataLayerAccessForRequest,getDataLayerRequestLogContext,getDataLayerResponseLogContext,getDataLayerSuccessCredits,+ getThirdPartyDataTermsRequiredResponse,isDataLayerSupportedUrl,isSuccessfulDataLayerStatusCode,isSupportedDataLayerFormatRequest,@@ -21,6 +23,20 @@ vi.mock("undici", () => ({const originalConfig = {FIRE_ENGINE_BETA_URL: config.FIRE_ENGINE_BETA_URL,};+const ENABLED_DATA_LAYER_FLAGS = {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ provider: {+ status: "enabled",+ termsKey: "professional_profile_company_data",+ termsVersion: "2026-07-03",+ termsAcceptedAt: "2026-07-03T00:00:00.000Z",+ enabledAt: "2026-07-03T00:00:00.000Z",+ disabledAt: null,+ disabledReason: null,+ },+ },+};describe("data layer routing", () => {beforeEach(() => {@@ -52,6 +68,28 @@ describe("data layer routing", () => {await expect(isDataLayerSupportedUrl("not a url")).resolves.toBe(false);});+ it("uses route capabilities when Fire Engine advertises path-level support", async () => {+ setDataLayerCapabilitiesForTest({+ domains: ["profiles.example"],+ routes: [+ {+ domains: ["profiles.example"],+ pathPrefixes: ["/person/", "/company/"],+ },+ ],+ });++ await expect(+ isDataLayerSupportedUrl("https://profiles.example/person/example-person"),+ ).resolves.toBe(true);+ await expect(+ isDataLayerSupportedUrl("https://profiles.example/company/example"),+ ).resolves.toBe(true);+ await expect(+ isDataLayerSupportedUrl("https://profiles.example/jobs/example"),+ ).resolves.toBe(false);+ });+it("caches failed Fire Engine capabilities lookups briefly", async () => {clearDataLayerCapabilitiesForTest();vi.mocked(fetch).mockResolvedValue({@@ -118,12 +156,12 @@ describe("data layer routing", () => {expect(isSupportedDataLayerFormatRequest([])).toBe(false);});- it("allows eligible requests through the blocklist when the team flag is enabled", async () => {+ it("allows eligible requests through the blocklist when the org data source beta flag is enabled", async () => {await expect(canUseDataLayerForRequest({url: "https://profiles.example/person/example-person",formats: [{ type: "markdown" }],- flags: { enrichBeta: true },+ flags: ENABLED_DATA_LAYER_FLAGS,}),).resolves.toBe(true);@@ -132,7 +170,7 @@ describe("data layer routing", () => {url: "https://profiles.example/person/example-person",formats: [{ type: "json" }],actions: [{ type: "wait" }],- flags: { enrichBeta: true },+ flags: ENABLED_DATA_LAYER_FLAGS,}),).resolves.toBe(false);@@ -141,12 +179,88 @@ describe("data layer routing", () => {url: "https://profiles.example/person/example-person",formats: [{ type: "json" }],zeroDataRetention: true,- flags: { enrichBeta: true },+ flags: ENABLED_DATA_LAYER_FLAGS,+ }),+ ).resolves.toBe(false);+ });++ it("requires current Third-Party Data terms before routing", async () => {+ await expect(+ getDataLayerAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: { professionalProfileCompanyDataBeta: true },+ }),+ ).resolves.toEqual({ allowed: false, termsRequired: true });++ expect(getThirdPartyDataTermsRequiredResponse()).toMatchObject({+ success: false,+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED",+ requiresAction: {+ type: "accept_terms",+ terms: "professional_profile_company_data",+ version: "2026-07-03",+ },+ });++ await expect(+ canUseDataLayerForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: { professionalProfileCompanyDataBeta: true },+ }),+ ).resolves.toBe(false);++ await expect(+ canUseDataLayerForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: { professionalProfileCompanyDataBeta: true },}),).resolves.toBe(false);});- it("does not bypass unless the team flag is enabled", async () => {+ it("requires current terms when organization data source access is stale", async () => {+ await expect(+ getDataLayerAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ provider: {+ status: "enabled",+ termsKey: "professional_profile_company_data",+ termsVersion: "2026-07-02",+ },+ },+ },+ }),+ ).resolves.toEqual({ allowed: false, termsRequired: true });+ });++ it("does not route or prompt for terms when organization data source access is disabled", async () => {+ await expect(+ getDataLayerAccessForRequest({+ url: "https://profiles.example/person/example-person",+ formats: [{ type: "markdown" }],+ flags: {+ professionalProfileCompanyDataBeta: true,+ organizationDataSourceAccess: {+ provider: {+ status: "disabled",+ termsKey: "professional_profile_company_data",+ termsVersion: "2026-07-03",+ disabledAt: "2026-07-04T00:00:00.000Z",+ disabledReason: "customer_disabled",+ },+ },+ },+ }),+ ).resolves.toEqual({ allowed: false, termsRequired: false });+ });++ it("does not bypass unless the org data source beta flag is enabled", async () => {await expect(canUseDataLayerForRequest({url: "https://profiles.example/person/example-person",@@ -158,7 +272,7 @@ describe("data layer routing", () => {canUseDataLayerForRequest({url: "https://profiles.example/person/example-person",formats: [{ type: "markdown" }],- flags: { enrichBeta: false },+ flags: { professionalProfileCompanyDataBeta: false },}),).resolves.toBe(false);});@@ -170,7 +284,7 @@ describe("data layer routing", () => {canUseDataLayerForRequest({url: "https://profiles.example/person/example-person",formats: [{ type: "markdown" }],- flags: { enrichBeta: true },+ flags: ENABLED_DATA_LAYER_FLAGS,}),).resolves.toBe(false);});diff --git a/apps/api/src/lib/data-layer.ts b/apps/api/src/lib/data-layer.tsindex b251a6422..eeedce349 100644--- a/apps/api/src/lib/data-layer.ts+++ b/apps/api/src/lib/data-layer.ts@@ -5,6 +5,21 @@ import { config } from "../config";import type { FormatObject } from "../controllers/v2/types";import { logger as rootLogger } from "./logger";+type OrganizationDataSourceAccessRecord = {+ status?: string | null;+ termsKey?: string | null;+ termsVersion?: string | null;+ termsAcceptedAt?: string | null;+ enabledAt?: string | null;+ disabledAt?: string | null;+ disabledReason?: string | null;+};++type OrganizationDataSourceAccess = Record<+ string,+ OrganizationDataSourceAccessRecord | null | undefined+>;+type RouteInput = {url: string;formats?: FormatObject[] | unknown[];@@ -17,7 +32,10 @@ type RouteInput = {blockAds?: boolean;zeroDataRetention?: boolean;lockdown?: boolean;- flags?: { enrichBeta?: boolean } | null;+ flags?: {+ professionalProfileCompanyDataBeta?: boolean;+ organizationDataSourceAccess?: OrganizationDataSourceAccess | null;+ } | null;};export type DataLayerScrapeMetadata = {@@ -27,23 +45,47 @@ export type DataLayerScrapeMetadata = {const SUPPORTED_FORMATS = new Set(["markdown", "json", "deterministicJson"]);const DATA_LAYER_SUCCESS_CREDITS = 15;+const PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID =+ "professional_profile_company_data";+const PROFESSIONAL_PROFILE_COMPANY_DATA_BETA_FLAG =+ "professionalProfileCompanyDataBeta";+const THIRD_PARTY_DATA_TERMS_VERSION = "2026-07-03";+const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";+const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE =+ "An organization admin must accept the Professional Profile & Company Data terms before this URL can be processed.";const DATA_LAYER_CAPABILITIES_PATH = "/v1/data-layer/capabilities";const DATA_LAYER_CAPABILITIES_TIMEOUT_MS = 2_000;const DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS = 30_000;+const dataLayerCapabilityRouteSchema = z+ .object({+ domains: z.string().array().optional(),+ baseDomains: z.string().array().optional(),+ pathPrefixes: z.string().array().optional(),+ })+ .passthrough();+const dataLayerCapabilitiesSchema = z.object({version: z.number().optional(),ttlSeconds: z.number().positive().optional(),domains: z.string().array().optional(),baseDomains: z.string().array().optional(),+ routes: dataLayerCapabilityRouteSchema.array().optional(),}).passthrough();+type DataLayerCapabilityRoute = {+ domains: Set<string>;+ baseDomains: Set<string>;+ pathPrefixes: string[];+};+type DataLayerCapabilities = {domains: Set<string>;baseDomains: Set<string>;+ routes: DataLayerCapabilityRoute[];ttlMs: number;};@@ -59,6 +101,15 @@ function normalizeHost(host: string): string {return host.trim().toLowerCase().replace(/\.$/, "");}+function normalizePathPrefix(prefix: string): string {+ const trimmed = prefix.trim();+ if (!trimmed) {+ return "/";+ }++ return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;+}+function normalizeCapabilities(raw: z.infer<typeof dataLayerCapabilitiesSchema>,): DataLayerCapabilities {@@ -70,6 +121,11 @@ function normalizeCapabilities(return {domains: new Set((raw.domains ?? []).map(normalizeHost)),baseDomains: new Set((raw.baseDomains ?? []).map(normalizeHost)),+ routes: (raw.routes ?? []).map(route => ({+ domains: new Set((route.domains ?? []).map(normalizeHost)),+ baseDomains: new Set((route.baseDomains ?? []).map(normalizeHost)),+ pathPrefixes: (route.pathPrefixes ?? []).map(normalizePathPrefix),+ })),ttlMs: Math.max(1_000, ttlMs),};}@@ -143,6 +199,14 @@ function dataLayerCapabilitiesMatchUrl(}const host = normalizeHost(parsed.hostname);+ const pathname = parsed.pathname || "/";++ if (capabilities.routes.length > 0) {+ return capabilities.routes.some(route =>+ dataLayerRouteMatchesUrl(route, host, pathname),+ );+ }+if (capabilities.domains.has(host)) {return true;}@@ -156,6 +220,28 @@ function dataLayerCapabilitiesMatchUrl(return false;}+function dataLayerRouteMatchesUrl(+ route: DataLayerCapabilityRoute,+ host: string,+ pathname: string,+): boolean {+ const hostMatches =+ route.domains.has(host) ||+ [...route.baseDomains].some(+ baseDomain => host === baseDomain || host.endsWith(`.${baseDomain}`),+ );++ if (!hostMatches) {+ return false;+ }++ if (route.pathPrefixes.length === 0) {+ return true;+ }++ return route.pathPrefixes.some(prefix => pathname.startsWith(prefix));+}+export async function isDataLayerSupportedUrl(inputUrl: string,): Promise<boolean> {@@ -245,10 +331,64 @@ export function isSupportedDataLayerFormatRequest(});}-export async function canUseDataLayerForRequest(- input: RouteInput,-): Promise<boolean> {- if (input.flags?.enrichBeta !== true) {+function getOrganizationDataSourceAccess(+ flags: RouteInput["flags"],+): OrganizationDataSourceAccessRecord | null {+ const access = flags?.organizationDataSourceAccess;+ if (typeof access !== "object" || access === null) {+ return null;+ }++ for (const record of Object.values(access)) {+ if (+ typeof record === "object" &&+ record !== null &&+ record.termsKey === PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID+ ) {+ return record;+ }+ }++ return null;+}++function hasCurrentOrganizationDataSourceTerms(+ access: OrganizationDataSourceAccessRecord,+ termsKey: string,+ version: string,+): boolean {+ return access.termsKey === termsKey && access.termsVersion === version;+}++type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled";++function getProfessionalProfileCompanyDataDecision(+ flags: RouteInput["flags"],+): DataSourceAccessDecision {+ if (config.USE_DB_AUTHENTICATION !== true) {+ return "allowed";+ }++ const access = getOrganizationDataSourceAccess(flags);+ if (access) {+ if (access.status !== "enabled") {+ return "not_enabled";+ }++ return hasCurrentOrganizationDataSourceTerms(+ access,+ PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID,+ THIRD_PARTY_DATA_TERMS_VERSION,+ )+ ? "allowed"+ : "terms_required";+ }++ return "terms_required";+}++function isDataLayerEligibleRequest(input: RouteInput): boolean {+ if (input.flags?.[PROFESSIONAL_PROFILE_COMPANY_DATA_BETA_FLAG] !== true) {return false;}@@ -288,7 +428,63 @@ export async function canUseDataLayerForRequest(return false;}- return isDataLayerSupportedUrl(input.url);+ return true;+}++export async function getDataLayerAccessForRequest(input: RouteInput): Promise<+ | {+ allowed: true;+ termsRequired: false;+ }+ | {+ allowed: false;+ termsRequired: boolean;+ }+> {+ if (!isDataLayerEligibleRequest(input)) {+ return { allowed: false, termsRequired: false };+ }++ const supported = await isDataLayerSupportedUrl(input.url);+ if (!supported) {+ return { allowed: false, termsRequired: false };+ }++ const dataSourceDecision = getProfessionalProfileCompanyDataDecision(+ input.flags,+ );+ if (dataSourceDecision === "terms_required") {+ return { allowed: false, termsRequired: true };+ }+ if (dataSourceDecision !== "allowed") {+ return { allowed: false, termsRequired: false };+ }++ return { allowed: true, termsRequired: false };+}++export async function canUseDataLayerForRequest(+ input: RouteInput,+): Promise<boolean> {+ return (await getDataLayerAccessForRequest(input)).allowed;+}++function getThirdPartyDataTermsSettingsUrl(): string {+ return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`;+}++export function getThirdPartyDataTermsRequiredResponse() {+ return {+ success: false,+ code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE,+ error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE,+ requiresAction: {+ type: "accept_terms",+ terms: PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID,+ version: THIRD_PARTY_DATA_TERMS_VERSION,+ url: getThirdPartyDataTermsSettingsUrl(),+ },+ };}export function getDataLayerSuccessCredits(input: {@@ -314,6 +510,11 @@ export function getDataLayerSuccessCredits(input: {export function setDataLayerCapabilitiesForTest(input: {domains?: string[];baseDomains?: string[];+ routes?: {+ domains?: string[];+ baseDomains?: string[];+ pathPrefixes?: string[];+ }[];ttlSeconds?: number;}) {cachedCapabilities = {diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex e5e7700ab..249b38af9 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -29,7 +29,10 @@ import {CREDITS_FEATURE_ID,} from "../services/autumn/autumn.service";import { getTeamBalance } from "../services/autumn/usage";-import { canUseDataLayerForRequest } from "../lib/data-layer";+import {+ getDataLayerAccessForRequest,+ getThirdPartyDataTermsRequiredResponse,+} from "../lib/data-layer";import { getScrapeZDR } from "../lib/zdr-helpers";export function checkCreditsMiddleware(@@ -298,9 +301,9 @@ export function blocklistMiddleware(getScrapeZDR(req.acuc?.flags) === "forced" ||req.body?.zeroDataRetention === true ||req.body?.lockdown === true;- const canUseDataLayer =+ const dataLayerAccess =typeof req.body.url === "string" &&- (await canUseDataLayerForRequest({+ (await getDataLayerAccessForRequest({url: req.body.url,formats: req.body.formats,actions: req.body.actions,@@ -314,6 +317,14 @@ export function blocklistMiddleware(lockdown: req.body.lockdown,flags: req.acuc?.flags ?? null,}));+ const canUseDataLayer =+ typeof dataLayerAccess === "object" && dataLayerAccess.allowed;++ if (typeof dataLayerAccess === "object" && dataLayerAccess.termsRequired) {+ if (!res.headersSent) {+ return res.status(403).json(getThirdPartyDataTermsRequiredResponse());+ }+ }if (typeof req.body.url === "string" &&