Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): gate data layer access (#3900)

Abimael Martell813642f7f8cd
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 21ab47125..1d8543806 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1334,6 +1334,19 @@ export type TeamFlags = {
researchBeta?: boolean;
highlightsBeta?: boolean;
enrichBeta?: boolean;
+ professionalProfileCompanyDataBeta?: boolean;
+ organizationDataSourceAccess?: Record<
+ string,
+ {
+ status?: "enabled" | "disabled" | "suspended" | string | null;
+ termsKey?: string | null;
+ termsVersion?: string | null;
+ termsAcceptedAt?: string | null;
+ enabledAt?: string | null;
+ disabledAt?: string | null;
+ disabledReason?: string | null;
+ }
+ >;
// routes the team's new queue work to the FoundationDB backend
nuqFdb?: boolean;
} | null;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 9fab09851..6ae3cece9 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1563,6 +1563,19 @@ export type TeamFlags = {
highlightsBeta?: boolean;
menuBeta?: boolean;
enrichBeta?: boolean;
+ professionalProfileCompanyDataBeta?: boolean;
+ organizationDataSourceAccess?: Record<
+ string,
+ {
+ status?: "enabled" | "disabled" | "suspended" | string | null;
+ termsKey?: string | null;
+ termsVersion?: string | null;
+ termsAcceptedAt?: string | null;
+ enabledAt?: string | null;
+ disabledAt?: string | null;
+ disabledReason?: string | null;
+ }
+ >;
} | null;
interface RequestWithMaybeACUC<
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.ts
index 3328e107c..a8c578f61 100644
--- a/apps/api/src/db/rpc.ts
+++ b/apps/api/src/db/rpc.ts
@@ -34,7 +34,7 @@ export async function authCreditUsageChunk(
): Promise<AuthCreditUsageChunkRow[]> {
const rows = await execRows<AuthCreditUsageChunkRow>(
database,
- sql`select * from auth_credit_usage_chunk_50(input_key => ${input_key})`,
+ sql`select * from auth_credit_usage_chunk_51(input_key => ${input_key})`,
);
// api_key_id is a bigint column, so the pg driver hands it back as a string.
for (const row of rows) {
@@ -51,7 +51,7 @@ export function authCreditUsageChunkFromTeam(
): Promise<AuthCreditUsageChunkRow[]> {
return execRows(
database,
- sql`select * from auth_credit_usage_chunk_50_from_team(input_team => ${input_team})`,
+ sql`select * from auth_credit_usage_chunk_51_from_team(input_team => ${input_team})`,
);
}
diff --git a/apps/api/src/lib/data-layer.test.ts b/apps/api/src/lib/data-layer.test.ts
index ca9a16193..d2231f4f2 100644
--- a/apps/api/src/lib/data-layer.test.ts
+++ b/apps/api/src/lib/data-layer.test.ts
@@ -5,9 +5,11 @@ import { config } from "../config";
import {
canUseDataLayerForRequest,
clearDataLayerCapabilitiesForTest,
+ getDataLayerAccessForRequest,
getDataLayerRequestLogContext,
getDataLayerResponseLogContext,
getDataLayerSuccessCredits,
+ getThirdPartyDataTermsRequiredResponse,
isDataLayerSupportedUrl,
isSuccessfulDataLayerStatusCode,
isSupportedDataLayerFormatRequest,
@@ -21,6 +23,20 @@ vi.mock("undici", () => ({
const originalConfig = {
FIRE_ENGINE_BETA_URL: config.FIRE_ENGINE_BETA_URL,
};
+const ENABLED_DATA_LAYER_FLAGS = {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ provider: {
+ status: "enabled",
+ termsKey: "professional_profile_company_data",
+ termsVersion: "2026-07-03",
+ termsAcceptedAt: "2026-07-03T00:00:00.000Z",
+ enabledAt: "2026-07-03T00:00:00.000Z",
+ disabledAt: null,
+ disabledReason: null,
+ },
+ },
+};
describe("data layer routing", () => {
beforeEach(() => {
@@ -52,6 +68,28 @@ describe("data layer routing", () => {
await expect(isDataLayerSupportedUrl("not a url")).resolves.toBe(false);
});
+ it("uses route capabilities when Fire Engine advertises path-level support", async () => {
+ setDataLayerCapabilitiesForTest({
+ domains: ["profiles.example"],
+ routes: [
+ {
+ domains: ["profiles.example"],
+ pathPrefixes: ["/person/", "/company/"],
+ },
+ ],
+ });
+
+ await expect(
+ isDataLayerSupportedUrl("https://profiles.example/person/example-person"),
+ ).resolves.toBe(true);
+ await expect(
+ isDataLayerSupportedUrl("https://profiles.example/company/example"),
+ ).resolves.toBe(true);
+ await expect(
+ isDataLayerSupportedUrl("https://profiles.example/jobs/example"),
+ ).resolves.toBe(false);
+ });
+
it("caches failed Fire Engine capabilities lookups briefly", async () => {
clearDataLayerCapabilitiesForTest();
vi.mocked(fetch).mockResolvedValue({
@@ -118,12 +156,12 @@ describe("data layer routing", () => {
expect(isSupportedDataLayerFormatRequest([])).toBe(false);
});
- it("allows eligible requests through the blocklist when the team flag is enabled", async () => {
+ it("allows eligible requests through the blocklist when the org data source beta flag is enabled", async () => {
await expect(
canUseDataLayerForRequest({
url: "https://profiles.example/person/example-person",
formats: [{ type: "markdown" }],
- flags: { enrichBeta: true },
+ flags: ENABLED_DATA_LAYER_FLAGS,
}),
).resolves.toBe(true);
@@ -132,7 +170,7 @@ describe("data layer routing", () => {
url: "https://profiles.example/person/example-person",
formats: [{ type: "json" }],
actions: [{ type: "wait" }],
- flags: { enrichBeta: true },
+ flags: ENABLED_DATA_LAYER_FLAGS,
}),
).resolves.toBe(false);
@@ -141,12 +179,88 @@ describe("data layer routing", () => {
url: "https://profiles.example/person/example-person",
formats: [{ type: "json" }],
zeroDataRetention: true,
- flags: { enrichBeta: true },
+ flags: ENABLED_DATA_LAYER_FLAGS,
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("requires current Third-Party Data terms before routing", async () => {
+ await expect(
+ getDataLayerAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: { professionalProfileCompanyDataBeta: true },
+ }),
+ ).resolves.toEqual({ allowed: false, termsRequired: true });
+
+ expect(getThirdPartyDataTermsRequiredResponse()).toMatchObject({
+ success: false,
+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
+ requiresAction: {
+ type: "accept_terms",
+ terms: "professional_profile_company_data",
+ version: "2026-07-03",
+ },
+ });
+
+ await expect(
+ canUseDataLayerForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: { professionalProfileCompanyDataBeta: true },
+ }),
+ ).resolves.toBe(false);
+
+ await expect(
+ canUseDataLayerForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: { professionalProfileCompanyDataBeta: true },
}),
).resolves.toBe(false);
});
- it("does not bypass unless the team flag is enabled", async () => {
+ it("requires current terms when organization data source access is stale", async () => {
+ await expect(
+ getDataLayerAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ provider: {
+ status: "enabled",
+ termsKey: "professional_profile_company_data",
+ termsVersion: "2026-07-02",
+ },
+ },
+ },
+ }),
+ ).resolves.toEqual({ allowed: false, termsRequired: true });
+ });
+
+ it("does not route or prompt for terms when organization data source access is disabled", async () => {
+ await expect(
+ getDataLayerAccessForRequest({
+ url: "https://profiles.example/person/example-person",
+ formats: [{ type: "markdown" }],
+ flags: {
+ professionalProfileCompanyDataBeta: true,
+ organizationDataSourceAccess: {
+ provider: {
+ status: "disabled",
+ termsKey: "professional_profile_company_data",
+ termsVersion: "2026-07-03",
+ disabledAt: "2026-07-04T00:00:00.000Z",
+ disabledReason: "customer_disabled",
+ },
+ },
+ },
+ }),
+ ).resolves.toEqual({ allowed: false, termsRequired: false });
+ });
+
+ it("does not bypass unless the org data source beta flag is enabled", async () => {
await expect(
canUseDataLayerForRequest({
url: "https://profiles.example/person/example-person",
@@ -158,7 +272,7 @@ describe("data layer routing", () => {
canUseDataLayerForRequest({
url: "https://profiles.example/person/example-person",
formats: [{ type: "markdown" }],
- flags: { enrichBeta: false },
+ flags: { professionalProfileCompanyDataBeta: false },
}),
).resolves.toBe(false);
});
@@ -170,7 +284,7 @@ describe("data layer routing", () => {
canUseDataLayerForRequest({
url: "https://profiles.example/person/example-person",
formats: [{ type: "markdown" }],
- flags: { enrichBeta: true },
+ flags: ENABLED_DATA_LAYER_FLAGS,
}),
).resolves.toBe(false);
});
diff --git a/apps/api/src/lib/data-layer.ts b/apps/api/src/lib/data-layer.ts
index b251a6422..eeedce349 100644
--- a/apps/api/src/lib/data-layer.ts
+++ b/apps/api/src/lib/data-layer.ts
@@ -5,6 +5,21 @@ import { config } from "../config";
import type { FormatObject } from "../controllers/v2/types";
import { logger as rootLogger } from "./logger";
+type OrganizationDataSourceAccessRecord = {
+ status?: string | null;
+ termsKey?: string | null;
+ termsVersion?: string | null;
+ termsAcceptedAt?: string | null;
+ enabledAt?: string | null;
+ disabledAt?: string | null;
+ disabledReason?: string | null;
+};
+
+type OrganizationDataSourceAccess = Record<
+ string,
+ OrganizationDataSourceAccessRecord | null | undefined
+>;
+
type RouteInput = {
url: string;
formats?: FormatObject[] | unknown[];
@@ -17,7 +32,10 @@ type RouteInput = {
blockAds?: boolean;
zeroDataRetention?: boolean;
lockdown?: boolean;
- flags?: { enrichBeta?: boolean } | null;
+ flags?: {
+ professionalProfileCompanyDataBeta?: boolean;
+ organizationDataSourceAccess?: OrganizationDataSourceAccess | null;
+ } | null;
};
export type DataLayerScrapeMetadata = {
@@ -27,23 +45,47 @@ export type DataLayerScrapeMetadata = {
const SUPPORTED_FORMATS = new Set(["markdown", "json", "deterministicJson"]);
const DATA_LAYER_SUCCESS_CREDITS = 15;
+const PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID =
+ "professional_profile_company_data";
+const PROFESSIONAL_PROFILE_COMPANY_DATA_BETA_FLAG =
+ "professionalProfileCompanyDataBeta";
+const THIRD_PARTY_DATA_TERMS_VERSION = "2026-07-03";
+const THIRD_PARTY_DATA_TERMS_REQUIRED_CODE = "THIRD_PARTY_DATA_TERMS_REQUIRED";
+const THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE =
+ "An organization admin must accept the Professional Profile & Company Data terms before this URL can be processed.";
const DATA_LAYER_CAPABILITIES_PATH = "/v1/data-layer/capabilities";
const DATA_LAYER_CAPABILITIES_TIMEOUT_MS = 2_000;
const DATA_LAYER_CAPABILITIES_FALLBACK_TTL_MS = 30_000;
+const dataLayerCapabilityRouteSchema = z
+ .object({
+ domains: z.string().array().optional(),
+ baseDomains: z.string().array().optional(),
+ pathPrefixes: z.string().array().optional(),
+ })
+ .passthrough();
+
const dataLayerCapabilitiesSchema = z
.object({
version: z.number().optional(),
ttlSeconds: z.number().positive().optional(),
domains: z.string().array().optional(),
baseDomains: z.string().array().optional(),
+ routes: dataLayerCapabilityRouteSchema.array().optional(),
})
.passthrough();
+type DataLayerCapabilityRoute = {
+ domains: Set<string>;
+ baseDomains: Set<string>;
+ pathPrefixes: string[];
+};
+
type DataLayerCapabilities = {
domains: Set<string>;
baseDomains: Set<string>;
+ routes: DataLayerCapabilityRoute[];
ttlMs: number;
};
@@ -59,6 +101,15 @@ function normalizeHost(host: string): string {
return host.trim().toLowerCase().replace(/\.$/, "");
}
+function normalizePathPrefix(prefix: string): string {
+ const trimmed = prefix.trim();
+ if (!trimmed) {
+ return "/";
+ }
+
+ return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
+}
+
function normalizeCapabilities(
raw: z.infer<typeof dataLayerCapabilitiesSchema>,
): DataLayerCapabilities {
@@ -70,6 +121,11 @@ function normalizeCapabilities(
return {
domains: new Set((raw.domains ?? []).map(normalizeHost)),
baseDomains: new Set((raw.baseDomains ?? []).map(normalizeHost)),
+ routes: (raw.routes ?? []).map(route => ({
+ domains: new Set((route.domains ?? []).map(normalizeHost)),
+ baseDomains: new Set((route.baseDomains ?? []).map(normalizeHost)),
+ pathPrefixes: (route.pathPrefixes ?? []).map(normalizePathPrefix),
+ })),
ttlMs: Math.max(1_000, ttlMs),
};
}
@@ -143,6 +199,14 @@ function dataLayerCapabilitiesMatchUrl(
}
const host = normalizeHost(parsed.hostname);
+ const pathname = parsed.pathname || "/";
+
+ if (capabilities.routes.length > 0) {
+ return capabilities.routes.some(route =>
+ dataLayerRouteMatchesUrl(route, host, pathname),
+ );
+ }
+
if (capabilities.domains.has(host)) {
return true;
}
@@ -156,6 +220,28 @@ function dataLayerCapabilitiesMatchUrl(
return false;
}
+function dataLayerRouteMatchesUrl(
+ route: DataLayerCapabilityRoute,
+ host: string,
+ pathname: string,
+): boolean {
+ const hostMatches =
+ route.domains.has(host) ||
+ [...route.baseDomains].some(
+ baseDomain => host === baseDomain || host.endsWith(`.${baseDomain}`),
+ );
+
+ if (!hostMatches) {
+ return false;
+ }
+
+ if (route.pathPrefixes.length === 0) {
+ return true;
+ }
+
+ return route.pathPrefixes.some(prefix => pathname.startsWith(prefix));
+}
+
export async function isDataLayerSupportedUrl(
inputUrl: string,
): Promise<boolean> {
@@ -245,10 +331,64 @@ export function isSupportedDataLayerFormatRequest(
});
}
-export async function canUseDataLayerForRequest(
- input: RouteInput,
-): Promise<boolean> {
- if (input.flags?.enrichBeta !== true) {
+function getOrganizationDataSourceAccess(
+ flags: RouteInput["flags"],
+): OrganizationDataSourceAccessRecord | null {
+ const access = flags?.organizationDataSourceAccess;
+ if (typeof access !== "object" || access === null) {
+ return null;
+ }
+
+ for (const record of Object.values(access)) {
+ if (
+ typeof record === "object" &&
+ record !== null &&
+ record.termsKey === PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID
+ ) {
+ return record;
+ }
+ }
+
+ return null;
+}
+
+function hasCurrentOrganizationDataSourceTerms(
+ access: OrganizationDataSourceAccessRecord,
+ termsKey: string,
+ version: string,
+): boolean {
+ return access.termsKey === termsKey && access.termsVersion === version;
+}
+
+type DataSourceAccessDecision = "allowed" | "terms_required" | "not_enabled";
+
+function getProfessionalProfileCompanyDataDecision(
+ flags: RouteInput["flags"],
+): DataSourceAccessDecision {
+ if (config.USE_DB_AUTHENTICATION !== true) {
+ return "allowed";
+ }
+
+ const access = getOrganizationDataSourceAccess(flags);
+ if (access) {
+ if (access.status !== "enabled") {
+ return "not_enabled";
+ }
+
+ return hasCurrentOrganizationDataSourceTerms(
+ access,
+ PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID,
+ THIRD_PARTY_DATA_TERMS_VERSION,
+ )
+ ? "allowed"
+ : "terms_required";
+ }
+
+ return "terms_required";
+}
+
+function isDataLayerEligibleRequest(input: RouteInput): boolean {
+ if (input.flags?.[PROFESSIONAL_PROFILE_COMPANY_DATA_BETA_FLAG] !== true) {
return false;
}
@@ -288,7 +428,63 @@ export async function canUseDataLayerForRequest(
return false;
}
- return isDataLayerSupportedUrl(input.url);
+ return true;
+}
+
+export async function getDataLayerAccessForRequest(input: RouteInput): Promise<
+ | {
+ allowed: true;
+ termsRequired: false;
+ }
+ | {
+ allowed: false;
+ termsRequired: boolean;
+ }
+> {
+ if (!isDataLayerEligibleRequest(input)) {
+ return { allowed: false, termsRequired: false };
+ }
+
+ const supported = await isDataLayerSupportedUrl(input.url);
+ if (!supported) {
+ return { allowed: false, termsRequired: false };
+ }
+
+ const dataSourceDecision = getProfessionalProfileCompanyDataDecision(
+ input.flags,
+ );
+ if (dataSourceDecision === "terms_required") {
+ return { allowed: false, termsRequired: true };
+ }
+ if (dataSourceDecision !== "allowed") {
+ return { allowed: false, termsRequired: false };
+ }
+
+ return { allowed: true, termsRequired: false };
+}
+
+export async function canUseDataLayerForRequest(
+ input: RouteInput,
+): Promise<boolean> {
+ return (await getDataLayerAccessForRequest(input)).allowed;
+}
+
+function getThirdPartyDataTermsSettingsUrl(): string {
+ return `${config.FIRECRAWL_DASHBOARD_URL.replace(/\/+$/, "")}/app/settings?tab=data-sources`;
+}
+
+export function getThirdPartyDataTermsRequiredResponse() {
+ return {
+ success: false,
+ code: THIRD_PARTY_DATA_TERMS_REQUIRED_CODE,
+ error: THIRD_PARTY_DATA_TERMS_REQUIRED_MESSAGE,
+ requiresAction: {
+ type: "accept_terms",
+ terms: PROFESSIONAL_PROFILE_COMPANY_DATA_TERMS_SOURCE_ID,
+ version: THIRD_PARTY_DATA_TERMS_VERSION,
+ url: getThirdPartyDataTermsSettingsUrl(),
+ },
+ };
}
export function getDataLayerSuccessCredits(input: {
@@ -314,6 +510,11 @@ export function getDataLayerSuccessCredits(input: {
export function setDataLayerCapabilitiesForTest(input: {
domains?: string[];
baseDomains?: string[];
+ routes?: {
+ domains?: string[];
+ baseDomains?: string[];
+ pathPrefixes?: string[];
+ }[];
ttlSeconds?: number;
}) {
cachedCapabilities = {
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index e5e7700ab..249b38af9 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -29,7 +29,10 @@ import {
CREDITS_FEATURE_ID,
} from "../services/autumn/autumn.service";
import { getTeamBalance } from "../services/autumn/usage";
-import { canUseDataLayerForRequest } from "../lib/data-layer";
+import {
+ getDataLayerAccessForRequest,
+ getThirdPartyDataTermsRequiredResponse,
+} from "../lib/data-layer";
import { getScrapeZDR } from "../lib/zdr-helpers";
export function checkCreditsMiddleware(
@@ -298,9 +301,9 @@ export function blocklistMiddleware(
getScrapeZDR(req.acuc?.flags) === "forced" ||
req.body?.zeroDataRetention === true ||
req.body?.lockdown === true;
- const canUseDataLayer =
+ const dataLayerAccess =
typeof req.body.url === "string" &&
- (await canUseDataLayerForRequest({
+ (await getDataLayerAccessForRequest({
url: req.body.url,
formats: req.body.formats,
actions: req.body.actions,
@@ -314,6 +317,14 @@ export function blocklistMiddleware(
lockdown: req.body.lockdown,
flags: req.acuc?.flags ?? null,
}));
+ const canUseDataLayer =
+ typeof dataLayerAccess === "object" && dataLayerAccess.allowed;
+
+ if (typeof dataLayerAccess === "object" && dataLayerAccess.termsRequired) {
+ if (!res.headersSent) {
+ return res.status(403).json(getThirdPartyDataTermsRequiredResponse());
+ }
+ }
if (
typeof req.body.url === "string" &&