Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { checkPermissions } from "./permissions";
import { ResolvedSafeMode } from "./safe-mode";
const strictSafeMode: ResolvedSafeMode = {
lockdown: false,
domainControls: true,
enforceRobots: true,
disableStealthProxy: true,
disableAuthentication: true,
disableSiteHandling: true,
exposeWebdriver: true,
useHeadlessUserAgent: true,
disablePlatformSelection: true,
disableCountrySelection: true,
disableAutomaticReferrer: true,
};
describe("checkPermissions β safe mode", () => {
it("passes untouched requests through", () => {
expect(checkPermissions({}, null, { safeMode: strictSafeMode })).toEqual(
{},
);
});
it("does nothing when safe mode is absent", () => {
expect(
checkPermissions({ proxy: "stealth" }, null, { safeMode: null }),
).toEqual({});
});
it.each(["stealth", "enhanced"])(
"rejects %s proxy when stealth proxy is disabled",
proxy => {
const result = checkPermissions({ proxy }, null, {
safeMode: strictSafeMode,
});
expect(result.error).toMatch(/prox/i);
expect(result.code).toBe("SAFE_MODE_BLOCKED");
},
);
it("allows stealth when the org relaxes disableStealthProxy", () => {
expect(
checkPermissions({ proxy: "stealth" }, null, {
safeMode: { ...strictSafeMode, disableStealthProxy: false },
}),
).toEqual({});
});
it("rejects ignoreRobotsTxt even when the org flag would allow it", () => {
const result = checkPermissions(
{ crawlerOptions: { ignoreRobotsTxt: true } },
{ ignoreRobots: "allowed" },
{ safeMode: strictSafeMode },
);
expect(result.code).toBe("SAFE_MODE_BLOCKED");
});
it("keeps the ignoreRobotsTxt entitlement check under lockdown", () => {
// Lockdown skips the Safe Mode robots rejection, so the generic
// enterprise gate must still apply to a non-entitled team.
const result = checkPermissions(
{ crawlerOptions: { ignoreRobotsTxt: true } },
null,
{ safeMode: { ...strictSafeMode, lockdown: true } },
);
expect(result.error).toMatch(/enterprise feature/i);
expect(result.code).toBeUndefined();
});
it("allows ignoreRobotsTxt when the org relaxes enforceRobots", () => {
expect(
checkPermissions(
{ crawlerOptions: { ignoreRobotsTxt: true } },
{ ignoreRobots: "allowed" },
{ safeMode: { ...strictSafeMode, enforceRobots: false } },
),
).toEqual({});
});
it("rejects profile, login actions, and credential headers", () => {
const requests: Parameters<typeof checkPermissions>[0][] = [
{ profile: { name: "p" } },
{ actions: [{ type: "press" }] },
{ headers: { AUTHORIZATION: "x" } },
{ headers: { "Proxy-Authorization": "x" } },
];
for (const request of requests) {
const result = checkPermissions(request, null, {
safeMode: strictSafeMode,
});
expect(result.code).toBe("SAFE_MODE_BLOCKED");
}
});
it("allows profile/login/credentials when the org relaxes disableAuthentication", () => {
const requests: Parameters<typeof checkPermissions>[0][] = [
{ profile: { name: "p" } },
{ actions: [{ type: "press" }] },
{ headers: { AUTHORIZATION: "x" } },
];
for (const request of requests) {
expect(
checkPermissions(request, null, {
safeMode: { ...strictSafeMode, disableAuthentication: false },
}),
).toEqual({});
}
});
it("allows benign actions and headers", () => {
expect(
checkPermissions(
{
actions: [{ type: "scroll" }, { type: "screenshot" }],
headers: { "User-Agent": "test" },
},
null,
{ safeMode: strictSafeMode },
),
).toEqual({});
});
it("treats threat protection as forced under domainControls", () => {
const result = checkPermissions(
{ threatProtection: { mode: "off" } },
{ threatProtection: "allowed" },
{ safeMode: strictSafeMode },
);
expect(result.error).toMatch(/cannot be disabled|disable/i);
const noFlag = checkPermissions(
{ threatProtection: { mode: "off" } },
null,
{ safeMode: { ...strictSafeMode, lockdown: true } },
);
expect(noFlag.error).toBeDefined();
expect(
checkPermissions({ threatProtection: { mode: "normal" } }, null, {
safeMode: strictSafeMode,
}),
).toEqual({});
});
it("skips every rule under effective lockdown", () => {
expect(
checkPermissions(
{
proxy: "stealth",
profile: { name: "p" },
actions: [{ type: "write" }],
headers: { Cookie: "x" },
},
null,
{ safeMode: { ...strictSafeMode, lockdown: true } },
),
).toEqual({});
});
});
describe("checkPermissions β threat protection", () => {
const requestWithOption = { threatProtection: { mode: "normal" } };
it("allows requests without a threatProtection option regardless of flags", () => {
expect(checkPermissions({}, null)).toEqual({});
expect(checkPermissions({}, { threatProtection: "disabled" })).toEqual({});
});
it("rejects a per-request option when the flag is missing or disabled", () => {
expect(checkPermissions(requestWithOption, null).error).toMatch(
/enterprise feature/,
);
expect(
checkPermissions(requestWithOption, { threatProtection: "disabled" })
.error,
).toMatch(/enterprise feature/);
});
it.each(["allowed", "forced"] as const)(
"allows a per-request option when the flag is %s",
mode => {
expect(
checkPermissions(requestWithOption, { threatProtection: mode }),
).toEqual({});
},
);
it("rejects a per-request option when the org disables overrides", () => {
const result = checkPermissions(
requestWithOption,
{ threatProtection: "allowed" },
{ threatProtectionOrgConfig: { allowRequestOverrides: false } },
);
expect(result.error).toMatch(/overrides are disabled/);
});
it("allows a per-request option when the org config allows overrides", () => {
expect(
checkPermissions(
requestWithOption,
{ threatProtection: "allowed" },
{ threatProtectionOrgConfig: { allowRequestOverrides: true } },
),
).toEqual({});
expect(
checkPermissions(
requestWithOption,
{ threatProtection: "allowed" },
{ threatProtectionOrgConfig: null },
),
).toEqual({});
});
});