Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import {
calculateCreditsToBeBilled,
calculateThreatScanCredits,
} from "./scrape-billing";
import { UnsafeDomainBlockedError } from "./threat-protection/error";
import type { ThreatDecision } from "./threat-protection/types";
import {
DNSResolutionError,
LockdownMissError,
} from "../scraper/scrapeURL/error";
describe("calculateCreditsToBeBilled", () => {
it("bills handled Exchange successes at the reported credit cost", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
orgId: null,
},
{
metadata: {
statusCode: 200,
url: "https://profiles.example/person/example-person",
proxyUsed: "basic",
},
} as any,
{
totalCost: 0,
} as any,
{} as any,
undefined,
undefined,
{ provider: { id: "acme", creditsCost: 12, steps: [] } },
);
expect(credits).toBe(12);
});
it("adds format surcharges on top of an Exchange access", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "json", prompt: "name" }],
} as any,
{
teamId: "team-id",
orgId: null,
},
{
metadata: {
statusCode: 200,
url: "https://profiles.example/person/example-person",
proxyUsed: "basic",
},
} as any,
{
totalCost: 0,
} as any,
{} as any,
undefined,
undefined,
{ provider: { id: "acme", creditsCost: 12, steps: [] } },
);
expect(credits).toBe(16);
});
it("bills an X/Twitter scrape at the Exchange's price, with no X surcharge", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
orgId: null,
},
{
metadata: {
statusCode: 200,
proxyUsed: "basic",
},
} as any,
{
totalCost: 0,
} as any,
{} as any,
undefined,
undefined,
{ provider: { id: "x-com", creditsCost: 5, steps: [] } },
);
expect(credits).toBe(5);
});
it("bills enhanced proxy scrapes the same as basic ones", async () => {
const bill = (unsupportedFeatures?: Set<any>) =>
calculateCreditsToBeBilled(
{
formats: [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
orgId: null,
},
{
metadata: {
statusCode: 200,
proxyUsed: "stealth",
},
} as any,
{
totalCost: 0,
} as any,
{} as any,
undefined,
unsupportedFeatures,
);
// No surcharge, whether or not the engine could honour Enhanced Mode (the
// old waiver for an unsupported enhanced proxy is moot now there is
// nothing to waive).
expect(await bill()).toBe(1);
expect(await bill(new Set(["stealthProxy"]))).toBe(1);
});
it("still bills enhanced proxy scrapes with json at 5 credits", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "json", schema: {} }],
} as any,
{
teamId: "team-id",
orgId: null,
},
{
metadata: {
statusCode: 200,
proxyUsed: "stealth",
},
} as any,
{
totalCost: 0,
} as any,
{} as any,
);
expect(credits).toBe(5);
});
it("bills deterministic JSON at 10 credits when the script was generated", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "deterministicJson", schema: {} }],
} as any,
{
teamId: "team-id",
orgId: null,
},
{
metadata: {
statusCode: 200,
proxyUsed: "basic",
},
} as any,
{
totalCost: 0.01,
calls: [
{
type: "other",
model: "vertex/gemini",
cost: 0.01,
metadata: { module: "deterministic-json", role: "codegen" },
},
],
} as any,
{} as any,
);
expect(credits).toBe(10);
});
it("bills nothing for DNS resolution failures", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
orgId: null,
},
null,
{
totalCost: 0,
} as any,
{} as any,
new DNSResolutionError("nonexistent.example.com"),
);
expect(credits).toBe(0);
});
it("bills 1 credit for lockdown cache misses", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
orgId: null,
},
null,
{
totalCost: 0,
} as any,
{} as any,
new LockdownMissError(),
);
expect(credits).toBe(1);
});
it("bills deterministic JSON at 3 credits when a cached script was reused", async () => {
const credits = await calculateCreditsToBeBilled(
{
formats: [{ type: "deterministicJson", schema: {} }],
} as any,
{
teamId: "team-id",
orgId: null,
},
{
metadata: {
statusCode: 200,
proxyUsed: "basic",
},
} as any,
{
totalCost: 0.001,
calls: [
{
type: "other",
model: "groq/llama",
cost: 0.001,
metadata: { module: "deterministic-json", role: "askLlm" },
},
],
} as any,
{} as any,
);
expect(credits).toBe(3);
});
// =========================================
// lockdown + json surcharge stacking
// =========================================
const promptInjectionGuardCall = (
verdict: "clean" | "injection" | "none" = "clean",
) => ({
type: "other",
model: "vertex/gemini",
cost: 0,
metadata: {
module: "scrapeURL",
method: "checkForPromptInjection",
verdict,
},
});
// `guard` means the caller asked for the prompt injection check and the
// guard actually ran. Both conditions must hold for the +4 guard fee.
// `guardVerdicts` overrides the per-chunk verdicts the guard recorded.
const billScrape = (args: {
lockdown?: boolean;
json?: boolean;
deterministicJson?: boolean;
guard?: boolean;
guardVerdicts?: ("clean" | "injection" | "none")[];
}) =>
calculateCreditsToBeBilled(
{
lockdown: args.lockdown,
checkPromptInjection: args.guard,
formats: args.json
? [{ type: "json", schema: {} }]
: args.deterministicJson
? [{ type: "deterministicJson", schema: {} }]
: [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
orgId: null,
},
{
metadata: {
statusCode: 200,
proxyUsed: "basic",
},
} as any,
{
totalCost: 0,
calls: args.guard
? (args.guardVerdicts ?? ["clean"]).map(promptInjectionGuardCall)
: [],
} as any,
{} as any,
);
// Public billing docs quote 5 credits for a json scrape. Guard that figure.
it("bills a json scrape alone at 5 credits", async () => {
expect(await billScrape({ json: true })).toBe(5);
});
it("bills a lockdown scrape alone at 5 credits", async () => {
expect(await billScrape({ lockdown: true })).toBe(5);
});
it("keeps the lockdown surcharge on a json scrape (9 credits)", async () => {
expect(await billScrape({ lockdown: true, json: true })).toBe(9);
});
it("stacks lockdown, json, and the prompt injection guard (13 credits)", async () => {
expect(await billScrape({ lockdown: true, json: true, guard: true })).toBe(
13,
);
});
it("bills json plus the prompt injection guard at 9 credits", async () => {
expect(await billScrape({ json: true, guard: true })).toBe(9);
});
it("bills the prompt injection guard on a markdown-only scrape (5 credits)", async () => {
expect(await billScrape({ guard: true })).toBe(5);
});
it("keeps the guard fee on top of the deterministicJson flat rate", async () => {
expect(await billScrape({ deterministicJson: true, guard: true })).toBe(7);
});
it("drops the guard fee when any chunk got no verdict", async () => {
expect(
await billScrape({
json: true,
guard: true,
guardVerdicts: ["clean", "none", "clean"],
}),
).toBe(5);
});
it("drops the guard fee when no chunk got a verdict", async () => {
expect(
await billScrape({ json: true, guard: true, guardVerdicts: ["none"] }),
).toBe(5);
});
// =========================================
// prompt injection guard on failed scrapes
// =========================================
const billFailedGuardedScrape = (
verdicts: ("clean" | "injection" | "none")[],
) =>
calculateCreditsToBeBilled(
{
checkPromptInjection: true,
formats: [{ type: "markdown" }],
} as any,
{
teamId: "team-id",
orgId: null,
},
null,
{
totalCost: 0,
calls: verdicts.map(promptInjectionGuardCall),
} as any,
{} as any,
);
it("bills 5 credits for a failed scrape whose guard scanned every chunk", async () => {
expect(await billFailedGuardedScrape(["clean", "clean"])).toBe(5);
});
it("bills 5 credits for a scrape the guard blocked, even if a concurrent chunk failed", async () => {
expect(await billFailedGuardedScrape(["injection", "none"])).toBe(5);
});
it("bills nothing for a failed scrape whose guard left a chunk unscanned", async () => {
expect(await billFailedGuardedScrape(["clean", "none"])).toBe(0);
});
});
// =========================================
// Threat protection scan fees (ENG-4985)
// =========================================
const consulted = (
allowed = true,
url = "http://scanned.example/",
): ThreatDecision => ({
allowed,
rule: allowed ? "default-allow" : "risk-score",
url,
domain: new URL(url).hostname,
providerConsulted: true,
verdict: {
provider: "google-web-risk",
riskScore: allowed ? 0 : 100,
categories: [],
fromCache: false,
raw: {},
},
mode: "normal",
});
const localOnly = (
rule: ThreatDecision["rule"],
allowed: boolean,
): ThreatDecision => ({
allowed,
rule,
url: "http://local.example/",
domain: "local.example",
providerConsulted: false,
verdict: null,
mode: "normal",
});
const billWithDecisions = (args: {
document: { metadata: Record<string, unknown> } | null;
error?: Error | null;
threatDecisions?: ThreatDecision[];
}) =>
calculateCreditsToBeBilled(
{ formats: [{ type: "markdown" }] } as any,
{ teamId: "team-id", orgId: null },
args.document as any,
{ totalCost: 0 } as any,
{} as any,
args.error,
undefined,
undefined,
args.threatDecisions,
);
const successDocument = {
metadata: { statusCode: 200, proxyUsed: "basic" },
};
describe("calculateCreditsToBeBilled — zero data retention", () => {
const bill = (
options: Record<string, unknown>,
zeroDataRetention: boolean,
flags: Record<string, unknown> = {},
) =>
calculateCreditsToBeBilled(
{ formats: [{ type: "markdown" }], ...options } as any,
{ teamId: "team-id", orgId: null, zeroDataRetention },
{ metadata: { statusCode: 200, proxyUsed: "basic" } } as any,
{ totalCost: 0 } as any,
flags as any,
);
it("adds 1 credit per page by default", async () => {
expect(await bill({}, false)).toBe(1);
expect(await bill({}, true)).toBe(2);
});
it("adds the team's zdrCost when it is set", async () => {
expect(await bill({}, true, { zdrCost: 3 })).toBe(4);
expect(await bill({}, false, { zdrCost: 3 })).toBe(1);
});
it("stacks on format surcharges", async () => {
const json = { formats: [{ type: "json", schema: {} }] };
expect(await bill(json, false)).toBe(5);
expect(await bill(json, true)).toBe(6);
});
it("is not added on top of lockdown", async () => {
expect(await bill({ lockdown: true }, true)).toBe(5);
});
it.each(["audio", "video"])(
"drops the %s surcharge it refuses, but not under lockdown",
async type => {
const media = { formats: [{ type }] };
expect(await bill(media, false)).toBe(5);
expect(await bill(media, true)).toBe(2);
expect(await bill({ ...media, lockdown: true }, true)).toBe(9);
},
);
});
describe("calculateThreatScanCredits", () => {
it("bills nothing for no decisions", () => {
expect(calculateThreatScanCredits([])).toBe(0);
});
it("bills +2 per unique consulted URL", () => {
expect(calculateThreatScanCredits([consulted()])).toBe(2);
expect(
calculateThreatScanCredits([
consulted(true, "http://a.example/"),
consulted(true, "http://b.example/"),
]),
).toBe(4);
});
it("bills every distinct URL, including URLs sharing a domain", () => {
expect(
calculateThreatScanCredits([
consulted(true, "http://one.example/a"),
consulted(true, "http://one.example/b"),
consulted(false, "http://one.example/c"),
]),
).toBe(6);
});
it("bills a URL once no matter how many decisions repeat it", () => {
expect(
calculateThreatScanCredits([
consulted(true, "http://one.example/a"),
consulted(true, "http://one.example/a"),
]),
).toBe(2);
});
it("bills legacy decisions without a url individually (mid-rollout jobs)", () => {
const legacy = () => {
const decision = consulted();
delete (decision as Partial<ThreatDecision>).url;
return decision;
};
expect(calculateThreatScanCredits([legacy(), legacy()])).toBe(4);
});
it("bills consulted decisions regardless of the allow/block outcome", () => {
expect(calculateThreatScanCredits([consulted(false)])).toBe(2);
});
it("bills cached verdicts the same as fresh ones", () => {
const cached = consulted();
cached.verdict = { ...cached.verdict!, fromCache: true };
expect(calculateThreatScanCredits([cached])).toBe(2);
});
it("never bills local-only decisions", () => {
expect(
calculateThreatScanCredits([
localOnly("whitelist", true),
localOnly("blacklist", false),
localOnly("blocked-tld", false),
localOnly("provider-failure", false),
]),
).toBe(0);
});
it("never bills zscaler-mode decisions, consulted or not", () => {
const zscalerConsulted: ThreatDecision = {
...consulted(false),
mode: "zscaler",
verdict: {
provider: "zscaler-zia",
riskScore: null,
categories: ["GAMBLING"],
fromCache: false,
raw: {},
},
};
expect(calculateThreatScanCredits([zscalerConsulted])).toBe(0);
// Mixed with a billable normal-mode decision, only that one bills.
expect(
calculateThreatScanCredits([
zscalerConsulted,
consulted(true, "http://a.example/"),
]),
).toBe(2);
});
it("sums mixed decisions across URLs", () => {
expect(
calculateThreatScanCredits([
consulted(true, "http://a.example/"),
consulted(false, "http://b.example/"),
localOnly("blacklist", false),
]),
).toBe(4);
});
});
describe("calculateCreditsToBeBilled — threat protection scan fees", () => {
it("adds +2 to a successful scrape with a consulted decision", async () => {
expect(
await billWithDecisions({
document: successDocument,
threatDecisions: [consulted()],
}),
).toBe(3);
});
it("bills each scanned URL on a redirect — including same-domain", async () => {
expect(
await billWithDecisions({
document: successDocument,
threatDecisions: [
consulted(true, "http://one.example/"),
consulted(true, "http://one.example/landing"),
],
}),
).toBe(5);
});
it("bills once when the redirect re-check resolves to the same URL", async () => {
expect(
await billWithDecisions({
document: successDocument,
threatDecisions: [
consulted(true, "http://one.example/"),
consulted(true, "http://one.example/"),
],
}),
).toBe(3);
});
it("adds nothing for local-only decisions on success", async () => {
expect(
await billWithDecisions({
document: successDocument,
threatDecisions: [localOnly("whitelist", true)],
}),
).toBe(1);
});
it("bills the scan fee (and no base cost) for a blocked scrape", async () => {
const decision = consulted(false);
expect(
await billWithDecisions({
document: null,
error: new UnsafeDomainBlockedError("blocked.example.com", decision),
threatDecisions: [decision],
}),
).toBe(2);
});
it("bills a blocked scrape from the error decision when the decisions array is missing", async () => {
const decision = consulted(false);
expect(
await billWithDecisions({
document: null,
error: new UnsafeDomainBlockedError("blocked.example.com", decision),
}),
).toBe(2);
});
it("does not double-bill when the error decision is also in the decisions array", async () => {
const initial = consulted(true, "http://a.example/");
const redirectBlock = consulted(false, "http://b.example/");
expect(
await billWithDecisions({
document: null,
error: new UnsafeDomainBlockedError(
"blocked.example.com",
redirectBlock,
),
threatDecisions: [initial, redirectBlock],
}),
).toBe(4);
});
it("bills nothing for a blocked scrape decided by local-only rules", async () => {
const decision = localOnly("blacklist", false);
expect(
await billWithDecisions({
document: null,
error: new UnsafeDomainBlockedError("blocked.example.com", decision),
threatDecisions: [decision],
}),
).toBe(0);
});
it("bills nothing for unrelated failures without decisions", async () => {
expect(
await billWithDecisions({
document: null,
error: new Error("engine failure"),
}),
).toBe(0);
});
it("adds the scan fee on top of other failure billing (scan happened before the failure)", async () => {
expect(
await billWithDecisions({
document: null,
error: new Error("engine failure"),
threatDecisions: [consulted()],
}),
).toBe(2);
});
});