Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

18.5 KB
Raw Blame
import {
  calculateCreditsToBeBilled,
  calculateThreatScanCredits,
} from "./scrape-billing";
import { UnsafeDomainBlockedError } from "./threat-protection/error";
import type { ThreatDecision } from "./threat-protection/types";
import {
  DNSResolutionError,
  LockdownMissError,
} from "../scraper/scrapeURL/error";

describe("calculateCreditsToBeBilled", () => {
  it("bills handled Exchange successes at the reported credit cost", async () => {
    const credits = await calculateCreditsToBeBilled(
      {
        formats: [{ type: "markdown" }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      {
        metadata: {
          statusCode: 200,
          url: "https://profiles.example/person/example-person",
          proxyUsed: "basic",
        },
      } as any,
      {
        totalCost: 0,
      } as any,
      {} as any,
      undefined,
      undefined,
      { provider: { id: "acme", creditsCost: 12, steps: [] } },
    );

    expect(credits).toBe(12);
  });

  it("adds format surcharges on top of an Exchange access", async () => {
    const credits = await calculateCreditsToBeBilled(
      {
        formats: [{ type: "json", prompt: "name" }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      {
        metadata: {
          statusCode: 200,
          url: "https://profiles.example/person/example-person",
          proxyUsed: "basic",
        },
      } as any,
      {
        totalCost: 0,
      } as any,
      {} as any,
      undefined,
      undefined,
      { provider: { id: "acme", creditsCost: 12, steps: [] } },
    );

    expect(credits).toBe(16);
  });

  it("bills an X/Twitter scrape at the Exchange's price, with no X surcharge", async () => {
    const credits = await calculateCreditsToBeBilled(
      {
        formats: [{ type: "markdown" }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      {
        metadata: {
          statusCode: 200,
          proxyUsed: "basic",
        },
      } as any,
      {
        totalCost: 0,
      } as any,
      {} as any,
      undefined,
      undefined,
      { provider: { id: "x-com", creditsCost: 5, steps: [] } },
    );

    expect(credits).toBe(5);
  });

  it("bills enhanced proxy scrapes the same as basic ones", async () => {
    const bill = (unsupportedFeatures?: Set<any>) =>
      calculateCreditsToBeBilled(
        {
          formats: [{ type: "markdown" }],
        } as any,
        {
          teamId: "team-id",
          orgId: null,
        },
        {
          metadata: {
            statusCode: 200,
            proxyUsed: "stealth",
          },
        } as any,
        {
          totalCost: 0,
        } as any,
        {} as any,
        undefined,
        unsupportedFeatures,
      );

    // No surcharge, whether or not the engine could honour Enhanced Mode (the
    // old waiver for an unsupported enhanced proxy is moot now there is
    // nothing to waive).
    expect(await bill()).toBe(1);
    expect(await bill(new Set(["stealthProxy"]))).toBe(1);
  });

  it("still bills enhanced proxy scrapes with json at 5 credits", async () => {
    const credits = await calculateCreditsToBeBilled(
      {
        formats: [{ type: "json", schema: {} }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      {
        metadata: {
          statusCode: 200,
          proxyUsed: "stealth",
        },
      } as any,
      {
        totalCost: 0,
      } as any,
      {} as any,
    );

    expect(credits).toBe(5);
  });

  it("bills deterministic JSON at 10 credits when the script was generated", async () => {
    const credits = await calculateCreditsToBeBilled(
      {
        formats: [{ type: "deterministicJson", schema: {} }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      {
        metadata: {
          statusCode: 200,
          proxyUsed: "basic",
        },
      } as any,
      {
        totalCost: 0.01,
        calls: [
          {
            type: "other",
            model: "vertex/gemini",
            cost: 0.01,
            metadata: { module: "deterministic-json", role: "codegen" },
          },
        ],
      } as any,
      {} as any,
    );

    expect(credits).toBe(10);
  });

  it("bills nothing for DNS resolution failures", async () => {
    const credits = await calculateCreditsToBeBilled(
      {
        formats: [{ type: "markdown" }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      null,
      {
        totalCost: 0,
      } as any,
      {} as any,
      new DNSResolutionError("nonexistent.example.com"),
    );

    expect(credits).toBe(0);
  });

  it("bills 1 credit for lockdown cache misses", async () => {
    const credits = await calculateCreditsToBeBilled(
      {
        formats: [{ type: "markdown" }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      null,
      {
        totalCost: 0,
      } as any,
      {} as any,
      new LockdownMissError(),
    );

    expect(credits).toBe(1);
  });

  it("bills deterministic JSON at 3 credits when a cached script was reused", async () => {
    const credits = await calculateCreditsToBeBilled(
      {
        formats: [{ type: "deterministicJson", schema: {} }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      {
        metadata: {
          statusCode: 200,
          proxyUsed: "basic",
        },
      } as any,
      {
        totalCost: 0.001,
        calls: [
          {
            type: "other",
            model: "groq/llama",
            cost: 0.001,
            metadata: { module: "deterministic-json", role: "askLlm" },
          },
        ],
      } as any,
      {} as any,
    );

    expect(credits).toBe(3);
  });

  // =========================================
  // lockdown + json surcharge stacking
  // =========================================

  const promptInjectionGuardCall = (
    verdict: "clean" | "injection" | "none" = "clean",
  ) => ({
    type: "other",
    model: "vertex/gemini",
    cost: 0,
    metadata: {
      module: "scrapeURL",
      method: "checkForPromptInjection",
      verdict,
    },
  });

  // `guard` means the caller asked for the prompt injection check and the
  // guard actually ran. Both conditions must hold for the +4 guard fee.
  // `guardVerdicts` overrides the per-chunk verdicts the guard recorded.
  const billScrape = (args: {
    lockdown?: boolean;
    json?: boolean;
    deterministicJson?: boolean;
    guard?: boolean;
    guardVerdicts?: ("clean" | "injection" | "none")[];
  }) =>
    calculateCreditsToBeBilled(
      {
        lockdown: args.lockdown,
        checkPromptInjection: args.guard,
        formats: args.json
          ? [{ type: "json", schema: {} }]
          : args.deterministicJson
            ? [{ type: "deterministicJson", schema: {} }]
            : [{ type: "markdown" }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      {
        metadata: {
          statusCode: 200,
          proxyUsed: "basic",
        },
      } as any,
      {
        totalCost: 0,
        calls: args.guard
          ? (args.guardVerdicts ?? ["clean"]).map(promptInjectionGuardCall)
          : [],
      } as any,
      {} as any,
    );

  // Public billing docs quote 5 credits for a json scrape. Guard that figure.
  it("bills a json scrape alone at 5 credits", async () => {
    expect(await billScrape({ json: true })).toBe(5);
  });

  it("bills a lockdown scrape alone at 5 credits", async () => {
    expect(await billScrape({ lockdown: true })).toBe(5);
  });

  it("keeps the lockdown surcharge on a json scrape (9 credits)", async () => {
    expect(await billScrape({ lockdown: true, json: true })).toBe(9);
  });

  it("stacks lockdown, json, and the prompt injection guard (13 credits)", async () => {
    expect(await billScrape({ lockdown: true, json: true, guard: true })).toBe(
      13,
    );
  });

  it("bills json plus the prompt injection guard at 9 credits", async () => {
    expect(await billScrape({ json: true, guard: true })).toBe(9);
  });

  it("bills the prompt injection guard on a markdown-only scrape (5 credits)", async () => {
    expect(await billScrape({ guard: true })).toBe(5);
  });

  it("keeps the guard fee on top of the deterministicJson flat rate", async () => {
    expect(await billScrape({ deterministicJson: true, guard: true })).toBe(7);
  });

  it("drops the guard fee when any chunk got no verdict", async () => {
    expect(
      await billScrape({
        json: true,
        guard: true,
        guardVerdicts: ["clean", "none", "clean"],
      }),
    ).toBe(5);
  });

  it("drops the guard fee when no chunk got a verdict", async () => {
    expect(
      await billScrape({ json: true, guard: true, guardVerdicts: ["none"] }),
    ).toBe(5);
  });

  // =========================================
  // prompt injection guard on failed scrapes
  // =========================================

  const billFailedGuardedScrape = (
    verdicts: ("clean" | "injection" | "none")[],
  ) =>
    calculateCreditsToBeBilled(
      {
        checkPromptInjection: true,
        formats: [{ type: "markdown" }],
      } as any,
      {
        teamId: "team-id",
        orgId: null,
      },
      null,
      {
        totalCost: 0,
        calls: verdicts.map(promptInjectionGuardCall),
      } as any,
      {} as any,
    );

  it("bills 5 credits for a failed scrape whose guard scanned every chunk", async () => {
    expect(await billFailedGuardedScrape(["clean", "clean"])).toBe(5);
  });

  it("bills 5 credits for a scrape the guard blocked, even if a concurrent chunk failed", async () => {
    expect(await billFailedGuardedScrape(["injection", "none"])).toBe(5);
  });

  it("bills nothing for a failed scrape whose guard left a chunk unscanned", async () => {
    expect(await billFailedGuardedScrape(["clean", "none"])).toBe(0);
  });
});

// =========================================
// Threat protection scan fees (ENG-4985)
// =========================================

const consulted = (
  allowed = true,
  url = "http://scanned.example/",
): ThreatDecision => ({
  allowed,
  rule: allowed ? "default-allow" : "risk-score",
  url,
  domain: new URL(url).hostname,
  providerConsulted: true,
  verdict: {
    provider: "google-web-risk",
    riskScore: allowed ? 0 : 100,
    categories: [],
    fromCache: false,
    raw: {},
  },
  mode: "normal",
});

const localOnly = (
  rule: ThreatDecision["rule"],
  allowed: boolean,
): ThreatDecision => ({
  allowed,
  rule,
  url: "http://local.example/",
  domain: "local.example",
  providerConsulted: false,
  verdict: null,
  mode: "normal",
});

const billWithDecisions = (args: {
  document: { metadata: Record<string, unknown> } | null;
  error?: Error | null;
  threatDecisions?: ThreatDecision[];
}) =>
  calculateCreditsToBeBilled(
    { formats: [{ type: "markdown" }] } as any,
    { teamId: "team-id", orgId: null },
    args.document as any,
    { totalCost: 0 } as any,
    {} as any,
    args.error,
    undefined,
    undefined,
    args.threatDecisions,
  );

const successDocument = {
  metadata: { statusCode: 200, proxyUsed: "basic" },
};

describe("calculateCreditsToBeBilled — zero data retention", () => {
  const bill = (
    options: Record<string, unknown>,
    zeroDataRetention: boolean,
    flags: Record<string, unknown> = {},
  ) =>
    calculateCreditsToBeBilled(
      { formats: [{ type: "markdown" }], ...options } as any,
      { teamId: "team-id", orgId: null, zeroDataRetention },
      { metadata: { statusCode: 200, proxyUsed: "basic" } } as any,
      { totalCost: 0 } as any,
      flags as any,
    );

  it("adds 1 credit per page by default", async () => {
    expect(await bill({}, false)).toBe(1);
    expect(await bill({}, true)).toBe(2);
  });

  it("adds the team's zdrCost when it is set", async () => {
    expect(await bill({}, true, { zdrCost: 3 })).toBe(4);
    expect(await bill({}, false, { zdrCost: 3 })).toBe(1);
  });

  it("stacks on format surcharges", async () => {
    const json = { formats: [{ type: "json", schema: {} }] };
    expect(await bill(json, false)).toBe(5);
    expect(await bill(json, true)).toBe(6);
  });

  it("is not added on top of lockdown", async () => {
    expect(await bill({ lockdown: true }, true)).toBe(5);
  });

  it.each(["audio", "video"])(
    "drops the %s surcharge it refuses, but not under lockdown",
    async type => {
      const media = { formats: [{ type }] };
      expect(await bill(media, false)).toBe(5);
      expect(await bill(media, true)).toBe(2);
      expect(await bill({ ...media, lockdown: true }, true)).toBe(9);
    },
  );
});

describe("calculateThreatScanCredits", () => {
  it("bills nothing for no decisions", () => {
    expect(calculateThreatScanCredits([])).toBe(0);
  });

  it("bills +2 per unique consulted URL", () => {
    expect(calculateThreatScanCredits([consulted()])).toBe(2);
    expect(
      calculateThreatScanCredits([
        consulted(true, "http://a.example/"),
        consulted(true, "http://b.example/"),
      ]),
    ).toBe(4);
  });

  it("bills every distinct URL, including URLs sharing a domain", () => {
    expect(
      calculateThreatScanCredits([
        consulted(true, "http://one.example/a"),
        consulted(true, "http://one.example/b"),
        consulted(false, "http://one.example/c"),
      ]),
    ).toBe(6);
  });

  it("bills a URL once no matter how many decisions repeat it", () => {
    expect(
      calculateThreatScanCredits([
        consulted(true, "http://one.example/a"),
        consulted(true, "http://one.example/a"),
      ]),
    ).toBe(2);
  });

  it("bills legacy decisions without a url individually (mid-rollout jobs)", () => {
    const legacy = () => {
      const decision = consulted();
      delete (decision as Partial<ThreatDecision>).url;
      return decision;
    };
    expect(calculateThreatScanCredits([legacy(), legacy()])).toBe(4);
  });

  it("bills consulted decisions regardless of the allow/block outcome", () => {
    expect(calculateThreatScanCredits([consulted(false)])).toBe(2);
  });

  it("bills cached verdicts the same as fresh ones", () => {
    const cached = consulted();
    cached.verdict = { ...cached.verdict!, fromCache: true };
    expect(calculateThreatScanCredits([cached])).toBe(2);
  });

  it("never bills local-only decisions", () => {
    expect(
      calculateThreatScanCredits([
        localOnly("whitelist", true),
        localOnly("blacklist", false),
        localOnly("blocked-tld", false),
        localOnly("provider-failure", false),
      ]),
    ).toBe(0);
  });

  it("never bills zscaler-mode decisions, consulted or not", () => {
    const zscalerConsulted: ThreatDecision = {
      ...consulted(false),
      mode: "zscaler",
      verdict: {
        provider: "zscaler-zia",
        riskScore: null,
        categories: ["GAMBLING"],
        fromCache: false,
        raw: {},
      },
    };
    expect(calculateThreatScanCredits([zscalerConsulted])).toBe(0);
    // Mixed with a billable normal-mode decision, only that one bills.
    expect(
      calculateThreatScanCredits([
        zscalerConsulted,
        consulted(true, "http://a.example/"),
      ]),
    ).toBe(2);
  });

  it("sums mixed decisions across URLs", () => {
    expect(
      calculateThreatScanCredits([
        consulted(true, "http://a.example/"),
        consulted(false, "http://b.example/"),
        localOnly("blacklist", false),
      ]),
    ).toBe(4);
  });
});

describe("calculateCreditsToBeBilled — threat protection scan fees", () => {
  it("adds +2 to a successful scrape with a consulted decision", async () => {
    expect(
      await billWithDecisions({
        document: successDocument,
        threatDecisions: [consulted()],
      }),
    ).toBe(3);
  });

  it("bills each scanned URL on a redirect — including same-domain", async () => {
    expect(
      await billWithDecisions({
        document: successDocument,
        threatDecisions: [
          consulted(true, "http://one.example/"),
          consulted(true, "http://one.example/landing"),
        ],
      }),
    ).toBe(5);
  });

  it("bills once when the redirect re-check resolves to the same URL", async () => {
    expect(
      await billWithDecisions({
        document: successDocument,
        threatDecisions: [
          consulted(true, "http://one.example/"),
          consulted(true, "http://one.example/"),
        ],
      }),
    ).toBe(3);
  });

  it("adds nothing for local-only decisions on success", async () => {
    expect(
      await billWithDecisions({
        document: successDocument,
        threatDecisions: [localOnly("whitelist", true)],
      }),
    ).toBe(1);
  });

  it("bills the scan fee (and no base cost) for a blocked scrape", async () => {
    const decision = consulted(false);
    expect(
      await billWithDecisions({
        document: null,
        error: new UnsafeDomainBlockedError("blocked.example.com", decision),
        threatDecisions: [decision],
      }),
    ).toBe(2);
  });

  it("bills a blocked scrape from the error decision when the decisions array is missing", async () => {
    const decision = consulted(false);
    expect(
      await billWithDecisions({
        document: null,
        error: new UnsafeDomainBlockedError("blocked.example.com", decision),
      }),
    ).toBe(2);
  });

  it("does not double-bill when the error decision is also in the decisions array", async () => {
    const initial = consulted(true, "http://a.example/");
    const redirectBlock = consulted(false, "http://b.example/");
    expect(
      await billWithDecisions({
        document: null,
        error: new UnsafeDomainBlockedError(
          "blocked.example.com",
          redirectBlock,
        ),
        threatDecisions: [initial, redirectBlock],
      }),
    ).toBe(4);
  });

  it("bills nothing for a blocked scrape decided by local-only rules", async () => {
    const decision = localOnly("blacklist", false);
    expect(
      await billWithDecisions({
        document: null,
        error: new UnsafeDomainBlockedError("blocked.example.com", decision),
        threatDecisions: [decision],
      }),
    ).toBe(0);
  });

  it("bills nothing for unrelated failures without decisions", async () => {
    expect(
      await billWithDecisions({
        document: null,
        error: new Error("engine failure"),
      }),
    ).toBe(0);
  });

  it("adds the scan fee on top of other failure billing (scan happened before the failure)", async () => {
    expect(
      await billWithDecisions({
        document: null,
        error: new Error("engine failure"),
        threatDecisions: [consulted()],
      }),
    ).toBe(2);
  });
});