Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

6.5 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import fc from "fast-check";
import { estimateScanCost, formatUsd, tokenUsage } from "../src/cost-model.js";
import { propertyOptions } from "./support/property.js";

const rates = [
  ["gpt-5.5", 5000n, 500n, 5000n, 30000n],
  ["gpt-5.5-2026-04-23", 5000n, 500n, 5000n, 30000n],
  ["gpt-6.1-sol", 2000n, 100n, 2500n, 10000n],
  ["gpt-6-astra", 10000n, 1000n, 12500n, 50000n],
  ["gpt-6-luna", 100n, 10n, 125n, 500n],
  ["gpt-5.6", 4000n, 400n, 5000n, 20000n],
  ["gpt-5.6-sol", 4000n, 400n, 5000n, 20000n],
  ["gpt-5.6-terra", 2000n, 200n, 2500n, 12000n],
  ["gpt-5.6-luna", 200n, 20n, 250n, 1200n],
  ["openai.gpt-6-sol", 2200n, 220n, 2750n, 11000n],
  ["openai.gpt-6.1-sol", 2200n, 110n, 2750n, 11000n],
  ["global.openai.gpt-6-sol", 2000n, 200n, 2500n, 10000n],
  ["openai.gpt-daybreak-blue-5.6-sol", 4400n, 440n, 5500n, 22000n],
] as const;
const count = fc.integer({ min: 0, max: 1_000_000_000 });
const usageParts = fc.record({
  uncached: count,
  cached: count,
  written: count,
  output: count,
});

function usage(parts: {
  uncached: number;
  cached: number;
  written: number;
  output: number;
}) {
  return {
    input_tokens: parts.uncached + parts.cached + parts.written,
    cached_input_tokens: parts.cached,
    cache_write_input_tokens: parts.written,
    output_tokens: parts.output,
    reasoning_output_tokens: parts.output,
  };
}

describe("cost-model invariants", () => {
  test("rejects non-object usage and formats small costs without rounding them away", () => {
    for (const value of [undefined, null, [], "usage", 1, true]) {
      expect(tokenUsage(value)).toBeNull();
      expect(estimateScanCost("gpt-5.6-sol", value)).toBeNull();
    }
    expect(formatUsd(0)).toBe("$0.00");
    expect(formatUsd(0.000000001)).toBe("$0.000000001");
    expect(formatUsd(1234.5)).toBe("$1,234.50");
  });

  test("matches an integer nanodollar oracle for direct and Bedrock models", () => {
    fc.assert(
      fc.property(usageParts, (parts) => {
        const tokens = usage(parts);
        expect(tokenUsage(tokens)).toEqual({
          ...tokens,
          total_tokens: tokens.input_tokens + tokens.output_tokens,
        });
        for (const [
          model,
          inputRate,
          cachedRate,
          writeRate,
          outputRate,
        ] of rates) {
          const nanos =
            BigInt(parts.uncached) * inputRate +
            BigInt(parts.cached) * cachedRate +
            BigInt(parts.written) * writeRate +
            BigInt(parts.output) * outputRate;
          expect(estimateScanCost(model, tokens)).toMatchObject({
            model,
            inputTokens: tokens.input_tokens,
            cachedInputTokens: parts.cached,
            cacheWriteInputTokens: parts.written,
            outputTokens: parts.output,
            estimatedUsd: Number(nanos) / 1_000_000_000,
          });
        }
      }),
      propertyOptions,
    );
  });

  test("matches an exact integer oracle for fractional-nanodollar Bedrock Red rates", () => {
    fc.assert(
      fc.property(usageParts, (parts) => {
        const halfNanodollars =
          BigInt(parts.uncached) * 27_500n +
          BigInt(parts.cached) * 2_750n +
          BigInt(parts.written) * 34_375n +
          BigInt(parts.output) * 165_000n;
        expect(
          estimateScanCost("openai.gpt-5.6-cyber", usage(parts)),
        ).toMatchObject({
          estimatedUsd: Number(halfNanodollars) / 2_000_000_000,
          estimatedUsdRange: { max: null },
        });
      }),
      propertyOptions,
    );
  });

  test("normalizes legacy cache writes without double charging", () => {
    fc.assert(
      fc.property(usageParts, (parts) => {
        const canonical = usage(parts);
        const { cache_write_input_tokens: written, ...rest } = canonical;
        const expected = estimateScanCost("gpt-5.6-sol", canonical);
        expect(
          estimateScanCost("gpt-5.6-sol", {
            ...rest,
            cache_write_tokens: written,
          }),
        ).toEqual(expected);
        expect(
          estimateScanCost("gpt-5.6-sol", {
            ...canonical,
            cache_write_tokens: canonical.input_tokens + 1,
          }),
        ).toEqual(expected);
        if (parts.written > 0) {
          expect(
            estimateScanCost("gpt-5.6-sol", {
              ...canonical,
              cache_write_tokens: parts.written - 1,
            }),
          ).toEqual(expected);
        }
        expect(
          estimateScanCost("gpt-5.6-sol", {
            ...rest,
            cache_write_input_tokens: 0,
            cache_write_tokens: written,
          }),
        ).toEqual(expected);
      }),
      propertyOptions,
    );
  });

  test("rejects inconsistent or non-integer token accounting", () => {
    fc.assert(
      fc.property(
        usageParts,
        fc.constantFrom(
          "input_tokens",
          "cached_input_tokens",
          "cache_write_input_tokens",
          "output_tokens",
          "reasoning_output_tokens",
        ),
        fc.constantFrom(
          -1,
          0.5,
          Number.MAX_SAFE_INTEGER + 1,
          NaN,
          Infinity,
          "1",
        ),
        (parts, field, invalid) => {
          const valid = usage(parts);
          expect(
            estimateScanCost("gpt-5.6-sol", { ...valid, [field]: invalid }),
          ).toBeNull();
          expect(
            estimateScanCost("gpt-5.6-sol", {
              ...valid,
              cached_input_tokens: valid.input_tokens + 1,
            }),
          ).toBeNull();
          expect(
            estimateScanCost("gpt-5.6-sol", {
              ...valid,
              reasoning_output_tokens: valid.output_tokens + 1,
            }),
          ).toBeNull();
          expect(
            estimateScanCost("unpriced-synthetic-model", valid),
          ).toBeNull();
          expect(estimateScanCost(undefined, valid)).toBeNull();
        },
      ),
      propertyOptions,
    );
  });

  test("returns no estimate when nanodollar arithmetic would lose precision", () => {
    fc.assert(
      fc.property(
        fc.integer({ min: 0, max: Number.MAX_SAFE_INTEGER }),
        (input) => {
          const nanos = BigInt(input) * 4000n;
          const result = estimateScanCost("gpt-5.6-sol", {
            input_tokens: input,
            output_tokens: 0,
          });
          if (nanos > BigInt(Number.MAX_SAFE_INTEGER))
            expect(result).toBeNull();
          else expect(result?.estimatedUsd).toBe(Number(nanos) / 1_000_000_000);
        },
      ),
      propertyOptions,
    );
  });
});