Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

perf(imports): count Cloud request bytes without reserializing batches (#2079)

Michael D'Angelof5113e876adb
diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts
index 37cc62ae8..d72ca8bd1 100644
--- a/sdk/typescript/src/external-findings-publish.ts
+++ b/sdk/typescript/src/external-findings-publish.ts
@@ -792,6 +792,7 @@ async function preparePublicationInput(
);
requests = [];
let current: FindingImportRequest | undefined;
+ let currentBytes = 0;
for (const [index, finding] of parsed.findings.entries()) {
const previous = previousReports[index];
const environmentId =
@@ -800,31 +801,40 @@ async function preparePublicationInput(
throw new CodexSecurityError(
"Configure an authorized Cloud environment for this repository before importing findings.",
);
- const repository = {
- id: destination.id,
- repo_connector_id: destination.repo_connector_id,
- environment_id: environmentId,
- reset_marker: destination.reset_marker,
- };
const item = {
client_id: `item-${index + 1}`,
source_finding_id: finding.source_finding_id,
expected_version: previous?.version ?? 0,
evidence: finding.evidence,
};
+ const itemBytes = Buffer.byteLength(JSON.stringify(item));
if (
current &&
(current.repository.environment_id !== environmentId ||
current.items.length === 100 ||
- Buffer.byteLength(
- JSON.stringify({ ...current, items: [...current.items, item] }),
- ) > MAX_REQUEST_BYTES)
+ currentBytes + 1 + itemBytes > MAX_REQUEST_BYTES)
) {
requests.push(validateImportRequest(current));
current = undefined;
}
- current ??= { request_id: randomUUID(), repository, source, items: [] };
+ if (!current) {
+ current = {
+ request_id: randomUUID(),
+ repository: {
+ id: destination.id,
+ repo_connector_id: destination.repo_connector_id,
+ environment_id: environmentId,
+ reset_marker: destination.reset_marker,
+ },
+ source,
+ items: [],
+ };
+ currentBytes = Buffer.byteLength(JSON.stringify(current));
+ } else {
+ currentBytes++;
+ }
current.items.push(item);
+ currentBytes += itemBytes;
}
if (current) requests.push(validateImportRequest(current));
}
diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts
index 971954292..3203f29c3 100644
--- a/sdk/typescript/tests-ts/external-findings-publish.test.ts
+++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts
@@ -1461,15 +1461,95 @@ test.each(["environment-example", "replacement-environment", null])(
},
);
-test("batching preserves item order and keeps final item errors visible", async () => {
- const f = await fixture(
- Array.from({ length: 101 }, (_, index) => normalized(`vendor-${index}`)),
- );
- const prepared = await prepareExternalPublication(f.file, options, f.deps);
- expect(
- prepared.preview.requests.map((request) => request.items.length),
- ).toEqual([100, 1]);
- expect((await prepared.publish()).counts.created).toBe(101);
+test("empty or oversized normalized input stops before Cloud discovery", async () => {
+ const oversized = {
+ source_finding_id: "oversized",
+ evidence: {
+ title: "Oversized evidence",
+ severity: "high" as const,
+ source_data: { raw: "x".repeat(5 * 1024 * 1024) },
+ },
+ };
+ const prepare = (findings: (typeof oversized)[]) =>
+ prepareNormalizedExternalPublication(
+ { read: findings.length, findings, excluded: [] },
+ options,
+ {
+ environment: { CODEX_SECURITY_ENABLE_EXTERNAL_IMPORTS: "1" },
+ credentials: async () => {
+ throw new Error("Unexpected authentication");
+ },
+ },
+ );
+ await expect(prepare([])).rejects.toThrow("No supported findings");
+ await expect(prepare([oversized])).rejects.toThrow("256 KiB");
+});
+
+test.each([0, 1])(
+ "batching accounts for exactly 5 MiB plus %i UTF-8 bytes",
+ async (extraByte) => {
+ const f = await fixture();
+ const findings = Array.from({ length: 21 }, (_, index) => ({
+ source_finding_id: `vendor-${index}`,
+ evidence: {
+ title: 'Quoted "雪"\\\n🌱',
+ severity: "high" as const,
+ source_data: { padding: "" },
+ },
+ }));
+ const prepare = () =>
+ prepareNormalizedExternalPublication(
+ { read: findings.length, findings, excluded: [] },
+ { ...options, sourceKey: 'tenant/"境界"' },
+ f.deps,
+ );
+ const template = (await prepare()).preview.requests[0]!;
+ const targetBytes = 5 * 1024 * 1024 + extraByte;
+ const padding = targetBytes - Buffer.byteLength(JSON.stringify(template));
+ for (const [index, finding] of findings.entries())
+ finding.evidence.source_data.padding = "x".repeat(
+ Math.floor(padding / findings.length) +
+ Number(index < padding % findings.length),
+ );
+ findings.push({ ...findings[0]!, source_finding_id: "trailing" });
+ const { requests } = (await prepare()).preview;
+ expect(requests.map((request) => request.items.length)).toEqual(
+ extraByte === 0 ? [21, 1] : [20, 2],
+ );
+ const items = requests.flatMap((request) => request.items).slice(0, -1);
+ expect(Buffer.byteLength(JSON.stringify({ ...requests[0], items }))).toBe(
+ targetBytes,
+ );
+ },
+);
+
+test.each([100, 101])(
+ "batching preserves order for %i items",
+ async (count) => {
+ const records = Array.from({ length: count }, (_, index) =>
+ normalized(`vendor-${index}`),
+ );
+ const f = await fixture(records);
+ const prepared = await prepareExternalPublication(f.file, options, f.deps);
+ const { requests } = prepared.preview;
+ expect(requests.map((request) => request.items.length)).toEqual(
+ count === 100 ? [100] : [100, 1],
+ );
+ expect(
+ requests.flatMap((request) =>
+ request.items.map((item) => [item.client_id, item.source_finding_id]),
+ ),
+ ).toEqual(
+ records.map((record, index) => [
+ `item-${index + 1}`,
+ record.source_finding_id,
+ ]),
+ );
+ expect((await prepared.publish()).counts.created).toBe(count);
+ },
+);
+
+test("publication keeps final item errors visible", async () => {
const errorFixture = await fixture();
errorFixture.state.finalError = true;
const cli = createCliTest(main);