Skip to content

OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security

build(sdk): ship only the embedding tokenizer and its encoding (#2333)

Michael D'Angelobd59acf5cc4f
diff --git a/.github/actions/audit-pnpm-lockfiles/action.yml b/.github/actions/audit-pnpm-lockfiles/action.yml
index c68a2cc33..618376a2e 100644
--- a/.github/actions/audit-pnpm-lockfiles/action.yml
+++ b/.github/actions/audit-pnpm-lockfiles/action.yml
@@ -2,7 +2,7 @@ name: Audit pnpm lockfiles
description: Check selected dependency trees locally against the vulnerability database.
inputs:
targets:
- description: Space-separated workspaces to audit (mcp and/or evals).
+ description: Space-separated inventories to audit (mcp, evals, tokenizer).
required: true
runs:
using: composite
@@ -18,14 +18,17 @@ runs:
run: |
read -ra targets <<< "$TARGETS"
for target in "${targets[@]}"; do
+ echo "Auditing $target dependencies"
+ sbom="$RUNNER_TEMP/$target.cdx.json"
case "$target" in
- mcp) scope=(--dir plugins/codex-security/mcp-app sbom --prod) ;;
- evals) scope=(--dir evals/triage-finding sbom) ;;
+ mcp) pnpm --dir plugins/codex-security/mcp-app sbom --prod --lockfile-only --sbom-format cyclonedx > "$sbom" ;;
+ evals) pnpm --dir evals/triage-finding sbom --lockfile-only --sbom-format cyclonedx > "$sbom" ;;
+ tokenizer)
+ pnpm --dir sdk/typescript exec node scripts/build-embedding-tokenizer.mjs
+ sbom="sdk/typescript/dist/server/embedding-tokenizer.cdx.json"
+ ;;
*) echo "Unknown audit target: $target" >&2; exit 1 ;;
esac
- echo "Auditing $target lockfile"
- sbom="$RUNNER_TEMP/$target.cdx.json"
- pnpm "${scope[@]}" --lockfile-only --sbom-format cyclonedx > "$sbom"
trivy sbom --offline-scan --disable-telemetry --skip-version-check \
--exit-code 1 --severity HIGH,CRITICAL "$sbom"
done
diff --git a/.github/scripts/select-ci-checks.mjs b/.github/scripts/select-ci-checks.mjs
index 71af63192..d7db15d29 100644
--- a/.github/scripts/select-ci-checks.mjs
+++ b/.github/scripts/select-ci-checks.mjs
@@ -51,12 +51,18 @@ export function selectLockfileAudits(paths, { event, baseChanged }) {
baseChanged ||
paths.includes(".github/actions/audit-pnpm-lockfiles/action.yml")
)
- return "mcp evals";
+ return "mcp evals tokenizer";
return [
["mcp", "plugins/codex-security/mcp-app/pnpm-lock.yaml"],
["evals", "evals/triage-finding/pnpm-lock.yaml"],
+ [
+ "tokenizer",
+ "sdk/typescript/pnpm-lock.yaml",
+ "sdk/typescript/src/server/embedding-tokenizer.ts",
+ "sdk/typescript/scripts/build-embedding-tokenizer.mjs",
+ ],
]
- .filter(([, lockfile]) => paths.includes(lockfile))
+ .filter(([, ...inputs]) => inputs.some((input) => paths.includes(input)))
.map(([target]) => target)
.join(" ");
}
diff --git a/.github/scripts/test-ci-selection.mjs b/.github/scripts/test-ci-selection.mjs
index 9a3cf4e2e..5c0d57954 100644
--- a/.github/scripts/test-ci-selection.mjs
+++ b/.github/scripts/test-ci-selection.mjs
@@ -558,25 +558,31 @@ test("only an authenticated, successful classifier can propagate a source result
}
});
-test("audits the changed lockfiles and checks both trees on full event runs", () => {
+test("audits changed dependency inputs and checks every inventory on full event runs", () => {
const pr = { event: "pull_request" };
const mcp = "plugins/codex-security/mcp-app/pnpm-lock.yaml";
const evals = "evals/triage-finding/pnpm-lock.yaml";
assert.equal(selectLockfileAudits([mcp], pr), "mcp");
assert.equal(selectLockfileAudits([evals], pr), "evals");
assert.equal(selectLockfileAudits([mcp, evals], pr), "mcp evals");
- assert.equal(selectLockfileAudits(["sdk/typescript/pnpm-lock.yaml"], pr), "");
+ for (const input of [
+ "sdk/typescript/pnpm-lock.yaml",
+ "sdk/typescript/scripts/build-embedding-tokenizer.mjs",
+ "sdk/typescript/src/server/embedding-tokenizer.ts",
+ ])
+ assert.equal(selectLockfileAudits([input], pr), "tokenizer");
+ assert.equal(selectLockfileAudits(["README.md"], pr), "");
for (const event of ["schedule", "workflow_dispatch", "push"])
- assert.equal(selectLockfileAudits([], { event }), "mcp evals");
+ assert.equal(selectLockfileAudits([], { event }), "mcp evals tokenizer");
assert.equal(
selectLockfileAudits([], { ...pr, baseChanged: true }),
- "mcp evals",
+ "mcp evals tokenizer",
);
assert.equal(
selectLockfileAudits(
[".github/actions/audit-pnpm-lockfiles/action.yml"],
pr,
),
- "mcp evals",
+ "mcp evals tokenizer",
);
});
diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml
index 7d418d2cb..579357e65 100644
--- a/.github/workflows/node-ci.yml
+++ b/.github/workflows/node-ci.yml
@@ -123,7 +123,7 @@ jobs:
run: pnpm --dir sdk/typescript run format
- name: Check MCP formatting
run: pnpm --dir plugins/codex-security/mcp-app exec prettier --check "**/*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
- - name: Audit changed MCP and eval lockfiles
+ - name: Audit changed dependency inventories
if: needs.classify-changes.outputs.audit-lockfiles != ''
uses: ./.github/actions/audit-pnpm-lockfiles
with:
diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml
index c0db6d361..2fd6dc7f9 100644
--- a/.github/workflows/node-release.yml
+++ b/.github/workflows/node-release.yml
@@ -159,10 +159,10 @@ jobs:
name: Install ripgrep
run: .github/scripts/install-ripgrep.sh
- - name: Audit bundled MCP production dependencies
+ - name: Audit bundled MCP and tokenizer dependencies
uses: ./.github/actions/audit-pnpm-lockfiles
with:
- targets: mcp
+ targets: mcp tokenizer
- name: Audit production dependencies
run: sfw pnpm --dir sdk/typescript run audit:prod
diff --git a/.github/workflows/wolfi-build.yml b/.github/workflows/wolfi-build.yml
index 6a484dc48..401ff5807 100644
--- a/.github/workflows/wolfi-build.yml
+++ b/.github/workflows/wolfi-build.yml
@@ -63,6 +63,8 @@ jobs:
with:
node-version: "24.15.0"
package-manager-cache: false
+ - name: Test Wolfi dependency inventories
+ run: python3 -m unittest discover -s docker/wolfi/tests
- name: Exercise release staging on a known published scanner
id: known
shell: bash
diff --git a/docker/wolfi/release-sbom.py b/docker/wolfi/release-sbom.py
index 9010007d3..3130a7585 100644
--- a/docker/wolfi/release-sbom.py
+++ b/docker/wolfi/release-sbom.py
@@ -38,6 +38,15 @@ def enrich(sbom, components):
"referenceLocator": purl}],
})
+ for i, item in enumerate(components.get("bundled_tokenizer", [])):
+ add(npm_ids[PurePosixPath("node_modules/@openai/codex-security")], {
+ "SPDXID": f"SPDXRef-tokenizer-{i}", "name": item["name"], "versionInfo": item["version"],
+ "downloadLocation": "NOASSERTION", "licenseDeclared": item["licenses"][0]["license"]["id"],
+ "filesAnalyzed": False, "sourceInfo": "Bundled in the SDK embedding tokenizer.",
+ "externalRefs": [{"referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl",
+ "referenceLocator": item["purl"]}],
+ })
+
for i, item in enumerate(components["bundled_native"]):
path = PurePosixPath(item["path"])
owner = next(npm_ids[p] for p in path.parents if p in npm_ids)
diff --git a/docker/wolfi/tests/test_tokenizer_inventory.py b/docker/wolfi/tests/test_tokenizer_inventory.py
new file mode 100644
index 000000000..ff769926b
--- /dev/null
+++ b/docker/wolfi/tests/test_tokenizer_inventory.py
@@ -0,0 +1,50 @@
+import json
+import runpy
+import unittest
+from pathlib import Path
+
+
+verifier = runpy.run_path(str(Path(__file__).resolve().parents[1] / "verify-public.py"))
+inventory = verifier["inventory_tokenizer"]
+SCANNER = verifier["SCANNER"]
+
+
+def component(name, version):
+ return {"name": name, "version": version, "type": "library", "purl": f"pkg:npm/{name}@{version}",
+ "licenses": [{"license": {"id": "MIT"}}]}
+
+
+def bom(items):
+ return json.dumps({"bomFormat": "CycloneDX", "components": items})
+
+
+class TokenizerInventoryTest(unittest.TestCase):
+ def setUp(self):
+ self.scanner = {"dependencies": {}, "devDependencies": {"js-tiktoken": "1.2.3"}}
+ self.manifests = {SCANNER: self.scanner}
+ self.items = [component("js-tiktoken", "1.2.3"), component("base64-js", "4.5.6")]
+
+ def test_bundled_components(self):
+ self.assertEqual(inventory(self.manifests, bom(self.items)), self.items)
+ for payload in (None, bom([]), bom(self.items[:1]), bom(self.items[:1] * 2)):
+ with self.subTest(payload=payload), self.assertRaises(SystemExit):
+ inventory(self.manifests, payload)
+
+ def test_invalid_identity(self):
+ for change in ({"version": ""}, {"purl": "pkg:npm/[email protected]"}, {"licenses": []},
+ {"type": "application"}, component("js-tiktoken", "9.0.0")):
+ with self.subTest(change=change), self.assertRaises(SystemExit):
+ inventory(self.manifests, bom([self.items[0] | change, self.items[1]]))
+
+ def test_only_the_direct_installed_dependency_supports_legacy_releases(self):
+ self.manifests["node_modules/js-tiktoken"] = {"name": "js-tiktoken", "version": "1.2.3"}
+ with self.assertRaises(SystemExit):
+ inventory(self.manifests, None)
+ self.scanner["dependencies"]["js-tiktoken"] = "1.2.3"
+ self.assertEqual(inventory(self.manifests, None), [])
+ self.manifests[SCANNER + "/node_modules/js-tiktoken"] = {"name": "js-tiktoken", "version": "0.9.0"}
+ with self.assertRaises(SystemExit):
+ inventory(self.manifests, None)
+ del self.manifests[SCANNER + "/node_modules/js-tiktoken"], self.manifests["node_modules/js-tiktoken"]
+ with self.assertRaises(SystemExit):
+ inventory(self.manifests, None)
diff --git a/docker/wolfi/verify-public.py b/docker/wolfi/verify-public.py
index 750e451f4..bd50cb2aa 100644
--- a/docker/wolfi/verify-public.py
+++ b/docker/wolfi/verify-public.py
@@ -7,11 +7,14 @@ import json
import re
import tarfile
from pathlib import Path
+from urllib.parse import quote
WOLFI = "https://packages.wolfi.dev/os"
REGISTRY = "https://registry.npmjs.org/"
APP = "usr/lib/codex-security/"
ENTRYPOINT = "/usr/local/bin/codex-security-entrypoint"
+SCANNER = "node_modules/@openai/codex-security"
+TOKENIZER_BOM = APP + SCANNER + "/dist/server/embedding-tokenizer.cdx.json"
SOURCES = "usr/share/licenses/codex-security/SOURCES.json"
APK_DATABASES = {"lib/apk/db/installed", "usr/lib/apk/db/installed"}
# A package can contain other package.json files (for example, in examples/).
@@ -66,7 +69,7 @@ def inspect_layer(fs):
check(ent.isfile(), f"Unexpected public key format: {path}")
keys[path.removeprefix("etc/apk/keys/")] = fs.extractfile(ent).read()
elif ent.isfile():
- if path in APK_DATABASES or path in {"etc/apk/repositories", APP + "package-lock.json", SOURCES}:
+ if path in APK_DATABASES or path in {"etc/apk/repositories", APP + "package-lock.json", SOURCES, TOKENIZER_BOM}:
raw[path] = fs.extractfile(ent).read()
elif path.startswith(APP) and NPM_MANIFEST.fullmatch(path[len(APP):]):
manifests[path[len(APP):-len("/package.json")]] = json.load(fs.extractfile(ent))
@@ -121,6 +124,32 @@ def inventory_npm(manifests, packages, sources):
return npm
+def inventory_tokenizer(manifests, bom):
+ scanner = manifests[SCANNER]
+ if bom is None:
+ # Older releases installed the pinned tokenizer as an ordinary npm dependency.
+ pin = scanner["dependencies"].get("js-tiktoken")
+ installed = next((manifests[p] for p in (SCANNER + "/node_modules/js-tiktoken", "node_modules/js-tiktoken")
+ if p in manifests), {})
+ check(pin and installed.get("name") == "js-tiktoken" and installed.get("version") == pin,
+ "Missing tokenizer BOM or verified legacy SDK dependency")
+ return []
+
+ document = json.loads(bom)
+ items = document["components"]
+ check(document.get("bomFormat") == "CycloneDX" and len(items) == 2 and
+ {item["name"] for item in items} == {"js-tiktoken", "base64-js"}, "Incomplete tokenizer BOM")
+ for item in items:
+ name, version = item["name"], item["version"]
+ check(item.get("type") == "library" and isinstance(version, str) and
+ re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?", version) and
+ item.get("purl") == f"pkg:npm/{name}@{quote(version, safe='')}" and
+ item.get("licenses") == [{"license": {"id": "MIT"}}], f"Invalid tokenizer BOM component: {name}")
+ if name == "js-tiktoken":
+ check(version == scanner["devDependencies"]["js-tiktoken"], "Tokenizer BOM differs from SDK build pin")
+ return items
+
+
def verify(archive, lock):
with tarfile.open(archive) as tar:
manifest, = json.load(tar.extractfile("manifest.json"))
@@ -173,6 +202,9 @@ def verify(archive, lock):
"public_upstream_packages":len(apks)-local, "local_scanner_packages":local,
"keyring_sha256":{k:"sha256:"+v for k,v in key_hashes.items()},
"signed_build_manifest":"SHA256SUMS", "manifest_signature":"SHA256SUMS.sig"}
+ tokenizer = inventory_tokenizer(manifests, raw.get(TOKENIZER_BOM))
+ if tokenizer:
+ components["bundled_tokenizer"] = tokenizer
return {"verification.json": verification, "public-components.json": components}
diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json
index 96e9bf7b9..4f22ff328 100644
--- a/sdk/typescript/package.json
+++ b/sdk/typescript/package.json
@@ -59,7 +59,7 @@
"scripts": {
"audit:prod": "pnpm audit --prod --audit-level high",
"clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"",
- "build": "node --run clean && node scripts/generate-plugin-defaults.mjs && tsc -p tsconfig.build.json && node scripts/generate-project-config-schema.mjs",
+ "build": "node --run clean && node scripts/generate-plugin-defaults.mjs && tsc -p tsconfig.build.json && node scripts/build-embedding-tokenizer.mjs && node scripts/generate-project-config-schema.mjs",
"build:ci": "tsc -p tsconfig.ci.json",
"build:evals": "node --run build && tsc -p ../../evals/tsconfig.json",
"build:examples": "tsc -p tsconfig.examples.json",
@@ -90,7 +90,6 @@
"fflate": "0.8.3",
"incur": "0.7.1",
"ink": "8.0.0",
- "js-tiktoken": "1.0.21",
"papaparse": "5.7.0",
"pdfjs-dist": "6.4.299",
"react": "19.3.1",
@@ -113,6 +112,7 @@
"esbuild": "0.28.2",
"fast-check": "4.10.2",
"ink-testing-library": "4.0.0",
+ "js-tiktoken": "1.0.21",
"json-schema-to-typescript": "16.0.0",
"json5": "2.2.3",
"prettier": "3.9.10",
diff --git a/sdk/typescript/pnpm-lock.yaml b/sdk/typescript/pnpm-lock.yaml
index be072d1aa..73371cfd4 100644
--- a/sdk/typescript/pnpm-lock.yaml
+++ b/sdk/typescript/pnpm-lock.yaml
@@ -35,9 +35,6 @@ importers:
ink:
specifier: 8.0.0
version: 8.0.0(@types/[email protected])([email protected])
- js-tiktoken:
- specifier: 1.0.21
- version: 1.0.21
papaparse:
specifier: 5.7.0
version: 5.7.0
@@ -99,6 +96,9 @@ importers:
ink-testing-library:
specifier: 4.0.0
version: 4.0.0(@types/[email protected])
+ js-tiktoken:
+ specifier: 1.0.21
+ version: 1.0.21
json-schema-to-typescript:
specifier: 16.0.0
version: 16.0.0
diff --git a/sdk/typescript/scripts/build-embedding-tokenizer.mjs b/sdk/typescript/scripts/build-embedding-tokenizer.mjs
new file mode 100644
index 000000000..1f0e704b2
--- /dev/null
+++ b/sdk/typescript/scripts/build-embedding-tokenizer.mjs
@@ -0,0 +1,56 @@
+import { build } from "esbuild";
+import { valid } from "semver";
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const output = "dist/server/embedding-tokenizer";
+const bundle = await build({
+ entryPoints: ["src/server/embedding-tokenizer.ts"],
+ outfile: output + ".js",
+ bundle: true,
+ format: "esm",
+ platform: "node",
+ sourcemap: true,
+ sourcesContent: false,
+ metafile: true,
+});
+
+// Inventory the modules actually included. pnpm can nest multiple node_modules
+// directories, including different versions of the same package.
+const roots = new Set();
+for (const path of Object.keys(bundle.metafile.inputs)) {
+ const dependency = /^(.*\bnode_modules\/(?:@[^/]+\/)?[^/]+)\//u.exec(path);
+ if (dependency) roots.add(dependency[1]);
+}
+const components = [];
+const required = new Set(["base64-js", "js-tiktoken"]);
+for (const directory of [...roots].sort()) {
+ const { name, version, license } = JSON.parse(
+ await readFile(join(directory, "package.json"), "utf8"),
+ );
+ if (!required.delete(name) || !valid(version) || license !== "MIT") {
+ throw new Error(
+ `Review the tokenizer's bundled component ${name}@${version} (${license}).`,
+ );
+ }
+ components.push({
+ type: "library",
+ name,
+ version,
+ purl: `pkg:npm/${name}@${encodeURIComponent(version)}`,
+ licenses: [{ license: { id: license } }],
+ });
+}
+if (required.size) {
+ throw new Error(
+ `Missing tokenizer bundle inputs: ${[...required].join(", ")}.`,
+ );
+}
+await writeFile(
+ output + ".cdx.json",
+ JSON.stringify(
+ { bomFormat: "CycloneDX", specVersion: "1.6", version: 1, components },
+ null,
+ 2,
+ ) + "\n",
+);
diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs
index cceca9ecd..01e6d793b 100644
--- a/sdk/typescript/scripts/package-dist-files.mjs
+++ b/sdk/typescript/scripts/package-dist-files.mjs
@@ -158,6 +158,7 @@ const distModules = [
"saved-scan",
"saved-scan-bootstrap",
"scan-skill-paths",
+ "server/embedding-tokenizer",
"server/embeddings",
"server/errors",
"server/sqlite-store",
@@ -173,8 +174,11 @@ const distModules = [
"worker-progress",
];
-export const packageDistFiles = distModules.flatMap((module) =>
- ["js", "js.map", "d.ts", "d.ts.map"].map(
- (extension) => `package/dist/${module}.${extension}`,
+export const packageDistFiles = [
+ "package/dist/server/embedding-tokenizer.cdx.json",
+ ...distModules.flatMap((module) =>
+ ["js", "js.map", "d.ts", "d.ts.map"].map(
+ (extension) => `package/dist/${module}.${extension}`,
+ ),
),
-);
+];
diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjs
index b9618f2aa..1cea9612c 100644
--- a/sdk/typescript/scripts/smoke-package.mjs
+++ b/sdk/typescript/scripts/smoke-package.mjs
@@ -1,5 +1,6 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
import {
chmod,
cp,
@@ -437,6 +438,38 @@ try {
{ cwd: consumer },
);
+ const { OpenAiFindingEmbedder } = await import(
+ pathToFileURL(join(installedRoot, "dist/server/api.js")).href
+ );
+ const finding = { description: "🔒 café 中文 ".repeat(1800) };
+ const chunks = [];
+ const embedder = new OpenAiFindingEmbedder(
+ "synthetic",
+ async (_url, init) => {
+ const { model, dimensions, input } = JSON.parse(init.body);
+ chunks.push(...input);
+ return Response.json({
+ model,
+ data: input.map((_, index) => ({
+ index,
+ embedding: Array(dimensions).fill(1),
+ })),
+ });
+ },
+ );
+ await embedder.embed([finding]);
+ assert.deepEqual(
+ chunks.map((chunk) => chunk.length),
+ [8192, 813],
+ );
+ // Generated with js-tiktoken 1.0.21 using cl100k_base on JSON.stringify(finding).
+ // The installed-package job runs in a clean checkout with no SDK build dependencies.
+ assert.equal(
+ createHash("sha256").update(JSON.stringify(chunks.flat())).digest("hex"),
+ "ed8b5af712556ad433415337fcc69ea7a564c89f892855eea729c6d5c49bcf04",
+ "Installed embedding token IDs differ from the upstream cl100k_base encoding.",
+ );
+
await cp(
join(packageRoot, "scripts", "fixtures", "package-consumer.ts"),
join(consumer, "consumer.ts"),
diff --git a/sdk/typescript/src/server/embedding-tokenizer.ts b/sdk/typescript/src/server/embedding-tokenizer.ts
new file mode 100644
index 000000000..87d52990d
--- /dev/null
+++ b/sdk/typescript/src/server/embedding-tokenizer.ts
@@ -0,0 +1,31 @@
+/*!
+ * MIT License
+ *
+ * js-tiktoken: Copyright (c) 2022 OpenAI, Shantanu Jain
+ * base64-js: Copyright (c) 2014 Jameson Little
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+import { Tiktoken } from "js-tiktoken/lite";
+import cl100kBase from "js-tiktoken/ranks/cl100k_base";
+
+export function createEmbeddingEncoder(): (text: string) => number[] {
+ const encoding = new Tiktoken(cl100kBase);
+ return (text) => encoding.encode(text, [], []);
+}
diff --git a/sdk/typescript/src/server/embeddings.ts b/sdk/typescript/src/server/embeddings.ts
index 515507ca5..c9bdef116 100644
--- a/sdk/typescript/src/server/embeddings.ts
+++ b/sdk/typescript/src/server/embeddings.ts
@@ -1,8 +1,7 @@
-import { Tiktoken } from "js-tiktoken/lite";
-import cl100kBase from "js-tiktoken/ranks/cl100k_base";
import { errorMessage } from "../errors.js";
import type { Finding } from "../models.js";
import { VERSION } from "../version.js";
+import { createEmbeddingEncoder } from "./embedding-tokenizer.js";
import { FindingsError } from "./errors.js";
import type { FindingEmbedding } from "./storage.js";
@@ -23,7 +22,7 @@ interface Chunk {
}
export class OpenAiFindingEmbedder implements FindingEmbedder {
- private readonly encoding = new Tiktoken(cl100kBase);
+ private readonly encode = createEmbeddingEncoder();
constructor(
private readonly apiKey:
@@ -51,7 +50,7 @@ export class OpenAiFindingEmbedder implements FindingEmbedder {
let chunks: Chunk[] = [];
let requestTokens = 0;
for (const [findingIndex, finding] of findings.entries()) {
- const tokens = this.encoding.encode(JSON.stringify(finding), [], []);
+ const tokens = this.encode(JSON.stringify(finding));
for (let start = 0; start < tokens.length; start += MAX_INPUT_TOKENS) {
const chunk = tokens.slice(start, start + MAX_INPUT_TOKENS);
if (
diff --git a/sdk/typescript/tests-ts/embedding-tokenizer-build.test.ts b/sdk/typescript/tests-ts/embedding-tokenizer-build.test.ts
new file mode 100644
index 000000000..4579a80a1
--- /dev/null
+++ b/sdk/typescript/tests-ts/embedding-tokenizer-build.test.ts
@@ -0,0 +1,26 @@
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { expect, test } from "bun:test";
+
+test("rejects a tokenizer build with no inventoried packages", async () => {
+ const root = await mkdtemp(join(tmpdir(), "embedding-tokenizer-"));
+ try {
+ await mkdir(join(root, "src/server"), { recursive: true });
+ await writeFile(
+ join(root, "src/server/embedding-tokenizer.ts"),
+ "export {};",
+ );
+ const build = fileURLToPath(
+ new URL("../scripts/build-embedding-tokenizer.mjs", import.meta.url),
+ );
+ await expect(
+ promisify(execFile)("node", [build], { cwd: root }),
+ ).rejects.toThrow("Missing tokenizer bundle inputs");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});