OpenAI's Codex Security CLI and TypeScript SDK for finding, validating, and fixing security vulnerabilities. npm: https://www.npmjs.com/package/@openai/codex-security
TypeScript ai-securityapplication-securityclicode-scanningcodexcodex-securitycybersecuritydevsecopsnodejsnpmopenaisdksecuritytypescriptvulnerability-scanning
build(sdk): ship only the embedding tokenizer and its encoding (#2333)
Michael D'Angelobd59acf5cc4f
diff --git a/.github/actions/audit-pnpm-lockfiles/action.yml b/.github/actions/audit-pnpm-lockfiles/action.ymlindex c68a2cc33..618376a2e 100644--- a/.github/actions/audit-pnpm-lockfiles/action.yml+++ b/.github/actions/audit-pnpm-lockfiles/action.yml@@ -2,7 +2,7 @@ name: Audit pnpm lockfilesdescription: Check selected dependency trees locally against the vulnerability database.inputs:targets:- description: Space-separated workspaces to audit (mcp and/or evals).+ description: Space-separated inventories to audit (mcp, evals, tokenizer).required: trueruns:using: composite@@ -18,14 +18,17 @@ runs:run: |read -ra targets <<< "$TARGETS"for target in "${targets[@]}"; do+ echo "Auditing $target dependencies"+ sbom="$RUNNER_TEMP/$target.cdx.json"case "$target" in- mcp) scope=(--dir plugins/codex-security/mcp-app sbom --prod) ;;- evals) scope=(--dir evals/triage-finding sbom) ;;+ mcp) pnpm --dir plugins/codex-security/mcp-app sbom --prod --lockfile-only --sbom-format cyclonedx > "$sbom" ;;+ evals) pnpm --dir evals/triage-finding sbom --lockfile-only --sbom-format cyclonedx > "$sbom" ;;+ tokenizer)+ pnpm --dir sdk/typescript exec node scripts/build-embedding-tokenizer.mjs+ sbom="sdk/typescript/dist/server/embedding-tokenizer.cdx.json"+ ;;*) echo "Unknown audit target: $target" >&2; exit 1 ;;esac- echo "Auditing $target lockfile"- sbom="$RUNNER_TEMP/$target.cdx.json"- pnpm "${scope[@]}" --lockfile-only --sbom-format cyclonedx > "$sbom"trivy sbom --offline-scan --disable-telemetry --skip-version-check \--exit-code 1 --severity HIGH,CRITICAL "$sbom"donediff --git a/.github/scripts/select-ci-checks.mjs b/.github/scripts/select-ci-checks.mjsindex 71af63192..d7db15d29 100644--- a/.github/scripts/select-ci-checks.mjs+++ b/.github/scripts/select-ci-checks.mjs@@ -51,12 +51,18 @@ export function selectLockfileAudits(paths, { event, baseChanged }) {baseChanged ||paths.includes(".github/actions/audit-pnpm-lockfiles/action.yml"))- return "mcp evals";+ return "mcp evals tokenizer";return [["mcp", "plugins/codex-security/mcp-app/pnpm-lock.yaml"],["evals", "evals/triage-finding/pnpm-lock.yaml"],+ [+ "tokenizer",+ "sdk/typescript/pnpm-lock.yaml",+ "sdk/typescript/src/server/embedding-tokenizer.ts",+ "sdk/typescript/scripts/build-embedding-tokenizer.mjs",+ ],]- .filter(([, lockfile]) => paths.includes(lockfile))+ .filter(([, ...inputs]) => inputs.some((input) => paths.includes(input))).map(([target]) => target).join(" ");}diff --git a/.github/scripts/test-ci-selection.mjs b/.github/scripts/test-ci-selection.mjsindex 9a3cf4e2e..5c0d57954 100644--- a/.github/scripts/test-ci-selection.mjs+++ b/.github/scripts/test-ci-selection.mjs@@ -558,25 +558,31 @@ test("only an authenticated, successful classifier can propagate a source result}});-test("audits the changed lockfiles and checks both trees on full event runs", () => {+test("audits changed dependency inputs and checks every inventory on full event runs", () => {const pr = { event: "pull_request" };const mcp = "plugins/codex-security/mcp-app/pnpm-lock.yaml";const evals = "evals/triage-finding/pnpm-lock.yaml";assert.equal(selectLockfileAudits([mcp], pr), "mcp");assert.equal(selectLockfileAudits([evals], pr), "evals");assert.equal(selectLockfileAudits([mcp, evals], pr), "mcp evals");- assert.equal(selectLockfileAudits(["sdk/typescript/pnpm-lock.yaml"], pr), "");+ for (const input of [+ "sdk/typescript/pnpm-lock.yaml",+ "sdk/typescript/scripts/build-embedding-tokenizer.mjs",+ "sdk/typescript/src/server/embedding-tokenizer.ts",+ ])+ assert.equal(selectLockfileAudits([input], pr), "tokenizer");+ assert.equal(selectLockfileAudits(["README.md"], pr), "");for (const event of ["schedule", "workflow_dispatch", "push"])- assert.equal(selectLockfileAudits([], { event }), "mcp evals");+ assert.equal(selectLockfileAudits([], { event }), "mcp evals tokenizer");assert.equal(selectLockfileAudits([], { ...pr, baseChanged: true }),- "mcp evals",+ "mcp evals tokenizer",);assert.equal(selectLockfileAudits([".github/actions/audit-pnpm-lockfiles/action.yml"],pr,),- "mcp evals",+ "mcp evals tokenizer",);});diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.ymlindex 7d418d2cb..579357e65 100644--- a/.github/workflows/node-ci.yml+++ b/.github/workflows/node-ci.yml@@ -123,7 +123,7 @@ jobs:run: pnpm --dir sdk/typescript run format- name: Check MCP formattingrun: pnpm --dir plugins/codex-security/mcp-app exec prettier --check "**/*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"- - name: Audit changed MCP and eval lockfiles+ - name: Audit changed dependency inventoriesif: needs.classify-changes.outputs.audit-lockfiles != ''uses: ./.github/actions/audit-pnpm-lockfileswith:diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.ymlindex c0db6d361..2fd6dc7f9 100644--- a/.github/workflows/node-release.yml+++ b/.github/workflows/node-release.yml@@ -159,10 +159,10 @@ jobs:name: Install ripgreprun: .github/scripts/install-ripgrep.sh- - name: Audit bundled MCP production dependencies+ - name: Audit bundled MCP and tokenizer dependenciesuses: ./.github/actions/audit-pnpm-lockfileswith:- targets: mcp+ targets: mcp tokenizer- name: Audit production dependenciesrun: sfw pnpm --dir sdk/typescript run audit:proddiff --git a/.github/workflows/wolfi-build.yml b/.github/workflows/wolfi-build.ymlindex 6a484dc48..401ff5807 100644--- a/.github/workflows/wolfi-build.yml+++ b/.github/workflows/wolfi-build.yml@@ -63,6 +63,8 @@ jobs:with:node-version: "24.15.0"package-manager-cache: false+ - name: Test Wolfi dependency inventories+ run: python3 -m unittest discover -s docker/wolfi/tests- name: Exercise release staging on a known published scannerid: knownshell: bashdiff --git a/docker/wolfi/release-sbom.py b/docker/wolfi/release-sbom.pyindex 9010007d3..3130a7585 100644--- a/docker/wolfi/release-sbom.py+++ b/docker/wolfi/release-sbom.py@@ -38,6 +38,15 @@ def enrich(sbom, components):"referenceLocator": purl}],})+ for i, item in enumerate(components.get("bundled_tokenizer", [])):+ add(npm_ids[PurePosixPath("node_modules/@openai/codex-security")], {+ "SPDXID": f"SPDXRef-tokenizer-{i}", "name": item["name"], "versionInfo": item["version"],+ "downloadLocation": "NOASSERTION", "licenseDeclared": item["licenses"][0]["license"]["id"],+ "filesAnalyzed": False, "sourceInfo": "Bundled in the SDK embedding tokenizer.",+ "externalRefs": [{"referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl",+ "referenceLocator": item["purl"]}],+ })+for i, item in enumerate(components["bundled_native"]):path = PurePosixPath(item["path"])owner = next(npm_ids[p] for p in path.parents if p in npm_ids)diff --git a/docker/wolfi/tests/test_tokenizer_inventory.py b/docker/wolfi/tests/test_tokenizer_inventory.pynew file mode 100644index 000000000..ff769926b--- /dev/null+++ b/docker/wolfi/tests/test_tokenizer_inventory.py@@ -0,0 +1,50 @@+import json+import runpy+import unittest+from pathlib import Path+++verifier = runpy.run_path(str(Path(__file__).resolve().parents[1] / "verify-public.py"))+inventory = verifier["inventory_tokenizer"]+SCANNER = verifier["SCANNER"]+++def component(name, version):+ return {"name": name, "version": version, "type": "library", "purl": f"pkg:npm/{name}@{version}",+ "licenses": [{"license": {"id": "MIT"}}]}+++def bom(items):+ return json.dumps({"bomFormat": "CycloneDX", "components": items})+++class TokenizerInventoryTest(unittest.TestCase):+ def setUp(self):+ self.scanner = {"dependencies": {}, "devDependencies": {"js-tiktoken": "1.2.3"}}+ self.manifests = {SCANNER: self.scanner}+ self.items = [component("js-tiktoken", "1.2.3"), component("base64-js", "4.5.6")]++ def test_bundled_components(self):+ self.assertEqual(inventory(self.manifests, bom(self.items)), self.items)+ for payload in (None, bom([]), bom(self.items[:1]), bom(self.items[:1] * 2)):+ with self.subTest(payload=payload), self.assertRaises(SystemExit):+ inventory(self.manifests, payload)++ def test_invalid_identity(self):+ for change in ({"version": ""}, {"purl": "pkg:npm/[email protected]"}, {"licenses": []},+ {"type": "application"}, component("js-tiktoken", "9.0.0")):+ with self.subTest(change=change), self.assertRaises(SystemExit):+ inventory(self.manifests, bom([self.items[0] | change, self.items[1]]))++ def test_only_the_direct_installed_dependency_supports_legacy_releases(self):+ self.manifests["node_modules/js-tiktoken"] = {"name": "js-tiktoken", "version": "1.2.3"}+ with self.assertRaises(SystemExit):+ inventory(self.manifests, None)+ self.scanner["dependencies"]["js-tiktoken"] = "1.2.3"+ self.assertEqual(inventory(self.manifests, None), [])+ self.manifests[SCANNER + "/node_modules/js-tiktoken"] = {"name": "js-tiktoken", "version": "0.9.0"}+ with self.assertRaises(SystemExit):+ inventory(self.manifests, None)+ del self.manifests[SCANNER + "/node_modules/js-tiktoken"], self.manifests["node_modules/js-tiktoken"]+ with self.assertRaises(SystemExit):+ inventory(self.manifests, None)diff --git a/docker/wolfi/verify-public.py b/docker/wolfi/verify-public.pyindex 750e451f4..bd50cb2aa 100644--- a/docker/wolfi/verify-public.py+++ b/docker/wolfi/verify-public.py@@ -7,11 +7,14 @@ import jsonimport reimport tarfilefrom pathlib import Path+from urllib.parse import quoteWOLFI = "https://packages.wolfi.dev/os"REGISTRY = "https://registry.npmjs.org/"APP = "usr/lib/codex-security/"ENTRYPOINT = "/usr/local/bin/codex-security-entrypoint"+SCANNER = "node_modules/@openai/codex-security"+TOKENIZER_BOM = APP + SCANNER + "/dist/server/embedding-tokenizer.cdx.json"SOURCES = "usr/share/licenses/codex-security/SOURCES.json"APK_DATABASES = {"lib/apk/db/installed", "usr/lib/apk/db/installed"}# A package can contain other package.json files (for example, in examples/).@@ -66,7 +69,7 @@ def inspect_layer(fs):check(ent.isfile(), f"Unexpected public key format: {path}")keys[path.removeprefix("etc/apk/keys/")] = fs.extractfile(ent).read()elif ent.isfile():- if path in APK_DATABASES or path in {"etc/apk/repositories", APP + "package-lock.json", SOURCES}:+ if path in APK_DATABASES or path in {"etc/apk/repositories", APP + "package-lock.json", SOURCES, TOKENIZER_BOM}:raw[path] = fs.extractfile(ent).read()elif path.startswith(APP) and NPM_MANIFEST.fullmatch(path[len(APP):]):manifests[path[len(APP):-len("/package.json")]] = json.load(fs.extractfile(ent))@@ -121,6 +124,32 @@ def inventory_npm(manifests, packages, sources):return npm+def inventory_tokenizer(manifests, bom):+ scanner = manifests[SCANNER]+ if bom is None:+ # Older releases installed the pinned tokenizer as an ordinary npm dependency.+ pin = scanner["dependencies"].get("js-tiktoken")+ installed = next((manifests[p] for p in (SCANNER + "/node_modules/js-tiktoken", "node_modules/js-tiktoken")+ if p in manifests), {})+ check(pin and installed.get("name") == "js-tiktoken" and installed.get("version") == pin,+ "Missing tokenizer BOM or verified legacy SDK dependency")+ return []++ document = json.loads(bom)+ items = document["components"]+ check(document.get("bomFormat") == "CycloneDX" and len(items) == 2 and+ {item["name"] for item in items} == {"js-tiktoken", "base64-js"}, "Incomplete tokenizer BOM")+ for item in items:+ name, version = item["name"], item["version"]+ check(item.get("type") == "library" and isinstance(version, str) and+ re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?", version) and+ item.get("purl") == f"pkg:npm/{name}@{quote(version, safe='')}" and+ item.get("licenses") == [{"license": {"id": "MIT"}}], f"Invalid tokenizer BOM component: {name}")+ if name == "js-tiktoken":+ check(version == scanner["devDependencies"]["js-tiktoken"], "Tokenizer BOM differs from SDK build pin")+ return items++def verify(archive, lock):with tarfile.open(archive) as tar:manifest, = json.load(tar.extractfile("manifest.json"))@@ -173,6 +202,9 @@ def verify(archive, lock):"public_upstream_packages":len(apks)-local, "local_scanner_packages":local,"keyring_sha256":{k:"sha256:"+v for k,v in key_hashes.items()},"signed_build_manifest":"SHA256SUMS", "manifest_signature":"SHA256SUMS.sig"}+ tokenizer = inventory_tokenizer(manifests, raw.get(TOKENIZER_BOM))+ if tokenizer:+ components["bundled_tokenizer"] = tokenizerreturn {"verification.json": verification, "public-components.json": components}diff --git a/sdk/typescript/package.json b/sdk/typescript/package.jsonindex 96e9bf7b9..4f22ff328 100644--- a/sdk/typescript/package.json+++ b/sdk/typescript/package.json@@ -59,7 +59,7 @@"scripts": {"audit:prod": "pnpm audit --prod --audit-level high","clean": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"",- "build": "node --run clean && node scripts/generate-plugin-defaults.mjs && tsc -p tsconfig.build.json && node scripts/generate-project-config-schema.mjs",+ "build": "node --run clean && node scripts/generate-plugin-defaults.mjs && tsc -p tsconfig.build.json && node scripts/build-embedding-tokenizer.mjs && node scripts/generate-project-config-schema.mjs","build:ci": "tsc -p tsconfig.ci.json","build:evals": "node --run build && tsc -p ../../evals/tsconfig.json","build:examples": "tsc -p tsconfig.examples.json",@@ -90,7 +90,6 @@"fflate": "0.8.3","incur": "0.7.1","ink": "8.0.0",- "js-tiktoken": "1.0.21","papaparse": "5.7.0","pdfjs-dist": "6.4.299","react": "19.3.1",@@ -113,6 +112,7 @@"esbuild": "0.28.2","fast-check": "4.10.2","ink-testing-library": "4.0.0",+ "js-tiktoken": "1.0.21","json-schema-to-typescript": "16.0.0","json5": "2.2.3","prettier": "3.9.10",diff --git a/sdk/typescript/pnpm-lock.yaml b/sdk/typescript/pnpm-lock.yamlindex be072d1aa..73371cfd4 100644--- a/sdk/typescript/pnpm-lock.yaml+++ b/sdk/typescript/pnpm-lock.yaml@@ -35,9 +35,6 @@ importers:ink:specifier: 8.0.0version: 8.0.0(@types/[email protected])([email protected])- js-tiktoken:- specifier: 1.0.21- version: 1.0.21papaparse:specifier: 5.7.0version: 5.7.0@@ -99,6 +96,9 @@ importers:ink-testing-library:specifier: 4.0.0version: 4.0.0(@types/[email protected])+ js-tiktoken:+ specifier: 1.0.21+ version: 1.0.21json-schema-to-typescript:specifier: 16.0.0version: 16.0.0diff --git a/sdk/typescript/scripts/build-embedding-tokenizer.mjs b/sdk/typescript/scripts/build-embedding-tokenizer.mjsnew file mode 100644index 000000000..1f0e704b2--- /dev/null+++ b/sdk/typescript/scripts/build-embedding-tokenizer.mjs@@ -0,0 +1,56 @@+import { build } from "esbuild";+import { valid } from "semver";+import { readFile, writeFile } from "node:fs/promises";+import { join } from "node:path";++const output = "dist/server/embedding-tokenizer";+const bundle = await build({+ entryPoints: ["src/server/embedding-tokenizer.ts"],+ outfile: output + ".js",+ bundle: true,+ format: "esm",+ platform: "node",+ sourcemap: true,+ sourcesContent: false,+ metafile: true,+});++// Inventory the modules actually included. pnpm can nest multiple node_modules+// directories, including different versions of the same package.+const roots = new Set();+for (const path of Object.keys(bundle.metafile.inputs)) {+ const dependency = /^(.*\bnode_modules\/(?:@[^/]+\/)?[^/]+)\//u.exec(path);+ if (dependency) roots.add(dependency[1]);+}+const components = [];+const required = new Set(["base64-js", "js-tiktoken"]);+for (const directory of [...roots].sort()) {+ const { name, version, license } = JSON.parse(+ await readFile(join(directory, "package.json"), "utf8"),+ );+ if (!required.delete(name) || !valid(version) || license !== "MIT") {+ throw new Error(+ `Review the tokenizer's bundled component ${name}@${version} (${license}).`,+ );+ }+ components.push({+ type: "library",+ name,+ version,+ purl: `pkg:npm/${name}@${encodeURIComponent(version)}`,+ licenses: [{ license: { id: license } }],+ });+}+if (required.size) {+ throw new Error(+ `Missing tokenizer bundle inputs: ${[...required].join(", ")}.`,+ );+}+await writeFile(+ output + ".cdx.json",+ JSON.stringify(+ { bomFormat: "CycloneDX", specVersion: "1.6", version: 1, components },+ null,+ 2,+ ) + "\n",+);diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjsindex cceca9ecd..01e6d793b 100644--- a/sdk/typescript/scripts/package-dist-files.mjs+++ b/sdk/typescript/scripts/package-dist-files.mjs@@ -158,6 +158,7 @@ const distModules = ["saved-scan","saved-scan-bootstrap","scan-skill-paths",+ "server/embedding-tokenizer","server/embeddings","server/errors","server/sqlite-store",@@ -173,8 +174,11 @@ const distModules = ["worker-progress",];-export const packageDistFiles = distModules.flatMap((module) =>- ["js", "js.map", "d.ts", "d.ts.map"].map(- (extension) => `package/dist/${module}.${extension}`,+export const packageDistFiles = [+ "package/dist/server/embedding-tokenizer.cdx.json",+ ...distModules.flatMap((module) =>+ ["js", "js.map", "d.ts", "d.ts.map"].map(+ (extension) => `package/dist/${module}.${extension}`,+ ),),-);+];diff --git a/sdk/typescript/scripts/smoke-package.mjs b/sdk/typescript/scripts/smoke-package.mjsindex b9618f2aa..1cea9612c 100644--- a/sdk/typescript/scripts/smoke-package.mjs+++ b/sdk/typescript/scripts/smoke-package.mjs@@ -1,5 +1,6 @@import assert from "node:assert/strict";import { spawnSync } from "node:child_process";+import { createHash } from "node:crypto";import {chmod,cp,@@ -437,6 +438,38 @@ try {{ cwd: consumer },);+ const { OpenAiFindingEmbedder } = await import(+ pathToFileURL(join(installedRoot, "dist/server/api.js")).href+ );+ const finding = { description: "🔒 café 中文 ".repeat(1800) };+ const chunks = [];+ const embedder = new OpenAiFindingEmbedder(+ "synthetic",+ async (_url, init) => {+ const { model, dimensions, input } = JSON.parse(init.body);+ chunks.push(...input);+ return Response.json({+ model,+ data: input.map((_, index) => ({+ index,+ embedding: Array(dimensions).fill(1),+ })),+ });+ },+ );+ await embedder.embed([finding]);+ assert.deepEqual(+ chunks.map((chunk) => chunk.length),+ [8192, 813],+ );+ // Generated with js-tiktoken 1.0.21 using cl100k_base on JSON.stringify(finding).+ // The installed-package job runs in a clean checkout with no SDK build dependencies.+ assert.equal(+ createHash("sha256").update(JSON.stringify(chunks.flat())).digest("hex"),+ "ed8b5af712556ad433415337fcc69ea7a564c89f892855eea729c6d5c49bcf04",+ "Installed embedding token IDs differ from the upstream cl100k_base encoding.",+ );+await cp(join(packageRoot, "scripts", "fixtures", "package-consumer.ts"),join(consumer, "consumer.ts"),diff --git a/sdk/typescript/src/server/embedding-tokenizer.ts b/sdk/typescript/src/server/embedding-tokenizer.tsnew file mode 100644index 000000000..87d52990d--- /dev/null+++ b/sdk/typescript/src/server/embedding-tokenizer.ts@@ -0,0 +1,31 @@+/*!+ * MIT License+ *+ * js-tiktoken: Copyright (c) 2022 OpenAI, Shantanu Jain+ * base64-js: Copyright (c) 2014 Jameson Little+ *+ * Permission is hereby granted, free of charge, to any person obtaining a copy+ * of this software and associated documentation files (the "Software"), to deal+ * in the Software without restriction, including without limitation the rights+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+ * copies of the Software, and to permit persons to whom the Software is+ * furnished to do so, subject to the following conditions:+ *+ * The above copyright notice and this permission notice shall be included in all+ * copies or substantial portions of the Software.+ *+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+ * SOFTWARE.+ */+import { Tiktoken } from "js-tiktoken/lite";+import cl100kBase from "js-tiktoken/ranks/cl100k_base";++export function createEmbeddingEncoder(): (text: string) => number[] {+ const encoding = new Tiktoken(cl100kBase);+ return (text) => encoding.encode(text, [], []);+}diff --git a/sdk/typescript/src/server/embeddings.ts b/sdk/typescript/src/server/embeddings.tsindex 515507ca5..c9bdef116 100644--- a/sdk/typescript/src/server/embeddings.ts+++ b/sdk/typescript/src/server/embeddings.ts@@ -1,8 +1,7 @@-import { Tiktoken } from "js-tiktoken/lite";-import cl100kBase from "js-tiktoken/ranks/cl100k_base";import { errorMessage } from "../errors.js";import type { Finding } from "../models.js";import { VERSION } from "../version.js";+import { createEmbeddingEncoder } from "./embedding-tokenizer.js";import { FindingsError } from "./errors.js";import type { FindingEmbedding } from "./storage.js";@@ -23,7 +22,7 @@ interface Chunk {}export class OpenAiFindingEmbedder implements FindingEmbedder {- private readonly encoding = new Tiktoken(cl100kBase);+ private readonly encode = createEmbeddingEncoder();constructor(private readonly apiKey:@@ -51,7 +50,7 @@ export class OpenAiFindingEmbedder implements FindingEmbedder {let chunks: Chunk[] = [];let requestTokens = 0;for (const [findingIndex, finding] of findings.entries()) {- const tokens = this.encoding.encode(JSON.stringify(finding), [], []);+ const tokens = this.encode(JSON.stringify(finding));for (let start = 0; start < tokens.length; start += MAX_INPUT_TOKENS) {const chunk = tokens.slice(start, start + MAX_INPUT_TOKENS);if (diff --git a/sdk/typescript/tests-ts/embedding-tokenizer-build.test.ts b/sdk/typescript/tests-ts/embedding-tokenizer-build.test.tsnew file mode 100644index 000000000..4579a80a1--- /dev/null+++ b/sdk/typescript/tests-ts/embedding-tokenizer-build.test.ts@@ -0,0 +1,26 @@+import { execFile } from "node:child_process";+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import { promisify } from "node:util";+import { expect, test } from "bun:test";++test("rejects a tokenizer build with no inventoried packages", async () => {+ const root = await mkdtemp(join(tmpdir(), "embedding-tokenizer-"));+ try {+ await mkdir(join(root, "src/server"), { recursive: true });+ await writeFile(+ join(root, "src/server/embedding-tokenizer.ts"),+ "export {};",+ );+ const build = fileURLToPath(+ new URL("../scripts/build-embedding-tokenizer.mjs", import.meta.url),+ );+ await expect(+ promisify(execFile)("node", [build], { cwd: root }),+ ).rejects.toThrow("Missing tokenizer bundle inputs");+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});