Skip to content

Write HTML. Render video. Built for agents.

fix(catalog): compare search vectors by direction so the publish PR can pass (#5359)

Miguel Ángel6ae1af747013
diff --git a/scripts/catalog-drift.test.ts b/scripts/catalog-drift.test.ts
index e8d90f352d..d721a2d2e1 100644
--- a/scripts/catalog-drift.test.ts
+++ b/scripts/catalog-drift.test.ts
@@ -5,7 +5,10 @@ import { join } from "node:path";
import { test } from "node:test";
import { treeDifferences } from "./catalog-drift.ts";
-function tree(files: Record<string, string>): string {
+const meta = JSON.stringify({ dimensions: 4 });
+const vectors = (rows: number[][]) => Buffer.from(new Float32Array(rows.flat()).buffer);
+
+function tree(files: Record<string, string | Buffer>): string {
const root = mkdtempSync(join(tmpdir(), "catalog-drift-test-"));
for (const [name, text] of Object.entries(files)) {
mkdirSync(join(root, name, ".."), { recursive: true });
@@ -33,3 +36,55 @@ test("a stale payload, a missing file and a leftover file each turn the check re
rmSync(generated, { recursive: true });
rmSync(committed, { recursive: true });
});
+
+test("vectors that differ only by float noise between machines are not stale", () => {
+ // Second row: cosine 0.995, just past the 0.9963 worst case measured between two machines.
+ const generated = tree({
+ "a/local-vectors.json": meta,
+ "a/local-vectors.bin": vectors([
+ [0.5, 0.5, 0.5, 0.5],
+ [1, 0, 0, 0],
+ ]),
+ });
+ const committed = tree({
+ "a/local-vectors.json": meta,
+ "a/local-vectors.bin": vectors([
+ [0.5, 0.51, 0.49, 0.5],
+ [1, 0.1, 0, 0],
+ ]),
+ });
+ assert.deepEqual(treeDifferences(generated, committed), []);
+ rmSync(generated, { recursive: true });
+ rmSync(committed, { recursive: true });
+});
+
+test("vectors with a row that points elsewhere, or a different row count, are stale", () => {
+ // a: cosine 0.984, below the closest two different catalog items (0.9897).
+ const generated = tree({
+ "a/local-vectors.json": meta,
+ "a/local-vectors.bin": vectors([
+ [1, 0, 0, 0],
+ [0, 1, 0, 0],
+ ]),
+ "b/local-vectors.json": meta,
+ "b/local-vectors.bin": vectors([[1, 0, 0, 0]]),
+ });
+ const committed = tree({
+ "a/local-vectors.json": meta,
+ "a/local-vectors.bin": vectors([
+ [1, 0, 0, 0],
+ [0.18, 1, 0, 0],
+ ]),
+ "b/local-vectors.json": meta,
+ "b/local-vectors.bin": vectors([
+ [1, 0, 0, 0],
+ [0, 1, 0, 0],
+ ]),
+ });
+ assert.deepEqual(treeDifferences(generated, committed), [
+ "stale: a/local-vectors.bin",
+ "stale: b/local-vectors.bin",
+ ]);
+ rmSync(generated, { recursive: true });
+ rmSync(committed, { recursive: true });
+});
diff --git a/scripts/catalog-drift.ts b/scripts/catalog-drift.ts
index 52586899ec..76f0ffdeb6 100644
--- a/scripts/catalog-drift.ts
+++ b/scripts/catalog-drift.ts
@@ -10,6 +10,9 @@ import { runAsCommand } from "./entrypoint.ts";
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const MAX_LISTED = 40;
+// The embedding model's float output differs slightly between CPUs, so vectors are compared by direction, not bytes.
+const VECTORS = /-vectors\.bin$/;
+const MIN_ROW_COSINE = 0.99;
function filesUnder(root: string): string[] {
return readdirSync(root, { recursive: true, withFileTypes: true })
@@ -25,9 +28,7 @@ export function treeDifferences(generatedRoot: string, committedRoot: string): s
const missing = [...generated].filter((file) => !committed.has(file));
const extra = [...committed].filter((file) => !generated.has(file));
const changed = [...generated].filter(
- (file) =>
- committed.has(file) &&
- !readFileSync(join(generatedRoot, file)).equals(readFileSync(join(committedRoot, file))),
+ (file) => committed.has(file) && !sameContent(generatedRoot, committedRoot, file),
);
return [
...missing.map((file) => `not committed: ${file}`),
@@ -36,6 +37,39 @@ export function treeDifferences(generatedRoot: string, committedRoot: string): s
];
}
+function sameContent(generatedRoot: string, committedRoot: string, file: string): boolean {
+ const generated = readFileSync(join(generatedRoot, file));
+ const committed = readFileSync(join(committedRoot, file));
+ if (generated.equals(committed)) return true;
+ if (!VECTORS.test(file)) return false;
+ const meta = join(generatedRoot, file.replace(/\.bin$/, ".json"));
+ const { dimensions } = JSON.parse(readFileSync(meta, "utf8")) as { dimensions: number };
+ return vectorsAgree(generated, committed, dimensions);
+}
+
+/** Same row count, and every row of Float32 vectors points the same way within MIN_ROW_COSINE. */
+export function vectorsAgree(a: Buffer, b: Buffer, dimensions: number): boolean {
+ const rowBytes = 4 * dimensions;
+ if (a.length !== b.length || a.length % rowBytes !== 0) return false;
+ return Array.from({ length: a.length / rowBytes }, (_, row) =>
+ rowCosine(a, b, row * rowBytes, (row + 1) * rowBytes),
+ ).every((cosine) => cosine >= MIN_ROW_COSINE);
+}
+
+function rowCosine(a: Buffer, b: Buffer, start: number, end: number): number {
+ let dot = 0;
+ let normA = 0;
+ let normB = 0;
+ for (let i = start; i < end; i += 4) {
+ const x = a.readFloatLE(i);
+ const y = b.readFloatLE(i);
+ dot += x * y;
+ normA += x * x;
+ normB += y * y;
+ }
+ return dot / Math.sqrt(normA * normB);
+}
+
async function main(): Promise<void> {
await generateCatalog();
for (const script of [