Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

perf(api): stop Postgres KV reads scanning whole tables (#2118)

Hampus14ae64f5f317
diff --git a/fluxer_api/src/api/database/PostgresKvExactnessFuzz.test.ts b/fluxer_api/src/api/database/PostgresKvExactnessFuzz.test.ts
new file mode 100644
index 000000000..d0e889650
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvExactnessFuzz.test.ts
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import cassandra from 'cassandra-driver';
+import {describe, expect, it} from 'vitest';
+import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
+import {buildCandidatePlan, keyFromColumns, matchesWhere} from './PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+type AnyMeta = KvQueryMeta<Row>;
+
+const Spec: KvTableSpec<Row> = {
+ name: 'fuzz_two',
+ columns: ['k', 'c', 'v'],
+ primaryKey: ['k', 'c'],
+ partitionKey: ['k', 'c'],
+};
+
+const SpecOne: KvTableSpec<Row> = {
+ name: 'fuzz_one',
+ columns: ['k', 'v'],
+ primaryKey: ['k'],
+ partitionKey: ['k'],
+};
+
+const VALUES: Array<[string, unknown]> = [
+ ['null', null],
+ ['undefined', undefined],
+ ['zero', 0],
+ ['neg-zero', -0],
+ ['one-num', 1],
+ ['one-big', 1n],
+ ['one-str', '1'],
+ ['empty-str', ''],
+ ['true', true],
+ ['false', false],
+ ['date-epoch', new Date(0)],
+ ['date-epoch-2', new Date(0)],
+ ['buf-a', Buffer.from('a')],
+ ['buf-a-2', Buffer.from('a')],
+ ['localdate', cassandra.types.LocalDate.fromString('2020-01-01')],
+ ['localdate-str', '2020-01-01'],
+];
+
+function rowKeyMatchesPlan(plan: ReturnType<typeof buildCandidatePlan>, rowKey: string): boolean {
+ const c = plan.candidates;
+ const buf = Buffer.from(rowKey, 'utf8');
+ switch (c.kind) {
+ case 'none':
+ return false;
+ case 'scan':
+ return true;
+ case 'rowKeys':
+ return c.rowKeys.includes(rowKey);
+ case 'range':
+ return (
+ Buffer.compare(buf, Buffer.from(c.lowerBound, 'utf8')) >= 0 &&
+ Buffer.compare(buf, Buffer.from(c.upperBound, 'utf8')) < 0
+ );
+ case 'ranges':
+ return c.lowerBounds.some(
+ (lower, i) =>
+ Buffer.compare(buf, Buffer.from(lower, 'utf8')) >= 0 &&
+ Buffer.compare(buf, Buffer.from(c.upperBounds[i]!, 'utf8')) < 0,
+ );
+ case 'partitionKeys':
+ return c.partitionKeys.includes(rowKey);
+ }
+}
+
+describe('postgres kv pushdown exactness', () => {
+ it('never excludes a row the javascript filter would keep, and is equal when exact', () => {
+ const narrowing: Array<string> = [];
+ const widening: Array<string> = [];
+ const meta = {
+ action: 'count',
+ table: Spec,
+ where: [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>,
+ } as AnyMeta;
+ for (const [paramName, paramValue] of VALUES) {
+ const params = {k: paramValue} as CassandraParams;
+ const plan = buildCandidatePlan(meta, params);
+ for (const [storedName, storedValue] of VALUES) {
+ const row: Row = {k: storedValue, c: 'x', v: 1};
+ const rowKey = keyFromColumns(Spec.primaryKey as Array<string>, row);
+ const js = matchesWhere(row, meta.where as Array<WhereExpr<Row>>, params);
+ const sql = rowKeyMatchesPlan(plan, rowKey);
+ const label = `param=${paramName} stored=${storedName} exact=${plan.exact} plan=${plan.candidates.kind}`;
+ if (js && !sql) narrowing.push(label);
+ if (plan.exact && sql && !js) widening.push(label);
+ }
+ }
+ expect({narrowing, widening}).toEqual({narrowing: [], widening: []});
+ });
+
+ it('single-column primary keys behave the same', () => {
+ const narrowing: Array<string> = [];
+ const widening: Array<string> = [];
+ const meta = {
+ action: 'count',
+ table: SpecOne,
+ where: [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>,
+ } as AnyMeta;
+ for (const [paramName, paramValue] of VALUES) {
+ const params = {k: paramValue} as CassandraParams;
+ const plan = buildCandidatePlan(meta, params);
+ for (const [storedName, storedValue] of VALUES) {
+ const row: Row = {k: storedValue, v: 1};
+ const rowKey = keyFromColumns(SpecOne.primaryKey as Array<string>, row);
+ const js = matchesWhere(row, meta.where as Array<WhereExpr<Row>>, params);
+ const sql = rowKeyMatchesPlan(plan, rowKey);
+ const label = `param=${paramName} stored=${storedName} exact=${plan.exact} plan=${plan.candidates.kind}`;
+ if (js && !sql) narrowing.push(label);
+ if (plan.exact && sql && !js) widening.push(label);
+ }
+ }
+ expect({narrowing, widening}).toEqual({narrowing: [], widening: []});
+ });
+
+ it('IN lists behave the same', () => {
+ const narrowing: Array<string> = [];
+ const widening: Array<string> = [];
+ const meta = {
+ action: 'count',
+ table: SpecOne,
+ where: [{kind: 'in', col: 'k', param: 'ks'}] as Array<WhereExpr<Row>>,
+ } as AnyMeta;
+ for (let i = 0; i < VALUES.length; i += 1) {
+ for (let j = i; j < VALUES.length; j += 1) {
+ const params = {ks: [VALUES[i]![1], VALUES[j]![1]]} as CassandraParams;
+ const plan = buildCandidatePlan(meta, params);
+ for (const [storedName, storedValue] of VALUES) {
+ const row: Row = {k: storedValue, v: 1};
+ const rowKey = keyFromColumns(SpecOne.primaryKey as Array<string>, row);
+ const js = matchesWhere(row, meta.where as Array<WhereExpr<Row>>, params);
+ const sql = rowKeyMatchesPlan(plan, rowKey);
+ const label = `in=[${VALUES[i]![0]},${VALUES[j]![0]}] stored=${storedName} exact=${plan.exact} plan=${plan.candidates.kind}`;
+ if (js && !sql) narrowing.push(label);
+ if (plan.exact && sql && !js) widening.push(label);
+ }
+ }
+ }
+ expect({narrowing: narrowing.slice(0, 10), widening: widening.slice(0, 10)}).toEqual({narrowing: [], widening: []});
+ });
+});
diff --git a/fluxer_api/src/api/database/PostgresKvLoggerSafety.test.ts b/fluxer_api/src/api/database/PostgresKvLoggerSafety.test.ts
new file mode 100644
index 000000000..6d8ce402c
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvLoggerSafety.test.ts
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {describe, expect, it, vi} from 'vitest';
+import {defineTable} from './CassandraTableDsl';
+import type {KvQueryMeta} from './CassandraTypes';
+
+vi.mock('../Logger', () => ({
+ Logger: new Proxy(
+ {},
+ {
+ get() {
+ throw new Error('Logger has not been initialized. Call initializeLogger() first.');
+ },
+ },
+ ),
+}));
+
+type Row = Record<string, unknown>;
+
+const Probe = defineTable<{k: string; v: string}, 'k'>({
+ name: 'kv_logger_probe',
+ columns: ['k', 'v'],
+ primaryKey: ['k'],
+});
+
+const client = {
+ async query() {
+ return {rows: [], rowCount: 0};
+ },
+ async connect() {},
+ async shutdown() {},
+ isConnected() {
+ return true;
+ },
+ async transaction(fn: (db: unknown) => Promise<unknown>) {
+ return fn(client);
+ },
+ kvTable() {
+ return 'kv_logger_probe_table';
+ },
+} as never;
+
+describe('PostgresKvQueryExecutor logging safety', () => {
+ it('runs full scan queries in a process whose logger was never initialized', async () => {
+ const {PostgresKvQueryExecutor} = await import('./PostgresKvQueryExecutor');
+ const executor = new PostgresKvQueryExecutor(client);
+ for (const action of ['select', 'count', 'delete'] as const) {
+ const meta = {action, table: Probe, where: [], columns: ['k', 'v']} as unknown as KvQueryMeta<Row>;
+ await expect(executor.executeQuery({cql: `__probe_${action}`, params: {}, kvMeta: meta})).resolves.toBeDefined();
+ }
+ });
+});
diff --git a/fluxer_api/src/api/database/PostgresKvNonSelectDrift.test.ts b/fluxer_api/src/api/database/PostgresKvNonSelectDrift.test.ts
new file mode 100644
index 000000000..1ed3f0f80
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvNonSelectDrift.test.ts
@@ -0,0 +1,335 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {
+ getDefaultPostgresClient,
+ type IPostgresClient,
+ initPostgres,
+ type PostgresQueryable,
+ shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
+import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
+import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+type AnyMeta = KvQueryMeta<Row>;
+
+const LEGACY_TABLE = 'kvns_legacy';
+const NEXT_TABLE = 'kvns_next';
+const CONTAINER = `fluxer-kvns-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+
+class TableClient implements IPostgresClient {
+ constructor(
+ private readonly inner: IPostgresClient,
+ private readonly table: string,
+ ) {}
+ async query<T extends Record<string, unknown>>(text: string, values: Array<unknown> = []) {
+ return (await this.inner.query(text, values)) as unknown as Awaited<ReturnType<IPostgresClient['query']>> & {
+ rows: Array<T>;
+ };
+ }
+ async connect(): Promise<void> {
+ await this.inner.connect();
+ }
+ async shutdown(): Promise<void> {}
+ isConnected(): boolean {
+ return this.inner.isConnected();
+ }
+ async transaction<T>(fn: (client: PostgresQueryable) => Promise<T>): Promise<T> {
+ return this.inner.transaction(fn);
+ }
+ kvTable(): string {
+ return this.table;
+ }
+}
+
+const Parent: KvTableSpec<Row> = {
+ name: 'ns_parent',
+ columns: ['user_id', 'channel_id', 'note', 'blob_'],
+ primaryKey: ['user_id', 'channel_id'],
+ partitionKey: ['user_id', 'channel_id'],
+};
+
+const Single: KvTableSpec<Row> = {
+ name: 'ns_single',
+ columns: ['token_', 'note'],
+ primaryKey: ['token_'],
+ partitionKey: ['token_'],
+};
+
+function eq(col: string, param: string): WhereExpr<Row> {
+ return {kind: 'eq', col, param} as WhereExpr<Row>;
+}
+function inClause(col: string, param: string): WhereExpr<Row> {
+ return {kind: 'in', col, param} as WhereExpr<Row>;
+}
+
+describe.skipIf(!dockerAvailable)('postgres kv non-select drift', () => {
+ let raw: IPostgresClient;
+ let legacy: LegacyPostgresKvQueryExecutor;
+ let next: PostgresKvQueryExecutor;
+
+ async function freePort(): Promise<number> {
+ return new Promise((resolve, reject) => {
+ const server = createServer();
+ server.on('error', reject);
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address();
+ if (typeof address === 'string' || address === null) {
+ reject(new Error('no port'));
+ return;
+ }
+ const port = address.port;
+ server.close(() => resolve(port));
+ });
+ });
+ }
+ async function sleep(ms: number): Promise<void> {
+ await new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ async function dump(kv: string, tableName: string): Promise<string> {
+ const result = await raw.query<{row_key: string; partition_key: string; row_data: unknown; expires_at: Date}>(
+ `SELECT row_key, partition_key, row_data, expires_at FROM ${kv} WHERE table_name = $1 ORDER BY row_key COLLATE "C"`,
+ [tableName],
+ );
+ return JSON.stringify(
+ result.rows.map((r) => [
+ r.row_key,
+ r.partition_key,
+ r.row_data,
+ r.expires_at === null ? null : r.expires_at instanceof Date ? 'ts' : String(r.expires_at),
+ ]),
+ );
+ }
+
+ async function reset(spec: KvTableSpec<Row>, rows: ReadonlyArray<Row>): Promise<void> {
+ for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
+ await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [spec.name]);
+ }
+ const meta = {action: 'upsert', table: spec} as AnyMeta;
+ for (const row of rows) {
+ await legacy.executeQuery({cql: '__seed__', params: row as CassandraParams, kvMeta: meta});
+ await next.executeQuery({cql: '__seed__', params: row as CassandraParams, kvMeta: meta});
+ }
+ }
+
+ beforeAll(async () => {
+ const port = await freePort();
+ execFileSync(
+ 'docker',
+ [
+ 'run',
+ '-d',
+ '--name',
+ CONTAINER,
+ '-e',
+ 'POSTGRES_USER=fluxer',
+ '-e',
+ 'POSTGRES_PASSWORD=fluxer',
+ '-e',
+ 'POSTGRES_DB=fluxer',
+ '-p',
+ `127.0.0.1:${port}:5432`,
+ 'postgres:16-alpine',
+ '-c',
+ 'fsync=off',
+ ],
+ {stdio: 'ignore'},
+ );
+ let ready = false;
+ for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+ await sleep(500);
+ const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+ stdio: 'ignore',
+ });
+ if (probe.status !== 0) continue;
+ try {
+ await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 8});
+ await getDefaultPostgresClient().query('SELECT 1');
+ ready = true;
+ } catch {
+ await shutdownPostgres().catch(() => {});
+ }
+ }
+ if (!ready) throw new Error('postgres never came up');
+ raw = getDefaultPostgresClient();
+ const legacyClient = new TableClient(raw, LEGACY_TABLE);
+ const nextClient = new TableClient(raw, NEXT_TABLE);
+ await ensurePostgresKvSchema(legacyClient);
+ await ensurePostgresKvSchema(nextClient);
+ legacy = new LegacyPostgresKvQueryExecutor(legacyClient);
+ next = new PostgresKvQueryExecutor(nextClient);
+ }, 900_000);
+
+ afterAll(async () => {
+ await shutdownPostgres().catch(() => {});
+ spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+ });
+
+ const seedRows: Array<Row> = [
+ {user_id: 1n, channel_id: 10n, note: 'a', blob_: null},
+ {user_id: 1n, channel_id: 11n, note: 'b', blob_: null},
+ {user_id: 2n, channel_id: 10n, note: 'c', blob_: null},
+ ];
+
+ it('upsert / insert / ifNotExists behave identically', async () => {
+ await reset(Parent, seedRows);
+ const insMeta = {action: 'upsert', table: Parent, ifNotExists: true} as AnyMeta;
+ const fresh = {user_id: 3n, channel_id: 30n, note: 'z', blob_: null};
+ const l1 = await legacy.executeQuery({cql: '__i__', params: fresh as CassandraParams, kvMeta: insMeta});
+ const n1 = await next.executeQuery({cql: '__i__', params: fresh as CassandraParams, kvMeta: insMeta});
+ expect(n1).toEqual(l1);
+ const l2 = await legacy.executeQuery({cql: '__i__', params: fresh as CassandraParams, kvMeta: insMeta});
+ const n2 = await next.executeQuery({cql: '__i__', params: fresh as CassandraParams, kvMeta: insMeta});
+ expect(n2).toEqual(l2);
+ expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
+ });
+
+ it('patch with and without ttl leaves identical state', async () => {
+ await reset(Parent, seedRows);
+ const patchMeta = {
+ action: 'patch',
+ table: Parent,
+ patchKeys: ['note'],
+ pkColumns: ['user_id', 'channel_id'],
+ } as unknown as AnyMeta;
+ const p = {user_id: 1n, channel_id: 10n, note: 'patched'} as CassandraParams;
+ await legacy.executeQuery({cql: '__p__', params: p, kvMeta: patchMeta});
+ await next.executeQuery({cql: '__p__', params: p, kvMeta: patchMeta});
+ const ttlMeta = {...patchMeta, ttlParamName: 'ttl_'} as unknown as AnyMeta;
+ const pt = {user_id: 2n, channel_id: 10n, note: 'ttl', ttl_: 600} as CassandraParams;
+ await legacy.executeQuery({cql: '__pt__', params: pt, kvMeta: ttlMeta});
+ await next.executeQuery({cql: '__pt__', params: pt, kvMeta: ttlMeta});
+ expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
+ });
+
+ it('count returns the same value and the same javascript type', async () => {
+ await reset(Parent, seedRows);
+ await raw.query(
+ `UPDATE ${NEXT_TABLE} SET expires_at = now() - interval '1 hour' WHERE table_name = $1 AND row_key LIKE '%11%'`,
+ [Parent.name],
+ );
+ await raw.query(
+ `UPDATE ${LEGACY_TABLE} SET expires_at = now() - interval '1 hour' WHERE table_name = $1 AND row_key LIKE '%11%'`,
+ [Parent.name],
+ );
+ const meta = {action: 'count', table: Parent, where: [eq('user_id', 'user_id')]} as AnyMeta;
+ const l = await legacy.executeQuery<{count: unknown}>({cql: '__c__', params: {user_id: 1n}, kvMeta: meta});
+ const n = await next.executeQuery<{count: unknown}>({cql: '__c__', params: {user_id: 1n}, kvMeta: meta});
+ expect(n).toEqual(l);
+ expect(typeof (n[0] as {count: unknown}).count).toBe(typeof (l[0] as {count: unknown}).count);
+ });
+
+ it('count over an empty logical table matches', async () => {
+ const meta = {action: 'count', table: Single, where: []} as AnyMeta;
+ for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
+ await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [Single.name]);
+ }
+ const l = await legacy.executeQuery({cql: '__ce__', params: {}, kvMeta: meta});
+ const n = await next.executeQuery({cql: '__ce__', params: {}, kvMeta: meta});
+ expect(n).toEqual(l);
+ });
+
+ it('count with a string bound to an IN parameter matches legacy', async () => {
+ await reset(Single, [
+ {token_: 'a', note: '1'},
+ {token_: 'b', note: '2'},
+ ]);
+ const meta = {action: 'count', table: Single, where: [inClause('token_', 'tokens')]} as AnyMeta;
+ let legacyOut: unknown;
+ let nextOut: unknown;
+ try {
+ legacyOut = await legacy.executeQuery({cql: '__cs__', params: {tokens: 'ab'} as CassandraParams, kvMeta: meta});
+ } catch (error) {
+ legacyOut = `throw:${(error as Error).constructor.name}`;
+ }
+ try {
+ nextOut = await next.executeQuery({cql: '__cs__', params: {tokens: 'ab'} as CassandraParams, kvMeta: meta});
+ } catch (error) {
+ nextOut = `throw:${(error as Error).constructor.name}`;
+ }
+ expect(nextOut).toEqual(legacyOut);
+ });
+
+ it('rejects malformed page tokens the way legacy does', async () => {
+ await reset(Single, [
+ {token_: 'a', note: '1'},
+ {token_: 'b', note: '2'},
+ {token_: 'c', note: '3'},
+ ]);
+ const meta = {action: 'select', table: Single, where: [], columns: Single.columns} as AnyMeta;
+ const tokens = ['eyJmb28iOjF9', 'e30', 'MTIz', 'InN0ciI', 'eyJvZmZzZXQiOi0xfQ', 'eyJvZmZzZXQiOjF9'];
+ const legacyOut: Array<string> = [];
+ const nextOut: Array<string> = [];
+ for (const token of tokens) {
+ for (const [exec, sink] of [
+ [legacy, legacyOut],
+ [next, nextOut],
+ ] as const) {
+ try {
+ const page = await exec.executePagedQuery(
+ {cql: '__pg__', params: {}, kvMeta: meta},
+ {pageSize: 1, pageState: token},
+ );
+ sink.push(`${token} -> rows=${page.rows.length}`);
+ } catch (error) {
+ sink.push(`${token} -> throw:${(error as Error).message}`);
+ }
+ }
+ }
+ expect(nextOut).toEqual(legacyOut);
+ });
+
+ it('delete inside an atomic batch rolls back identically', async () => {
+ await reset(Parent, seedRows);
+ const delMeta = {action: 'delete', table: Parent, where: [eq('user_id', 'user_id')]} as AnyMeta;
+ for (const [exec, kv] of [
+ [legacy, LEGACY_TABLE],
+ [next, NEXT_TABLE],
+ ] as const) {
+ await expect(
+ exec.executeBatch([
+ {query: '__d__', params: {user_id: 1n}, meta: delMeta as KvQueryMeta},
+ {query: '__boom__', params: {}, meta: {action: 'bogus'} as unknown as KvQueryMeta},
+ ]),
+ ).rejects.toThrow();
+ void kv;
+ }
+ expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
+ });
+
+ it('delete of an expired-but-unpruned row matches legacy', async () => {
+ await reset(Parent, seedRows);
+ for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
+ await raw.query(`UPDATE ${kv} SET expires_at = now() - interval '1 hour' WHERE table_name = $1`, [Parent.name]);
+ }
+ const delMeta = {action: 'delete', table: Parent, where: [eq('user_id', 'user_id')]} as AnyMeta;
+ await legacy.executeQuery({cql: '__d__', params: {user_id: 1n}, kvMeta: delMeta});
+ await next.executeQuery({cql: '__d__', params: {user_id: 1n}, kvMeta: delMeta});
+ expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
+ });
+
+ it('delete with no where clause matches legacy', async () => {
+ await reset(Parent, seedRows);
+ const delMeta = {action: 'delete', table: Parent, where: []} as AnyMeta;
+ await legacy.executeQuery({cql: '__da__', params: {}, kvMeta: delMeta});
+ await next.executeQuery({cql: '__da__', params: {}, kvMeta: delMeta});
+ expect(await dump(NEXT_TABLE, Parent.name)).toBe(await dump(LEGACY_TABLE, Parent.name));
+ });
+
+ it('delete does not touch a neighbouring logical table', async () => {
+ await reset(Parent, seedRows);
+ await reset(Single, [{token_: 'a', note: '1'}]);
+ const delMeta = {action: 'delete', table: Parent, where: []} as AnyMeta;
+ await next.executeQuery({cql: '__da2__', params: {}, kvMeta: delMeta});
+ const left = await raw.query<{count: string}>(`SELECT count(*) AS count FROM ${NEXT_TABLE} WHERE table_name = $1`, [
+ Single.name,
+ ]);
+ expect(left.rows[0]!.count).toBe('1');
+ });
+});
diff --git a/fluxer_api/src/api/database/PostgresKvNonSelectMatrix.test.ts b/fluxer_api/src/api/database/PostgresKvNonSelectMatrix.test.ts
new file mode 100644
index 000000000..91acbf54f
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvNonSelectMatrix.test.ts
@@ -0,0 +1,361 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {
+ getDefaultPostgresClient,
+ type IPostgresClient,
+ initPostgres,
+ type PostgresQueryable,
+ shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import cassandra from 'cassandra-driver';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
+import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
+import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+type AnyMeta = KvQueryMeta<Row>;
+
+const LEGACY_TABLE = 'kvm_legacy';
+const NEXT_TABLE = 'kvm_next';
+const CONTAINER = `fluxer-kvm-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+
+class TableClient implements IPostgresClient {
+ constructor(
+ private readonly inner: IPostgresClient,
+ private readonly table: string,
+ ) {}
+ async query<T extends Record<string, unknown>>(text: string, values: Array<unknown> = []) {
+ return (await this.inner.query(text, values)) as unknown as Awaited<ReturnType<IPostgresClient['query']>> & {
+ rows: Array<T>;
+ };
+ }
+ async connect(): Promise<void> {
+ await this.inner.connect();
+ }
+ async shutdown(): Promise<void> {}
+ isConnected(): boolean {
+ return this.inner.isConnected();
+ }
+ async transaction<T>(fn: (client: PostgresQueryable) => Promise<T>): Promise<T> {
+ return this.inner.transaction(fn);
+ }
+ kvTable(): string {
+ return this.table;
+ }
+}
+
+const Two: KvTableSpec<Row> = {
+ name: 'mx_two',
+ columns: ['k', 'c', 'v'],
+ primaryKey: ['k', 'c'],
+ partitionKey: ['k', 'c'],
+};
+const PartTwo: KvTableSpec<Row> = {
+ name: 'mx_part',
+ columns: ['k', 'c', 'v'],
+ primaryKey: ['k', 'c'],
+ partitionKey: ['k'],
+};
+const Three: KvTableSpec<Row> = {
+ name: 'mx_three',
+ columns: ['k', 'c', 'd', 'v'],
+ primaryKey: ['k', 'c', 'd'],
+ partitionKey: ['k', 'c', 'd'],
+};
+
+const KEY_VALUES: Array<[string, unknown]> = [
+ ['null', null],
+ ['num0', 0],
+ ['num1', 1],
+ ['num10', 10],
+ ['big1', 1n],
+ ['str1', '1'],
+ ['strq', 'a"b\\c'],
+ ['strctl', 'xy'],
+ ['strastral', '\u{1f600}'],
+ ['strspace', 'a b'],
+ ['true', true],
+ ['false', false],
+ ['date', new Date(0)],
+ ['buf', Buffer.from('a')],
+ ['localdate', cassandra.types.LocalDate.fromString('2020-01-01')],
+ ['localdate-str', '2020-01-01'],
+ ['emptystr', ''],
+];
+
+function bigintJson(_key: string, value: unknown): unknown {
+ if (typeof value === 'bigint') return `bigint:${value.toString()}`;
+ if (Buffer.isBuffer(value)) return `buf:${value.toString('base64')}`;
+ return value;
+}
+
+describe.skipIf(!dockerAvailable)('postgres kv non-select matrix', () => {
+ let raw: IPostgresClient;
+ let legacy: LegacyPostgresKvQueryExecutor;
+ let next: PostgresKvQueryExecutor;
+
+ async function freePort(): Promise<number> {
+ return new Promise((resolve, reject) => {
+ const server = createServer();
+ server.on('error', reject);
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address();
+ if (typeof address === 'string' || address === null) {
+ reject(new Error('no port'));
+ return;
+ }
+ const port = address.port;
+ server.close(() => resolve(port));
+ });
+ });
+ }
+ async function sleep(ms: number): Promise<void> {
+ await new Promise((resolve) => setTimeout(resolve, ms));
+ }
+ async function dump(kv: string, tableName: string): Promise<string> {
+ const result = await raw.query<{row_key: string}>(
+ `SELECT row_key FROM ${kv} WHERE table_name = $1 ORDER BY row_key COLLATE "C"`,
+ [tableName],
+ );
+ return JSON.stringify(result.rows.map((r) => r.row_key));
+ }
+ async function seed(spec: KvTableSpec<Row>, rows: ReadonlyArray<Row>): Promise<void> {
+ for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
+ await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [spec.name]);
+ }
+ const meta = {action: 'upsert', table: spec} as AnyMeta;
+ for (const row of rows) {
+ await legacy.executeQuery({cql: '__s__', params: row as CassandraParams, kvMeta: meta});
+ await next.executeQuery({cql: '__s__', params: row as CassandraParams, kvMeta: meta});
+ }
+ }
+
+ beforeAll(async () => {
+ const port = await freePort();
+ execFileSync(
+ 'docker',
+ [
+ 'run',
+ '-d',
+ '--name',
+ CONTAINER,
+ '-e',
+ 'POSTGRES_USER=fluxer',
+ '-e',
+ 'POSTGRES_PASSWORD=fluxer',
+ '-e',
+ 'POSTGRES_DB=fluxer',
+ '-p',
+ `127.0.0.1:${port}:5432`,
+ 'postgres:16-alpine',
+ '-c',
+ 'fsync=off',
+ ],
+ {stdio: 'ignore'},
+ );
+ let ready = false;
+ for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+ await sleep(500);
+ if (
+ spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {stdio: 'ignore'})
+ .status !== 0
+ )
+ continue;
+ try {
+ await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 8});
+ await getDefaultPostgresClient().query('SELECT 1');
+ ready = true;
+ } catch {
+ await shutdownPostgres().catch(() => {});
+ }
+ }
+ if (!ready) throw new Error('postgres never came up');
+ raw = getDefaultPostgresClient();
+ await ensurePostgresKvSchema(new TableClient(raw, LEGACY_TABLE));
+ await ensurePostgresKvSchema(new TableClient(raw, NEXT_TABLE));
+ legacy = new LegacyPostgresKvQueryExecutor(new TableClient(raw, LEGACY_TABLE));
+ next = new PostgresKvQueryExecutor(new TableClient(raw, NEXT_TABLE));
+ }, 900_000);
+
+ afterAll(async () => {
+ await shutdownPostgres().catch(() => {});
+ spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+ });
+
+ it('count and delete agree with legacy for every key value type', async () => {
+ const rows = KEY_VALUES.map(([, value], i) => ({k: value, c: BigInt(i), v: i}));
+ const countMismatch: Array<string> = [];
+ const deleteMismatch: Array<string> = [];
+ const selectMismatch: Array<string> = [];
+ for (const [name, value] of KEY_VALUES) {
+ await seed(Two, rows);
+ const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>;
+ const params = {k: value} as CassandraParams;
+ const sel = {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta;
+ const l = await legacy.executeQuery({cql: '__sel__', params, kvMeta: sel});
+ const n = await next.executeQuery({cql: '__sel__', params, kvMeta: sel});
+ if (JSON.stringify(l, bigintJson) !== JSON.stringify(n, bigintJson)) {
+ selectMismatch.push(`${name}: legacy=${JSON.stringify(l, bigintJson)} next=${JSON.stringify(n, bigintJson)}`);
+ }
+ const cnt = {action: 'count', table: Two, where} as AnyMeta;
+ const lc = await legacy.executeQuery({cql: '__cnt__', params, kvMeta: cnt});
+ const nc = await next.executeQuery({cql: '__cnt__', params, kvMeta: cnt});
+ if (JSON.stringify(lc) !== JSON.stringify(nc)) {
+ countMismatch.push(`${name}: legacy=${JSON.stringify(lc)} next=${JSON.stringify(nc)}`);
+ }
+ const del = {action: 'delete', table: Two, where} as AnyMeta;
+ await legacy.executeQuery({cql: '__del__', params, kvMeta: del});
+ await next.executeQuery({cql: '__del__', params, kvMeta: del});
+ const ld = await dump(LEGACY_TABLE, Two.name);
+ const nd = await dump(NEXT_TABLE, Two.name);
+ if (ld !== nd) deleteMismatch.push(`${name}: legacy=${ld} next=${nd}`);
+ }
+ expect({selectMismatch, countMismatch, deleteMismatch}).toEqual({
+ selectMismatch: [],
+ countMismatch: [],
+ deleteMismatch: [],
+ });
+ }, 300_000);
+
+ it('explicit partition key tables agree for count and delete', async () => {
+ const rows = [
+ {k: 1n, c: 1n, v: 1},
+ {k: 1n, c: 2n, v: 2},
+ {k: 2n, c: 1n, v: 3},
+ ];
+ await seed(PartTwo, rows);
+ const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>;
+ const params = {k: 1n} as CassandraParams;
+ const lc = await legacy.executeQuery({
+ cql: '__pc__',
+ params,
+ kvMeta: {action: 'count', table: PartTwo, where} as AnyMeta,
+ });
+ const nc = await next.executeQuery({
+ cql: '__pc__',
+ params,
+ kvMeta: {action: 'count', table: PartTwo, where} as AnyMeta,
+ });
+ expect(nc).toEqual(lc);
+ await legacy.executeQuery({cql: '__pd__', params, kvMeta: {action: 'delete', table: PartTwo, where} as AnyMeta});
+ await next.executeQuery({cql: '__pd__', params, kvMeta: {action: 'delete', table: PartTwo, where} as AnyMeta});
+ expect(await dump(NEXT_TABLE, PartTwo.name)).toBe(await dump(LEGACY_TABLE, PartTwo.name));
+ });
+
+ it('multi column IN products agree, including above the combination cap', async () => {
+ const rows: Array<Row> = [];
+ for (let a = 0; a < 40; a += 1) {
+ for (let b = 0; b < 40; b += 1) rows.push({k: BigInt(a), c: BigInt(b), v: a * b});
+ }
+ await seed(Two, rows);
+ const ks = Array.from({length: 40}, (_, i) => BigInt(i));
+ const cs = Array.from({length: 40}, (_, i) => BigInt(i));
+ const where = [
+ {kind: 'in', col: 'k', param: 'ks'},
+ {kind: 'in', col: 'c', param: 'cs'},
+ ] as Array<WhereExpr<Row>>;
+ for (const [label, params] of [
+ ['small', {ks: ks.slice(0, 3), cs: cs.slice(0, 3)}],
+ ['cap', {ks, cs}],
+ ['dupes', {ks: [1n, 1n, 2n], cs: [1n, 1n]}],
+ ['empty', {ks: [], cs}],
+ ] as Array<[string, CassandraParams]>) {
+ const lc = await legacy.executeQuery({
+ cql: `__mc_${label}__`,
+ params,
+ kvMeta: {action: 'count', table: Two, where} as AnyMeta,
+ });
+ const nc = await next.executeQuery({
+ cql: `__mc_${label}__`,
+ params,
+ kvMeta: {action: 'count', table: Two, where} as AnyMeta,
+ });
+ expect([label, nc]).toEqual([label, lc]);
+ const ls = await legacy.executeQuery({
+ cql: `__ms_${label}__`,
+ params,
+ kvMeta: {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta,
+ });
+ const ns = await next.executeQuery({
+ cql: `__ms_${label}__`,
+ params,
+ kvMeta: {action: 'select', table: Two, where, columns: Two.columns} as AnyMeta,
+ });
+ expect([label, JSON.stringify(ns, bigintJson)]).toEqual([label, JSON.stringify(ls, bigintJson)]);
+ }
+ await legacy.executeQuery({
+ cql: '__md__',
+ params: {ks, cs} as CassandraParams,
+ kvMeta: {action: 'delete', table: Two, where} as AnyMeta,
+ });
+ await next.executeQuery({
+ cql: '__md__',
+ params: {ks, cs} as CassandraParams,
+ kvMeta: {action: 'delete', table: Two, where} as AnyMeta,
+ });
+ expect(await dump(NEXT_TABLE, Two.name)).toBe(await dump(LEGACY_TABLE, Two.name));
+ }, 300_000);
+
+ it('count with a limit or an order by matches legacy', async () => {
+ await seed(Two, [
+ {k: 1n, c: 1n, v: 1},
+ {k: 1n, c: 2n, v: 2},
+ {k: 1n, c: 3n, v: 3},
+ ]);
+ const where = [{kind: 'eq', col: 'k', param: 'k'}] as Array<WhereExpr<Row>>;
+ for (const extra of [{limit: 2}, {orderBy: {col: 'c', direction: 'DESC'}}, {}]) {
+ const meta = {action: 'count', table: Two, where, ...extra} as AnyMeta;
+ const l = await legacy.executeQuery({cql: `__cl__${JSON.stringify(extra)}`, params: {k: 1n}, kvMeta: meta});
+ const n = await next.executeQuery({cql: `__cl__${JSON.stringify(extra)}`, params: {k: 1n}, kvMeta: meta});
+ expect([JSON.stringify(extra), n]).toEqual([JSON.stringify(extra), l]);
+ }
+ });
+
+ it('count and delete with a gap in the primary key match legacy', async () => {
+ const rows: Array<Row> = [];
+ for (const c of [1n, 2n]) {
+ for (const d of [1n, 2n]) rows.push({k: 1n, c, d, v: 1});
+ }
+ rows.push({k: 2n, c: 1n, d: 1n, v: 9});
+ await seed(Three, rows);
+ const where = [
+ {kind: 'eq', col: 'k', param: 'k'},
+ {kind: 'eq', col: 'd', param: 'd'},
+ ] as Array<WhereExpr<Row>>;
+ const params = {k: 1n, d: 1n} as CassandraParams;
+ const lc = await legacy.executeQuery({
+ cql: '__gc__',
+ params,
+ kvMeta: {action: 'count', table: Three, where} as AnyMeta,
+ });
+ const nc = await next.executeQuery({
+ cql: '__gc__',
+ params,
+ kvMeta: {action: 'count', table: Three, where} as AnyMeta,
+ });
+ expect(nc).toEqual(lc);
+ await legacy.executeQuery({cql: '__gd__', params, kvMeta: {action: 'delete', table: Three, where} as AnyMeta});
+ await next.executeQuery({cql: '__gd__', params, kvMeta: {action: 'delete', table: Three, where} as AnyMeta});
+ expect(await dump(NEXT_TABLE, Three.name)).toBe(await dump(LEGACY_TABLE, Three.name));
+ });
+
+ it('delete with a non key where column matches legacy', async () => {
+ await seed(Two, [
+ {k: 1n, c: 1n, v: 5},
+ {k: 1n, c: 2n, v: 6},
+ {k: 2n, c: 1n, v: 5},
+ ]);
+ const where = [
+ {kind: 'eq', col: 'k', param: 'k'},
+ {kind: 'eq', col: 'v', param: 'v'},
+ ] as Array<WhereExpr<Row>>;
+ const params = {k: 1n, v: 5} as CassandraParams;
+ await legacy.executeQuery({cql: '__nd__', params, kvMeta: {action: 'delete', table: Two, where} as AnyMeta});
+ await next.executeQuery({cql: '__nd__', params, kvMeta: {action: 'delete', table: Two, where} as AnyMeta});
+ expect(await dump(NEXT_TABLE, Two.name)).toBe(await dump(LEGACY_TABLE, Two.name));
+ });
+});
diff --git a/fluxer_api/src/api/database/PostgresKvOrderingAttack.test.ts b/fluxer_api/src/api/database/PostgresKvOrderingAttack.test.ts
new file mode 100644
index 000000000..627f12efb
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvOrderingAttack.test.ts
@@ -0,0 +1,544 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {
+ getDefaultPostgresClient,
+ type IPostgresClient,
+ initPostgres,
+ type PostgresQueryable,
+ shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import cassandra from 'cassandra-driver';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
+import {defineTable} from './CassandraTableDsl';
+import type {CassandraParam, CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
+import {buildCandidatePlan, ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+type AnyMeta = KvQueryMeta<Row>;
+type AnyWhere = WhereExpr<Row>;
+
+const KV = 'kv_order_attack';
+const POSTGRES_IMAGE = 'postgres:16-alpine';
+const CONTAINER = `fluxer-kvorder-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+
+const BigChild = defineTable<{owner: unknown; seq: unknown; v: unknown}, 'owner' | 'seq'>({
+ name: 'attack_bigint_child',
+ columns: ['owner', 'seq', 'v'],
+ primaryKey: ['owner', 'seq'],
+});
+
+const NumChild = defineTable<{owner: unknown; seq: unknown; v: unknown}, 'owner' | 'seq'>({
+ name: 'attack_number_child',
+ columns: ['owner', 'seq', 'v'],
+ primaryKey: ['owner', 'seq'],
+});
+
+const DateChild = defineTable<{owner: unknown; at: unknown; id: unknown; v: unknown}, 'owner' | 'at' | 'id'>({
+ name: 'attack_date_child',
+ columns: ['owner', 'at', 'id', 'v'],
+ primaryKey: ['owner', 'at', 'id'],
+});
+
+const StrChild = defineTable<{owner: unknown; seq: unknown; v: unknown}, 'owner' | 'seq'>({
+ name: 'attack_string_child',
+ columns: ['owner', 'seq', 'v'],
+ primaryKey: ['owner', 'seq'],
+});
+
+const ScanBig = defineTable<{id: unknown; v: unknown}, 'id'>({
+ name: 'attack_scan_bigint',
+ columns: ['id', 'v'],
+ primaryKey: ['id'],
+});
+
+const Messages = defineTable<
+ {channel_id: unknown; bucket: unknown; message_id: unknown; content: unknown},
+ 'channel_id' | 'bucket' | 'message_id',
+ 'channel_id' | 'bucket'
+>({
+ name: 'attack_messages',
+ columns: ['channel_id', 'bucket', 'message_id', 'content'],
+ primaryKey: ['channel_id', 'bucket', 'message_id'],
+ partitionKey: ['channel_id', 'bucket'],
+});
+
+const LocalDateChild = defineTable<{owner: unknown; seq: unknown; v: unknown}, 'owner' | 'seq'>({
+ name: 'attack_localdate_child',
+ columns: ['owner', 'seq', 'v'],
+ primaryKey: ['owner', 'seq'],
+});
+
+function eq(col: string, param: string): AnyWhere {
+ return {kind: 'eq', col, param} as AnyWhere;
+}
+
+function inClause(col: string, param: string): AnyWhere {
+ return {kind: 'in', col, param} as AnyWhere;
+}
+
+function cmp(kind: 'lt' | 'lte' | 'gt' | 'gte', col: string, param: string): AnyWhere {
+ return {kind, col, param} as AnyWhere;
+}
+
+function spec(table: unknown): KvTableSpec<Row> {
+ return table as unknown as KvTableSpec<Row>;
+}
+
+function selectMeta(table: unknown, where: Array<AnyWhere>, extra: Partial<AnyMeta> = {}): AnyMeta {
+ const s = spec(table);
+ return {action: 'select', table: s, where, columns: s.columns, ...extra} as AnyMeta;
+}
+
+function canonical(value: unknown): unknown {
+ if (value === undefined) return {__c: 'undefined'};
+ if (value === null) return null;
+ if (typeof value === 'bigint') return {__c: 'bigint', v: value.toString()};
+ if (typeof value === 'number') {
+ if (Number.isNaN(value)) return {__c: 'nan'};
+ if (!Number.isFinite(value)) return {__c: 'inf', v: value > 0 ? 1 : -1};
+ return value;
+ }
+ if (value instanceof Date) return {__c: 'date', v: value.toISOString()};
+ if (Buffer.isBuffer(value)) return {__c: 'buffer', v: value.toString('base64')};
+ if (typeof value === 'object' && value.constructor?.name === 'LocalDate') return {__c: 'localdate', v: String(value)};
+ if (Array.isArray(value)) return value.map(canonical);
+ if (typeof value === 'object') {
+ const source = value as Record<string, unknown>;
+ return {
+ __c: 'obj',
+ v: Object.keys(source)
+ .sort()
+ .map((k) => [k, canonical(source[k])]),
+ };
+ }
+ return value;
+}
+
+function fp(rows: ReadonlyArray<unknown>): string {
+ return JSON.stringify(rows.map(canonical));
+}
+
+function multiset(rows: ReadonlyArray<unknown>): string {
+ return JSON.stringify(rows.map((r) => JSON.stringify(canonical(r))).sort());
+}
+
+const suite = dockerAvailable ? describe : describe.skip;
+
+suite('PostgresKvQueryExecutor ordering and limit attack', () => {
+ let raw: IPostgresClient;
+ let legacy: LegacyPostgresKvQueryExecutor;
+ let next: PostgresKvQueryExecutor;
+
+ async function freePort(): Promise<number> {
+ return new Promise((resolve, reject) => {
+ const server = createServer();
+ server.on('error', reject);
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address();
+ if (typeof address === 'string' || address === null) {
+ reject(new Error('no port'));
+ return;
+ }
+ const port = address.port;
+ server.close(() => resolve(port));
+ });
+ });
+ }
+
+ async function sleep(ms: number): Promise<void> {
+ await new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ function client(): IPostgresClient {
+ const inner = raw;
+ return {
+ query: (text: string, values?: Array<unknown>) => inner.query(text, values),
+ connect: () => inner.connect(),
+ shutdown: async () => {},
+ isConnected: () => inner.isConnected(),
+ transaction: <T>(fn: (db: PostgresQueryable) => Promise<T>) => inner.transaction(fn),
+ kvTable: () => KV,
+ } as unknown as IPostgresClient;
+ }
+
+ async function seed(table: unknown, rows: ReadonlyArray<Row>): Promise<void> {
+ const s = spec(table);
+ await raw.query(`DELETE FROM ${KV} WHERE table_name = $1`, [s.name]);
+ const meta = {action: 'upsert', table: s} as AnyMeta;
+ for (const row of rows) {
+ await legacy.executeQuery({cql: `__seed__${s.name}`, params: row as CassandraParams, kvMeta: meta});
+ }
+ }
+
+ async function collectPages(
+ executor: LegacyPostgresKvQueryExecutor | PostgresKvQueryExecutor,
+ meta: AnyMeta,
+ params: CassandraParams,
+ pageSize: number,
+ ): Promise<Array<Array<unknown>>> {
+ const pages: Array<Array<unknown>> = [];
+ let pageState: string | null = null;
+ for (let guard = 0; guard < 200; guard += 1) {
+ const page: {rows: Array<unknown>; pageState: string | null} = await executor.executePagedQuery(
+ {cql: '__page__', params, kvMeta: meta},
+ {pageSize, pageState},
+ );
+ pages.push(page.rows);
+ pageState = page.pageState;
+ if (pageState === null) break;
+ }
+ return pages;
+ }
+
+ beforeAll(async () => {
+ const port = await freePort();
+ execFileSync(
+ 'docker',
+ [
+ 'run',
+ '-d',
+ '--name',
+ CONTAINER,
+ '-e',
+ 'POSTGRES_USER=fluxer',
+ '-e',
+ 'POSTGRES_PASSWORD=fluxer',
+ '-e',
+ 'POSTGRES_DB=fluxer',
+ '-p',
+ `127.0.0.1:${port}:5432`,
+ POSTGRES_IMAGE,
+ '-c',
+ 'fsync=off',
+ '-c',
+ 'synchronous_commit=off',
+ '-c',
+ 'full_page_writes=off',
+ ],
+ {stdio: 'ignore'},
+ );
+ let ready = false;
+ for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+ await sleep(500);
+ const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+ stdio: 'ignore',
+ });
+ if (probe.status !== 0) continue;
+ try {
+ await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 6});
+ await getDefaultPostgresClient().query('SELECT 1');
+ ready = true;
+ } catch {
+ await shutdownPostgres().catch(() => {});
+ }
+ }
+ if (!ready) throw new Error('postgres not ready');
+ raw = getDefaultPostgresClient();
+ const c = client();
+ await ensurePostgresKvSchema(c);
+ legacy = new LegacyPostgresKvQueryExecutor(c);
+ next = new PostgresKvQueryExecutor(c);
+ }, 900_000);
+
+ afterAll(async () => {
+ await shutdownPostgres().catch(() => {});
+ spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+ });
+
+ it('A1 keyset paging over a bigint clustering column keeps legacy page boundaries', async () => {
+ const seqs = [1n, 2n, 9n, 10n, 11n, 100n, 101n, 999n, 1000n];
+ await seed(
+ BigChild,
+ seqs.map((seq) => ({owner: 'o', seq, v: seq.toString()})),
+ );
+ const meta = selectMeta(BigChild, [eq('owner', 'owner')]);
+ const params: CassandraParams = {owner: 'o'};
+ const plan = buildCandidatePlan(meta, params);
+ expect(plan.exact).toBe(true);
+ expect(plan.candidates.kind).toBe('range');
+ const left = await collectPages(legacy, meta, params, 3);
+ const right = await collectPages(next, meta, params, 3);
+ expect(multiset(left.flat())).toBe(multiset(right.flat()));
+ expect(right.map((p) => p.map((r) => String((r as Row).seq)))).toStrictEqual(
+ left.map((p) => p.map((r) => String((r as Row).seq))),
+ );
+ }, 120_000);
+
+ it('A2 keyset paging over a plain number clustering column keeps legacy page boundaries', async () => {
+ const seqs = [1, 2, 9, 10, 11, 100, 1e21, -1];
+ await seed(
+ NumChild,
+ seqs.map((seq) => ({owner: 'o', seq, v: String(seq)})),
+ );
+ const meta = selectMeta(NumChild, [eq('owner', 'owner')]);
+ const params: CassandraParams = {owner: 'o'};
+ const left = await collectPages(legacy, meta, params, 3);
+ const right = await collectPages(next, meta, params, 3);
+ expect(multiset(left.flat())).toBe(multiset(right.flat()));
+ expect(fp(right.flat())).toBe(fp(left.flat()));
+ }, 120_000);
+
+ it('A3 keyset paging over a whole-table bigint scan keeps legacy page boundaries', async () => {
+ const ids = [1n, 2n, 9n, 10n, 11n, 100n, 1000n];
+ await seed(
+ ScanBig,
+ ids.map((id) => ({id, v: id.toString()})),
+ );
+ const meta = selectMeta(ScanBig, []);
+ const plan = buildCandidatePlan(meta, {});
+ expect(plan.exact).toBe(true);
+ expect(plan.candidates.kind).toBe('scan');
+ const left = await collectPages(legacy, meta, {}, 3);
+ const right = await collectPages(next, meta, {}, 3);
+ expect(multiset(left.flat())).toBe(multiset(right.flat()));
+ expect(fp(right.flat())).toBe(fp(left.flat()));
+ }, 120_000);
+
+ it('A4 keyset paging over a Date clustering column keeps legacy page boundaries', async () => {
+ const dates = [
+ new Date('1969-12-31T23:59:59.999Z'),
+ new Date(0),
+ new Date('2020-01-02T03:04:05.006Z'),
+ new Date('2024-06-07T08:09:10.011Z'),
+ new Date('2999-01-01T00:00:00.000Z'),
+ new Date(-62167219200000),
+ new Date(8640000000000000),
+ ];
+ await seed(
+ DateChild,
+ dates.map((at, index) => ({owner: 'o', at, id: BigInt(index), v: at.toISOString()})),
+ );
+ const meta = selectMeta(DateChild, [eq('owner', 'owner')]);
+ const params: CassandraParams = {owner: 'o'};
+ const left = await collectPages(legacy, meta, params, 2);
+ const right = await collectPages(next, meta, params, 2);
+ expect(multiset(left.flat())).toBe(multiset(right.flat()));
+ expect(fp(right.flat())).toBe(fp(left.flat()));
+ }, 120_000);
+
+ it('A5 keyset paging over a string clustering column keeps legacy page boundaries', async () => {
+ const seqs = ['a', 'b', 'A', 'ab', 'a b', '\u{1f600}', '�', 'a"b', 'a\\b', ''];
+ await seed(
+ StrChild,
+ seqs.map((seq, index) => ({owner: 'o', seq, v: index})),
+ );
+ const meta = selectMeta(StrChild, [eq('owner', 'owner')]);
+ const params: CassandraParams = {owner: 'o'};
+ const left = await collectPages(legacy, meta, params, 3);
+ const right = await collectPages(next, meta, params, 3);
+ expect(multiset(left.flat())).toBe(multiset(right.flat()));
+ expect(fp(right.flat())).toBe(fp(left.flat()));
+ }, 120_000);
+
+ it('A6 a keyset page is never empty while it reports more pages', async () => {
+ const seqs = [1n, 2n, 9n, 10n, 11n, 100n, 101n];
+ await seed(
+ BigChild,
+ seqs.map((seq) => ({owner: 'o', seq, v: seq.toString()})),
+ );
+ const meta = selectMeta(BigChild, [eq('owner', 'owner')]);
+ const pages = await collectPages(next, meta, {owner: 'o'}, 2);
+ for (let index = 0; index < pages.length - 1; index += 1) {
+ expect(pages[index]!.length).toBeGreaterThan(0);
+ }
+ }, 120_000);
+
+ it('B1 select with a LIMIT returns the same rows as the legacy executor', async () => {
+ const seqs = [1n, 2n, 9n, 10n, 11n, 100n, 101n, 999n, 1000n];
+ const rows: Array<Row> = [];
+ for (const owner of ['o', 'p']) for (const seq of seqs) rows.push({owner, seq, v: seq.toString()});
+ await seed(BigChild, rows);
+ const mismatches: Array<string> = [];
+ for (const limit of [1, 2, 3, 5, 100]) {
+ for (const direction of [undefined, 'ASC', 'DESC'] as const) {
+ const extra: Partial<AnyMeta> = {limit};
+ if (direction) extra.orderBy = {col: 'seq', direction} as AnyMeta['orderBy'];
+ const meta = selectMeta(BigChild, [eq('owner', 'owner')], extra);
+ const params: CassandraParams = {owner: 'o'};
+ const l = await legacy.executeQuery({cql: '__l__', params, kvMeta: meta});
+ const r = await next.executeQuery({cql: '__l__', params, kvMeta: meta});
+ if (fp(l) !== fp(r)) mismatches.push(`limit=${limit} order=${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
+ }
+ }
+ expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
+ }, 120_000);
+
+ it('B2 select with a LIMIT and an IN over the leading key returns the same rows', async () => {
+ const rows: Array<Row> = [];
+ for (const owner of ['o', 'p', 'q']) {
+ for (const seq of [1n, 2n, 9n, 10n, 11n, 100n]) rows.push({owner, seq, v: `${owner}${seq}`});
+ }
+ await seed(BigChild, rows);
+ const mismatches: Array<string> = [];
+ for (const limit of [1, 2, 4, 7, 100]) {
+ for (const direction of [undefined, 'ASC', 'DESC'] as const) {
+ const extra: Partial<AnyMeta> = {limit};
+ if (direction) extra.orderBy = {col: 'seq', direction} as AnyMeta['orderBy'];
+ const meta = selectMeta(BigChild, [inClause('owner', 'owners')], extra);
+ const params: CassandraParams = {owners: ['o', 'q', 'o'] as unknown as CassandraParam};
+ const l = await legacy.executeQuery({cql: '__l2__', params, kvMeta: meta});
+ const r = await next.executeQuery({cql: '__l2__', params, kvMeta: meta});
+ if (fp(l) !== fp(r)) mismatches.push(`limit=${limit} order=${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
+ }
+ }
+ expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
+ }, 120_000);
+
+ it('B3 select with a LIMIT and a range clause on the clustering column returns the same rows', async () => {
+ const rows: Array<Row> = [];
+ for (const owner of ['o', 'p']) {
+ for (const seq of [1n, 2n, 9n, 10n, 11n, 100n, 101n, 1000n]) rows.push({owner, seq, v: `${owner}${seq}`});
+ }
+ await seed(BigChild, rows);
+ const mismatches: Array<string> = [];
+ for (const kind of ['lt', 'lte', 'gt', 'gte'] as const) {
+ for (const bound of [1n, 10n, 11n, 100n]) {
+ for (const limit of [1, 2, 3, 100]) {
+ for (const direction of ['ASC', 'DESC'] as const) {
+ const meta = selectMeta(BigChild, [eq('owner', 'owner'), cmp(kind, 'seq', 'bound')], {
+ limit,
+ orderBy: {col: 'seq', direction} as AnyMeta['orderBy'],
+ });
+ const params: CassandraParams = {owner: 'o', bound};
+ const l = await legacy.executeQuery({cql: '__l3__', params, kvMeta: meta});
+ const r = await next.executeQuery({cql: '__l3__', params, kvMeta: meta});
+ if (fp(l) !== fp(r)) {
+ mismatches.push(`${kind} ${bound} limit=${limit} ${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
+ }
+ }
+ }
+ }
+ }
+ expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
+ }, 240_000);
+
+ it('B4 message history shaped select returns the same page as the legacy executor', async () => {
+ const rows: Array<Row> = [];
+ for (const channel of [10n, 11n]) {
+ for (const bucket of [0, 1, 2, 10, 11]) {
+ for (const messageId of [1n, 2n, 9n, 10n, 11n, 100n, 101n, 999n, 1000n, 9999999999999999999n]) {
+ rows.push({channel_id: channel, bucket, message_id: messageId, content: `${channel}/${bucket}/${messageId}`});
+ }
+ }
+ }
+ await seed(Messages, rows);
+ const mismatches: Array<string> = [];
+ for (const bucket of [0, 1, 10]) {
+ for (const cursor of [1n, 10n, 11n, 100n, 1000n, 9999999999999999999n]) {
+ for (const kind of ['lt', 'gt'] as const) {
+ for (const direction of ['ASC', 'DESC'] as const) {
+ for (const limit of [1, 3, 50]) {
+ const meta = selectMeta(
+ Messages,
+ [eq('channel_id', 'channel_id'), eq('bucket', 'bucket'), cmp(kind, 'message_id', 'cursor')],
+ {limit, orderBy: {col: 'message_id', direction} as AnyMeta['orderBy']},
+ );
+ const params: CassandraParams = {channel_id: 10n, bucket, cursor};
+ const l = await legacy.executeQuery({cql: '__b4__', params, kvMeta: meta});
+ const r = await next.executeQuery({cql: '__b4__', params, kvMeta: meta});
+ if (fp(l) !== fp(r)) {
+ mismatches.push(`bucket=${bucket} ${kind} ${cursor} ${direction} limit=${limit}`);
+ }
+ }
+ }
+ }
+ }
+ }
+ expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
+ }, 300_000);
+
+ it('B5 select with a LIMIT under a demoted over-cap cartesian plan returns the same rows', async () => {
+ const rows: Array<Row> = [];
+ for (const owner of ['o', 'p', 'q']) {
+ for (const seq of [1n, 2n, 9n, 10n, 11n, 100n]) rows.push({owner, seq, v: `${owner}${seq}`});
+ }
+ await seed(BigChild, rows);
+ const owners = Array.from({length: 40}, (_unused, index) => `x${index}`);
+ owners.push('o', 'q');
+ const seqs = Array.from({length: 40}, (_unused, index) => BigInt(index));
+ seqs.push(100n);
+ const mismatches: Array<string> = [];
+ for (const limit of [1, 3, 100]) {
+ for (const direction of ['ASC', 'DESC'] as const) {
+ const meta = selectMeta(BigChild, [inClause('owner', 'owners'), inClause('seq', 'seqs')], {
+ limit,
+ orderBy: {col: 'seq', direction} as AnyMeta['orderBy'],
+ });
+ const params: CassandraParams = {
+ owners: owners as unknown as CassandraParam,
+ seqs: seqs as unknown as CassandraParam,
+ };
+ const l = await legacy.executeQuery({cql: '__b5__', params, kvMeta: meta});
+ const r = await next.executeQuery({cql: '__b5__', params, kvMeta: meta});
+ if (fp(l) !== fp(r)) mismatches.push(`limit=${limit} ${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
+ }
+ }
+ expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
+ }, 180_000);
+
+ it('B6 select ordered by a column outside the projection returns the same rows', async () => {
+ const rows: Array<Row> = [];
+ for (const seq of [1n, 2n, 9n, 10n, 11n, 100n]) rows.push({owner: 'o', seq, v: `v${seq}`});
+ await seed(BigChild, rows);
+ const mismatches: Array<string> = [];
+ for (const limit of [1, 3, 100]) {
+ for (const direction of ['ASC', 'DESC'] as const) {
+ const meta = selectMeta(BigChild, [eq('owner', 'owner')], {
+ columns: ['v'],
+ limit,
+ orderBy: {col: 'seq', direction} as AnyMeta['orderBy'],
+ });
+ const params: CassandraParams = {owner: 'o'};
+ const l = await legacy.executeQuery({cql: '__b6__', params, kvMeta: meta});
+ const r = await next.executeQuery({cql: '__b6__', params, kvMeta: meta});
+ if (fp(l) !== fp(r)) mismatches.push(`limit=${limit} ${direction}\n legacy ${fp(l)}\n next ${fp(r)}`);
+ }
+ }
+ expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
+ }, 180_000);
+
+ it('B7 keyset page sizes match legacy page sizes exactly', async () => {
+ const rows: Array<Row> = [];
+ for (const seq of [1n, 2n, 9n, 10n, 11n, 100n, 101n]) rows.push({owner: 'o', seq, v: `v${seq}`});
+ await seed(BigChild, rows);
+ const meta = selectMeta(BigChild, [eq('owner', 'owner')]);
+ const mismatches: Array<string> = [];
+ for (const pageSize of [1, 2, 3, 6, 7, 8, 100]) {
+ const left = await collectPages(legacy, meta, {owner: 'o'}, pageSize);
+ const right = await collectPages(next, meta, {owner: 'o'}, pageSize);
+ const l = left.map((p) => p.length).join(',');
+ const r = right.map((p) => p.length).join(',');
+ if (l !== r) mismatches.push(`pageSize=${pageSize} legacy=${l} next=${r}`);
+ if (multiset(left.flat()) !== multiset(right.flat())) mismatches.push(`pageSize=${pageSize} rowset`);
+ }
+ expect(mismatches.join('\n'), mismatches.join('\n')).toBe('');
+ }, 180_000);
+
+ it('C1 a LocalDate key value is still found by an equal string parameter', async () => {
+ await seed(LocalDateChild, [
+ {owner: cassandra.types.LocalDate.fromString('2020-01-01'), seq: 1n, v: 'localdate'},
+ {owner: '2020-01-01', seq: 2n, v: 'string'},
+ ]);
+ const meta = selectMeta(LocalDateChild, [eq('owner', 'owner')], {limit: 10});
+ const params: CassandraParams = {owner: '2020-01-01'};
+ const l = await legacy.executeQuery({cql: '__c1__', params, kvMeta: meta});
+ const r = await next.executeQuery({cql: '__c1__', params, kvMeta: meta});
+ expect(fp(r), `legacy ${fp(l)} next ${fp(r)}`).toBe(fp(l));
+ }, 120_000);
+
+ it('C2 a string key value is still found by an equal LocalDate parameter', async () => {
+ await seed(LocalDateChild, [
+ {owner: cassandra.types.LocalDate.fromString('2020-01-01'), seq: 1n, v: 'localdate'},
+ {owner: '2020-01-01', seq: 2n, v: 'string'},
+ ]);
+ const meta = selectMeta(LocalDateChild, [eq('owner', 'owner')], {limit: 10});
+ const params: CassandraParams = {owner: cassandra.types.LocalDate.fromString('2020-01-01')};
+ const l = await legacy.executeQuery({cql: '__c2__', params, kvMeta: meta});
+ const r = await next.executeQuery({cql: '__c2__', params, kvMeta: meta});
+ expect(fp(r), `legacy ${fp(l)} next ${fp(r)}`).toBe(fp(l));
+ }, 120_000);
+});
diff --git a/fluxer_api/src/api/database/PostgresKvPagingAdversarial.test.ts b/fluxer_api/src/api/database/PostgresKvPagingAdversarial.test.ts
new file mode 100644
index 000000000..dd7f2cbd0
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvPagingAdversarial.test.ts
@@ -0,0 +1,562 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {
+ getDefaultPostgresClient,
+ type IPostgresClient,
+ initPostgres,
+ type PostgresQueryable,
+ shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
+import type {CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
+import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+type AnyMeta = KvQueryMeta<Row>;
+type AnyExec = LegacyPostgresKvQueryExecutor | PostgresKvQueryExecutor;
+
+const KV = 'kv_pagead';
+const ICU_KV = 'kv_pagead_icu';
+const ICU_COLLATION = 'kvpagead_icu';
+const POSTGRES_IMAGE = 'postgres:16-alpine';
+const CONTAINER = `fluxer-kvpage-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+
+const PagedTable: KvTableSpec<Row> = {
+ name: 'pagead_items',
+ columns: ['owner_id', 'item_id', 'payload'],
+ primaryKey: ['owner_id', 'item_id'],
+ partitionKey: ['owner_id', 'item_id'],
+};
+
+const FlatTable: KvTableSpec<Row> = {
+ name: 'pagead_flat',
+ columns: ['k', 'v'],
+ primaryKey: ['k'],
+ partitionKey: ['k'],
+};
+
+class PlainClient implements IPostgresClient {
+ constructor(
+ private readonly inner: IPostgresClient,
+ private readonly table: string,
+ ) {}
+ async query<T extends Record<string, unknown>>(text: string, values: Array<unknown> = []) {
+ return (await this.inner.query(text, values)) as unknown as Awaited<ReturnType<IPostgresClient['query']>> & {
+ rows: Array<T>;
+ };
+ }
+ async connect(): Promise<void> {
+ await this.inner.connect();
+ }
+ async shutdown(): Promise<void> {}
+ isConnected(): boolean {
+ return this.inner.isConnected();
+ }
+ async transaction<T>(fn: (client: PostgresQueryable) => Promise<T>): Promise<T> {
+ return this.inner.transaction(fn);
+ }
+ kvTable(): string {
+ return this.table;
+ }
+}
+
+function selectMeta(spec: KvTableSpec<Row>, where: Array<WhereExpr<Row>> = [], extra: Partial<AnyMeta> = {}): AnyMeta {
+ return {action: 'select', table: spec, where, columns: spec.columns, ...extra} as AnyMeta;
+}
+
+const upsertMeta = (spec: KvTableSpec<Row>) => ({action: 'upsert', table: spec}) as AnyMeta;
+const deleteMeta = (spec: KvTableSpec<Row>, where: Array<WhereExpr<Row>>) =>
+ ({action: 'delete', table: spec, where}) as AnyMeta;
+
+function token(value: unknown): string {
+ return Buffer.from(JSON.stringify(value)).toString('base64url');
+}
+
+describe.skipIf(!dockerAvailable)('postgres kv paging adversarial', () => {
+ let raw: IPostgresClient;
+ let legacy: LegacyPostgresKvQueryExecutor;
+ let next: PostgresKvQueryExecutor;
+ let icuNext: PostgresKvQueryExecutor;
+ let icuLegacy: LegacyPostgresKvQueryExecutor;
+
+ async function sleep(ms: number): Promise<void> {
+ await new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ async function freePort(): Promise<number> {
+ return new Promise((resolve, reject) => {
+ const server = createServer();
+ server.on('error', reject);
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address();
+ if (typeof address === 'string' || address === null) {
+ reject(new Error('no port'));
+ return;
+ }
+ const port = address.port;
+ server.close(() => resolve(port));
+ });
+ });
+ }
+
+ async function upsert(exec: AnyExec, spec: KvTableSpec<Row>, row: Row) {
+ await exec.executeQuery({cql: `__seed__${spec.name}`, params: row as CassandraParams, kvMeta: upsertMeta(spec)});
+ }
+
+ async function wipe(kv: string) {
+ await raw.query(`DELETE FROM ${kv}`);
+ }
+
+ async function pageAll(
+ exec: AnyExec,
+ meta: AnyMeta,
+ params: CassandraParams,
+ pageSize: number,
+ hook?: (pageIndex: number) => Promise<void>,
+ ): Promise<{pages: Array<Array<Row>>; error: string | null}> {
+ const pages: Array<Array<Row>> = [];
+ let pageState: string | null = null;
+ try {
+ for (let guard = 0; guard < 400; guard += 1) {
+ const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery<Row>(
+ {cql: `__page__${meta.table.name}`, params, kvMeta: meta},
+ {pageSize, pageState},
+ );
+ pages.push(page.rows);
+ pageState = page.pageState;
+ if (hook) await hook(guard);
+ if (pageState === null) break;
+ if (guard === 399) return {pages, error: 'NON_TERMINATING'};
+ }
+ } catch (error) {
+ return {pages, error: (error as Error).message};
+ }
+ return {pages, error: null};
+ }
+
+ beforeAll(async () => {
+ const port = await freePort();
+ execFileSync(
+ 'docker',
+ [
+ 'run',
+ '-d',
+ '--name',
+ CONTAINER,
+ '-e',
+ 'POSTGRES_USER=fluxer',
+ '-e',
+ 'POSTGRES_PASSWORD=fluxer',
+ '-e',
+ 'POSTGRES_DB=fluxer',
+ '-p',
+ `127.0.0.1:${port}:5432`,
+ POSTGRES_IMAGE,
+ '-c',
+ 'fsync=off',
+ '-c',
+ 'synchronous_commit=off',
+ ],
+ {stdio: 'ignore'},
+ );
+ let ready = false;
+ for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+ await sleep(500);
+ const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+ stdio: 'ignore',
+ });
+ if (probe.status !== 0) continue;
+ try {
+ await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 8});
+ await getDefaultPostgresClient().query('SELECT 1');
+ ready = true;
+ } catch {
+ await shutdownPostgres().catch(() => {});
+ }
+ }
+ if (!ready) throw new Error('postgres not ready');
+ raw = getDefaultPostgresClient();
+ const kvClient = new PlainClient(raw, KV);
+ const icuClient = new PlainClient(raw, ICU_KV);
+ await ensurePostgresKvSchema(kvClient);
+ await raw.query(`CREATE COLLATION ${ICU_COLLATION} (provider = icu, locale = 'en-US')`);
+ await ensurePostgresKvSchema(icuClient);
+ for (const column of ['row_key', 'partition_key']) {
+ await raw.query(`ALTER TABLE ${ICU_KV} ALTER COLUMN ${column} TYPE text COLLATE ${ICU_COLLATION}`);
+ }
+ legacy = new LegacyPostgresKvQueryExecutor(kvClient);
+ next = new PostgresKvQueryExecutor(kvClient);
+ icuLegacy = new LegacyPostgresKvQueryExecutor(icuClient);
+ icuNext = new PostgresKvQueryExecutor(icuClient);
+ }, 900_000);
+
+ afterAll(async () => {
+ await shutdownPostgres().catch(() => {});
+ spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+ });
+
+ it('pages a bare scan without skipping or repeating rows', async () => {
+ await wipe(KV);
+ for (let i = 0; i < 23; i += 1) await upsert(next, FlatTable, {k: BigInt(i), v: `v${i}`});
+ for (const pageSize of [1, 2, 3, 5, 7, 23, 24, 100]) {
+ const result = await pageAll(next, selectMeta(FlatTable), {}, pageSize);
+ expect(result.error, `pageSize=${pageSize}`).toBeNull();
+ const keys = result.pages.flat().map((row) => String(row.k));
+ expect(new Set(keys).size, `dupes at pageSize=${pageSize}: ${keys.join(',')}`).toBe(23);
+ expect(keys.length, `count at pageSize=${pageSize}`).toBe(23);
+ }
+ }, 300_000);
+
+ it('pages a prefix range without skipping or repeating rows', async () => {
+ await wipe(KV);
+ for (let owner = 0; owner < 3; owner += 1) {
+ for (let i = 0; i < 17; i += 1) {
+ await upsert(next, PagedTable, {owner_id: BigInt(owner), item_id: BigInt(i), payload: `p${owner}-${i}`});
+ }
+ }
+ const meta = selectMeta(PagedTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]);
+ for (const pageSize of [1, 2, 4, 17, 18]) {
+ const result = await pageAll(next, meta, {owner_id: 1n}, pageSize);
+ expect(result.error).toBeNull();
+ const rows = result.pages.flat();
+ expect(rows.length, `pageSize=${pageSize}`).toBe(17);
+ expect(new Set(rows.map((r) => String(r.item_id))).size).toBe(17);
+ expect(rows.every((r) => String(r.owner_id) === '1')).toBe(true);
+ }
+ }, 300_000);
+
+ it('never loses a row that existed for the whole scan while rows are inserted between pages', async () => {
+ await wipe(KV);
+ const stable = new Set<string>();
+ for (let i = 0; i < 30; i += 1) {
+ await upsert(next, FlatTable, {k: BigInt(i), v: `v${i}`});
+ stable.add(String(i));
+ }
+ let inserted = 100;
+ const result = await pageAll(next, selectMeta(FlatTable), {}, 4, async () => {
+ await upsert(next, FlatTable, {k: BigInt(inserted), v: `late${inserted}`});
+ inserted += 1;
+ });
+ expect(result.error).toBeNull();
+ const seen = result.pages.flat().map((r) => String(r.k));
+ const dupes = seen.filter((key, index) => seen.indexOf(key) !== index);
+ const missing = [...stable].filter((key) => !seen.includes(key));
+ expect(missing, `missing stable rows: ${missing.join(',')}`).toEqual([]);
+ expect(dupes, `duplicate rows: ${dupes.join(',')}`).toEqual([]);
+ }, 300_000);
+
+ it('pages identically to the legacy executor while rows are deleted between pages', async () => {
+ const run = async (exec: AnyExec) => {
+ await wipe(KV);
+ for (let i = 0; i < 30; i += 1) await upsert(exec, FlatTable, {k: BigInt(i), v: `v${i}`});
+ let pageIndex = 0;
+ const result = await pageAll(exec, selectMeta(FlatTable), {}, 4, async () => {
+ const victim = pageIndex;
+ pageIndex += 1;
+ await exec.executeQuery({
+ cql: '__del__',
+ params: {k: BigInt(victim)},
+ kvMeta: deleteMeta(FlatTable, [{kind: 'eq', col: 'k', param: 'k'} as WhereExpr<Row>]),
+ });
+ });
+ return {
+ error: result.error,
+ pages: result.pages.map((page) => page.map((row) => String(row.k)).join(',')).join('|'),
+ };
+ };
+ const legacyRun = await run(legacy);
+ const nextRun = await run(next);
+ expect(nextRun.error).toBeNull();
+ expect(nextRun, `legacy=${legacyRun.pages} next=${nextRun.pages}`).toEqual(legacyRun);
+ }, 300_000);
+
+ it('pages identically to the legacy executor when the last row of each page is deleted', async () => {
+ const run = async (exec: AnyExec) => {
+ await wipe(KV);
+ for (let i = 0; i < 20; i += 1)
+ await upsert(exec, FlatTable, {k: `key${String(i).padStart(2, '0')}`, v: `v${i}`});
+ const meta = selectMeta(FlatTable);
+ const seen: Array<string> = [];
+ let pageState: string | null = null;
+ for (let guard = 0; guard < 100; guard += 1) {
+ const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery<Row>(
+ {cql: '__cur__', params: {}, kvMeta: meta},
+ {pageSize: 3, pageState},
+ );
+ for (const row of page.rows) seen.push(String(row.k));
+ const cursorRow = page.rows[page.rows.length - 1];
+ pageState = page.pageState;
+ if (cursorRow) {
+ await exec.executeQuery({
+ cql: '__delcur__',
+ params: {k: cursorRow.k as string},
+ kvMeta: deleteMeta(FlatTable, [{kind: 'eq', col: 'k', param: 'k'} as WhereExpr<Row>]),
+ });
+ }
+ if (pageState === null) break;
+ }
+ return seen.join(',');
+ };
+ const legacySeen = await run(legacy);
+ const nextSeen = await run(next);
+ expect(nextSeen, `legacy=${legacySeen} next=${nextSeen}`).toBe(legacySeen);
+ }, 300_000);
+
+ it('never returns an empty page together with a non-null page state', async () => {
+ await wipe(KV);
+ for (let owner = 0; owner < 4; owner += 1) {
+ for (let i = 0; i < 11; i += 1) {
+ await upsert(next, PagedTable, {owner_id: BigInt(owner), item_id: BigInt(i), payload: `p${owner}-${i}`});
+ }
+ }
+ const shapes: Array<[string, AnyMeta, CassandraParams]> = [
+ ['scan', selectMeta(PagedTable), {}],
+ [
+ 'eq-owner',
+ selectMeta(PagedTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]),
+ {owner_id: 2n},
+ ],
+ [
+ 'in-owner-single',
+ selectMeta(PagedTable, [{kind: 'in', col: 'owner_id', param: 'owner_ids'} as WhereExpr<Row>]),
+ {owner_ids: [2n]},
+ ],
+ [
+ 'in-owner-multi',
+ selectMeta(PagedTable, [{kind: 'in', col: 'owner_id', param: 'owner_ids'} as WhereExpr<Row>]),
+ {owner_ids: [1n, 3n]},
+ ],
+ [
+ 'in-owner-empty',
+ selectMeta(PagedTable, [{kind: 'in', col: 'owner_id', param: 'owner_ids'} as WhereExpr<Row>]),
+ {owner_ids: []},
+ ],
+ ];
+ const offenders: Array<string> = [];
+ for (const [name, meta, params] of shapes) {
+ for (const pageSize of [1, 2, 3, 5, 11]) {
+ let pageState: string | null = null;
+ const flat: Array<Row> = [];
+ for (let guard = 0; guard < 200; guard += 1) {
+ const page: {rows: Array<Row>; pageState: string | null} = await next.executePagedQuery<Row>(
+ {cql: `__empty__${name}`, params, kvMeta: meta},
+ {pageSize, pageState},
+ );
+ if (page.rows.length === 0 && page.pageState !== null) {
+ offenders.push(`${name}/size${pageSize} page ${guard}`);
+ }
+ flat.push(...page.rows);
+ pageState = page.pageState;
+ if (pageState === null) break;
+ }
+ const unpaged = await next.executeQuery<Row>({cql: `__whole__${name}`, params, kvMeta: meta});
+ expect(flat.length, `${name}/size${pageSize} row count`).toBe(unpaged.length);
+ const pagedKeys = flat.map((r) => `${String(r.owner_id)}:${String(r.item_id)}`).sort();
+ const wholeKeys = unpaged.map((r) => `${String(r.owner_id)}:${String(r.item_id)}`).sort();
+ expect(pagedKeys, `${name}/size${pageSize} multiset`).toEqual(wholeKeys);
+ }
+ }
+ expect(offenders, `empty page with continuation: ${offenders.join(', ')}`).toEqual([]);
+ }, 300_000);
+
+ it('matches legacy throw/no-throw behaviour for hand-crafted and corrupt page states', async () => {
+ await wipe(KV);
+ for (let i = 0; i < 9; i += 1) await upsert(next, FlatTable, {k: BigInt(i), v: `v${i}`});
+ const meta = selectMeta(FlatTable);
+ const query = {cql: '__tok__', params: {}, kvMeta: meta};
+ const tokens: Array<[string, string]> = [
+ ['empty-object', token({})],
+ ['number', token(123)],
+ ['string', token('hello')],
+ ['array', token([1, 2])],
+ ['unknown-key', token({foo: 1})],
+ ['offset-negative', token({offset: -1})],
+ ['offset-fractional', token({offset: 1.5})],
+ ['offset-string', token({offset: '2'})],
+ ['offset-huge', token({offset: Number.MAX_SAFE_INTEGER})],
+ ['after-number', token({after: 5})],
+ ['after-null', token({after: null})],
+ ['after-object', token({after: {a: 1}})],
+ ['after-and-offset', token({after: 'x', offset: 3})],
+ ['garbage', 'not-base64-at-all'],
+ ];
+ const deltas: Array<string> = [];
+ for (const [name, value] of tokens) {
+ const legacyResult = await legacy
+ .executePagedQuery<Row>(query, {pageSize: 3, pageState: value})
+ .then((r) => ({err: null as string | null, rows: r.rows.length}))
+ .catch((e: Error) => ({err: e.message, rows: -1}));
+ const nextResult = await next
+ .executePagedQuery<Row>(query, {pageSize: 3, pageState: value})
+ .then((r) => ({err: null as string | null, rows: r.rows.length}))
+ .catch((e: Error) => ({err: e.message, rows: -1}));
+ if ((legacyResult.err === null) !== (nextResult.err === null)) {
+ deltas.push(
+ `${name}: legacy=${legacyResult.err ?? `ok(${legacyResult.rows})`} next=${nextResult.err ?? `ok(${nextResult.rows})`}`,
+ );
+ }
+ }
+ expect(deltas, `page-state error behaviour deltas:\n${deltas.join('\n')}`).toEqual([]);
+ }, 300_000);
+
+ it('matches legacy behaviour for degenerate page sizes', async () => {
+ await wipe(KV);
+ for (let i = 0; i < 9; i += 1) await upsert(next, FlatTable, {k: BigInt(i), v: `v${i}`});
+ const query = {cql: '__ps__', params: {}, kvMeta: selectMeta(FlatTable)};
+ const deltas: Array<string> = [];
+ for (const pageSize of [0, -1, 1.5, 2.9, Number.MAX_SAFE_INTEGER, 2 ** 31, Number.NaN]) {
+ const l = await legacy
+ .executePagedQuery<Row>(query, {pageSize})
+ .then((r) => `ok rows=${r.rows.length} more=${r.pageState !== null}`)
+ .catch((e: Error) => `throw ${e.message}`);
+ const n = await next
+ .executePagedQuery<Row>(query, {pageSize})
+ .then((r) => `ok rows=${r.rows.length} more=${r.pageState !== null}`)
+ .catch((e: Error) => `throw ${e.message}`);
+ if (l !== n) deltas.push(`pageSize=${String(pageSize)}: legacy=${l} next=${n}`);
+ }
+ expect(deltas, `page size deltas:\n${deltas.join('\n')}`).toEqual([]);
+ }, 300_000);
+
+ it('does not silently truncate when a page token is reused across a different query', async () => {
+ await wipe(KV);
+ for (let owner = 0; owner < 3; owner += 1) {
+ for (let i = 0; i < 9; i += 1) {
+ await upsert(next, PagedTable, {owner_id: BigInt(owner), item_id: BigInt(i), payload: `p${owner}-${i}`});
+ }
+ }
+ const meta = selectMeta(PagedTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]);
+ const firstNext = await next.executePagedQuery<Row>(
+ {cql: '__x__', params: {owner_id: 0n}, kvMeta: meta},
+ {pageSize: 3},
+ );
+ const firstLegacy = await legacy.executePagedQuery<Row>(
+ {cql: '__x__', params: {owner_id: 0n}, kvMeta: meta},
+ {pageSize: 3},
+ );
+ expect(firstNext.pageState).not.toBeNull();
+ const crossNext = await next.executePagedQuery<Row>(
+ {cql: '__x__', params: {owner_id: 2n}, kvMeta: meta},
+ {pageSize: 3, pageState: firstNext.pageState},
+ );
+ const crossLegacy = await legacy.executePagedQuery<Row>(
+ {cql: '__x__', params: {owner_id: 2n}, kvMeta: meta},
+ {pageSize: 3, pageState: firstLegacy.pageState},
+ );
+ expect(crossNext.rows.length, 'cross-query token row count diverges from legacy').toBe(crossLegacy.rows.length);
+ }, 300_000);
+
+ it('does not restart the scan when a keyset token reaches a non-keyset plan', async () => {
+ await wipe(KV);
+ for (let owner = 0; owner < 3; owner += 1) {
+ for (let i = 0; i < 9; i += 1) {
+ await upsert(next, PagedTable, {owner_id: BigInt(owner), item_id: BigInt(i), payload: `p${owner}-${i}`});
+ }
+ }
+ const inMeta = selectMeta(PagedTable, [{kind: 'in', col: 'owner_id', param: 'owner_ids'} as WhereExpr<Row>]);
+ const first = await next.executePagedQuery<Row>(
+ {cql: '__flip__', params: {owner_ids: [1n]}, kvMeta: inMeta},
+ {pageSize: 3},
+ );
+ expect(first.pageState).not.toBeNull();
+ const flipped = await next.executePagedQuery<Row>(
+ {cql: '__flip__', params: {owner_ids: [1n, 2n]}, kvMeta: inMeta},
+ {pageSize: 3, pageState: first.pageState},
+ );
+ const firstKeys = first.rows.map((r) => String(r.item_id));
+ const flippedKeys = flipped.rows.map((r) => String(r.item_id));
+ const overlap = flippedKeys.filter((key) => firstKeys.includes(key));
+ expect(overlap, `keyset token silently restarted the scan: first=${firstKeys} flipped=${flippedKeys}`).toEqual([]);
+ }, 300_000);
+
+ it('pages correctly on a database whose row_key column has a linguistic collation', async () => {
+ await wipe(ICU_KV);
+ const keys = ['a', 'A', 'a b', 'ab', 'B', 'b', '\u{1f600}', 'z', 'Z', '', '"', '\\', 'ab'];
+ for (const key of keys) await upsert(icuNext, FlatTable, {k: key, v: `v:${key}`});
+ for (const pageSize of [1, 2, 3, 5]) {
+ const result = await pageAll(icuNext, selectMeta(FlatTable), {}, pageSize);
+ expect(result.error, `icu pageSize=${pageSize}`).toBeNull();
+ const seen = result.pages.flat().map((r) => String(r.k));
+ const dupes = seen.filter((key, index) => seen.indexOf(key) !== index);
+ expect(dupes, `icu dupes pageSize=${pageSize}: ${JSON.stringify(dupes)}`).toEqual([]);
+ expect(seen.length, `icu count pageSize=${pageSize}`).toBe(keys.length);
+ }
+ const legacyAll = await pageAll(icuLegacy, selectMeta(FlatTable), {}, 3);
+ expect(legacyAll.pages.flat().length).toBe(keys.length);
+ }, 300_000);
+
+ it('reports the paged order delta against legacy for key shapes that reach real callers', async () => {
+ const DatedTable: KvTableSpec<Row> = {
+ name: 'pagead_dated',
+ columns: ['customer_id', 'created_at', 'provider_id', 'total'],
+ primaryKey: ['customer_id', 'created_at', 'provider_id'],
+ partitionKey: ['customer_id'],
+ };
+ const NumericTable: KvTableSpec<Row> = {
+ name: 'pagead_numeric',
+ columns: ['owner_id', 'seq', 'payload'],
+ primaryKey: ['owner_id', 'seq'],
+ partitionKey: ['owner_id'],
+ };
+ await wipe(KV);
+ for (let i = 0; i < 15; i += 1) {
+ await upsert(next, DatedTable, {
+ customer_id: 'cus_1',
+ created_at: new Date(Date.UTC(2024, 0, 1 + i, i, i)),
+ provider_id: `in_${String(i).padStart(3, '0')}`,
+ total: BigInt(i),
+ });
+ await upsert(next, NumericTable, {owner_id: 7n, seq: BigInt(i), payload: `p${i}`});
+ }
+ const datedMeta = selectMeta(DatedTable, [
+ {kind: 'eq', col: 'customer_id', param: 'customer_id'} as WhereExpr<Row>,
+ ]);
+ const numericMeta = selectMeta(NumericTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]);
+ const datedLegacy = await pageAll(legacy, datedMeta, {customer_id: 'cus_1'}, 4);
+ const datedNext = await pageAll(next, datedMeta, {customer_id: 'cus_1'}, 4);
+ const numericLegacy = await pageAll(legacy, numericMeta, {owner_id: 7n}, 4);
+ const numericNext = await pageAll(next, numericMeta, {owner_id: 7n}, 4);
+ const datedLegacyOrder = datedLegacy.pages.flat().map((r) => String(r.provider_id));
+ const datedNextOrder = datedNext.pages.flat().map((r) => String(r.provider_id));
+ const numericLegacyOrder = numericLegacy.pages.flat().map((r) => String(r.seq));
+ const numericNextOrder = numericNext.pages.flat().map((r) => String(r.seq));
+ const deltas: Array<string> = [];
+ if (datedLegacyOrder.join(',') !== datedNextOrder.join(',')) {
+ deltas.push(`date-keyed: legacy=${datedLegacyOrder.join(',')} next=${datedNextOrder.join(',')}`);
+ }
+ if (numericLegacyOrder.join(',') !== numericNextOrder.join(',')) {
+ deltas.push(`bigint-keyed: legacy=${numericLegacyOrder.join(',')} next=${numericNextOrder.join(',')}`);
+ }
+ expect(datedNextOrder.slice().sort(), 'date-keyed row set').toEqual(datedLegacyOrder.slice().sort());
+ expect(numericNextOrder.slice().sort(), 'bigint-keyed row set').toEqual(numericLegacyOrder.slice().sort());
+ expect(deltas, `paged order deltas:\n${deltas.join('\n')}`).toEqual([]);
+ }, 300_000);
+
+ it('pages a prefix range whose keys sit adjacent to the range bounds', async () => {
+ await wipe(KV);
+ const owners = ['a', 'a b', 'ab', 'a', 'a"'];
+ for (const owner of owners) {
+ for (let i = 0; i < 5; i += 1) {
+ await upsert(next, PagedTable, {owner_id: owner, item_id: BigInt(i), payload: `${owner}#${i}`});
+ }
+ }
+ const meta = selectMeta(PagedTable, [{kind: 'eq', col: 'owner_id', param: 'owner_id'} as WhereExpr<Row>]);
+ for (const owner of owners) {
+ for (const pageSize of [1, 2, 5]) {
+ const result = await pageAll(next, meta, {owner_id: owner}, pageSize);
+ expect(result.error).toBeNull();
+ const rows = result.pages.flat();
+ expect(rows.length, `owner=${JSON.stringify(owner)} pageSize=${pageSize}`).toBe(5);
+ expect(
+ rows.every((r) => r.owner_id === owner),
+ `owner leak for ${JSON.stringify(owner)}`,
+ ).toBe(true);
+ }
+ }
+ }, 300_000);
+});
diff --git a/fluxer_api/src/api/database/PostgresKvQueryDifferential.test.ts b/fluxer_api/src/api/database/PostgresKvQueryDifferential.test.ts
new file mode 100644
index 000000000..41bc3f861
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvQueryDifferential.test.ts
@@ -0,0 +1,1467 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {
+ getDefaultPostgresClient,
+ type IPostgresClient,
+ initPostgres,
+ type PostgresQueryable,
+ shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import cassandra from 'cassandra-driver';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+import * as Tables from '../Tables';
+import {LegacyPostgresKvQueryExecutor} from './__testref__/LegacyPostgresKvQueryExecutor';
+import {defineTable} from './CassandraTableDsl';
+import type {CassandraParam, CassandraParams, KvQueryMeta, KvTableSpec, WhereExpr} from './CassandraTypes';
+import {buildCandidatePlan, ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+type AnyMeta = KvQueryMeta<Row>;
+type AnyWhere = WhereExpr<Row>;
+type AnyExecutor = LegacyPostgresKvQueryExecutor | PostgresKvQueryExecutor;
+
+interface Shape {
+ name: string;
+ meta: AnyMeta;
+ params: CassandraParams;
+}
+
+interface Outcome {
+ fingerprint: string;
+ rowsRead: number;
+ threw: string | null;
+}
+
+interface Mismatch {
+ kind: 'rowset' | 'order' | 'throw' | 'rowsread' | 'state';
+ table: string;
+ shape: string;
+ legacy: string;
+ next: string;
+}
+
+const LEGACY_TABLE = 'kv_legacy';
+const NEXT_TABLE = 'kv_next';
+const SEED_TABLE = 'kv_seed';
+const ICU_TABLE = 'kv_icu';
+const ICU_COLLATION = 'kvdiff_icu';
+const MAX_ROWS_PER_TABLE = 24;
+const POSTGRES_IMAGE = 'postgres:16-alpine';
+const CONTAINER = `fluxer-kvdiff-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+
+const dockerAvailable = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+
+const BIGINT_DOMAIN: ReadonlyArray<CassandraParam> = [1n, 2n, 9n, 10n, 11n, 100n, -1n, 99999999999999999999n, 0n];
+const NUMBER_DOMAIN: ReadonlyArray<CassandraParam> = [0, 1, 2, 9, 10, 11, 100, -1, 1e21];
+const STRING_DOMAIN: ReadonlyArray<CassandraParam> = [
+ '',
+ 'a',
+ 'b',
+ 'a"b',
+ 'a\\b',
+ 'a\u0001b',
+ 'a\u001fb',
+ 'a b',
+ 'a\u007fb',
+ '\u{1f600}',
+ '\ufffd',
+ 'z',
+];
+const DATE_DOMAIN: ReadonlyArray<CassandraParam> = [
+ new Date(0),
+ new Date('2020-01-02T03:04:05.006Z'),
+ new Date('2021-06-07T08:09:10.011Z'),
+ new Date('1969-12-31T23:59:59.999Z'),
+ new Date('2999-01-01T00:00:00.000Z'),
+];
+const BUFFER_DOMAIN: ReadonlyArray<CassandraParam> = [
+ Buffer.alloc(0),
+ Buffer.from('a', 'utf8'),
+ Buffer.from([0x00]),
+ Buffer.from([0x1f]),
+ Buffer.from([0x20]),
+ Buffer.from('\u{1f600}', 'utf8'),
+];
+const BOOLEAN_DOMAIN: ReadonlyArray<CassandraParam> = [true, false];
+const NULLABLE_STRING_DOMAIN: ReadonlyArray<CassandraParam> = [null, 'a', 'b', '', 'a\u001fb'];
+const LOCAL_DATE_DOMAIN: ReadonlyArray<CassandraParam> = [
+ cassandra.types.LocalDate.fromString('2020-01-01'),
+ cassandra.types.LocalDate.fromString('2020-01-02'),
+ cassandra.types.LocalDate.fromString('1999-12-31'),
+];
+
+const KEY_DOMAINS: ReadonlyArray<ReadonlyArray<CassandraParam>> = [
+ BIGINT_DOMAIN,
+ STRING_DOMAIN,
+ NUMBER_DOMAIN,
+ DATE_DOMAIN,
+ BUFFER_DOMAIN,
+ BOOLEAN_DOMAIN,
+ NULLABLE_STRING_DOMAIN,
+ LOCAL_DATE_DOMAIN,
+];
+
+const WILD_DOMAIN: ReadonlyArray<CassandraParam> = [
+ null,
+ 0,
+ 1,
+ 42,
+ -1,
+ 1.5,
+ 'value',
+ '',
+ 'a"b\\c\u001f',
+ true,
+ false,
+ 5n,
+ new Date('2022-02-02T02:02:02.002Z'),
+ Buffer.from('xy', 'utf8'),
+ new Set([1, 'a']),
+ new Map<unknown, unknown>([['k', 'v']]),
+ [1, 2, 3],
+ {a: 1, b: null},
+ cassandra.types.LocalDate.fromString('2020-03-04'),
+ Number.NaN,
+ Number.POSITIVE_INFINITY,
+];
+
+function hashString(value: string): number {
+ let hash = 2166136261;
+ for (let index = 0; index < value.length; index += 1) {
+ hash ^= value.charCodeAt(index);
+ hash = Math.imul(hash, 16777619);
+ }
+ return hash >>> 0;
+}
+
+function keyDomain(tableName: string, column: string): ReadonlyArray<CassandraParam> {
+ return KEY_DOMAINS[hashString(`${tableName}.${column}`) % KEY_DOMAINS.length]!;
+}
+
+function sortableDomain(tableName: string, column: string): ReadonlyArray<CassandraParam> {
+ return KEY_DOMAINS[hashString(`sort:${tableName}.${column}`) % KEY_DOMAINS.length]!;
+}
+
+function wildValue(tableName: string, column: string, index: number): CassandraParam {
+ return WILD_DOMAIN[(hashString(`${tableName}#${column}`) + index * 5) % WILD_DOMAIN.length]!;
+}
+
+function canonical(value: unknown): unknown {
+ if (value === undefined) return {__c: 'undefined'};
+ if (value === null) return null;
+ if (typeof value === 'bigint') return {__c: 'bigint', v: value.toString()};
+ if (typeof value === 'number') {
+ if (Number.isNaN(value)) return {__c: 'nan'};
+ if (!Number.isFinite(value)) return {__c: 'inf', v: value > 0 ? 1 : -1};
+ return Object.is(value, -0) ? {__c: 'negzero'} : value;
+ }
+ if (value instanceof Date) {
+ return {__c: 'date', v: Number.isNaN(value.getTime()) ? 'invalid' : value.toISOString()};
+ }
+ if (Buffer.isBuffer(value)) return {__c: 'buffer', v: value.toString('base64')};
+ if (value instanceof Set) return {__c: 'set', v: [...value.values()].map(canonical)};
+ if (value instanceof Map) {
+ return {__c: 'map', v: [...value.entries()].map(([key, entry]) => [canonical(key), canonical(entry)])};
+ }
+ if (typeof value === 'object' && value.constructor?.name === 'LocalDate') {
+ return {__c: 'localdate', v: String(value)};
+ }
+ if (Array.isArray(value)) return value.map(canonical);
+ if (typeof value === 'object') {
+ const source = value as Record<string, unknown>;
+ return {
+ __c: 'obj',
+ v: Object.keys(source)
+ .sort()
+ .map((key) => [key, canonical(source[key])]),
+ };
+ }
+ return value;
+}
+
+function fingerprint(rows: ReadonlyArray<unknown>): string {
+ return JSON.stringify(rows.map(canonical));
+}
+
+function multisetFingerprint(rows: ReadonlyArray<unknown>): string {
+ return JSON.stringify(rows.map((row) => JSON.stringify(canonical(row))).sort());
+}
+
+class CountingClient implements IPostgresClient {
+ rowsRead = 0;
+ queries = 0;
+
+ constructor(
+ private readonly inner: IPostgresClient,
+ private readonly table: string,
+ ) {}
+
+ async query<T extends Record<string, unknown>>(text: string, values: Array<unknown> = []) {
+ const result = await this.inner.query(text, values);
+ this.record(text, result.rows.length);
+ return result as unknown as Awaited<ReturnType<IPostgresClient['query']>> & {rows: Array<T>};
+ }
+
+ private record(text: string, rows: number): void {
+ this.queries += 1;
+ if (/^\s*SELECT/iu.test(text) && text.includes('row_data')) this.rowsRead += rows;
+ }
+
+ async connect(): Promise<void> {
+ await this.inner.connect();
+ }
+
+ async shutdown(): Promise<void> {}
+
+ isConnected(): boolean {
+ return this.inner.isConnected();
+ }
+
+ async transaction<T>(fn: (client: PostgresQueryable) => Promise<T>): Promise<T> {
+ return this.inner.transaction(async (db) =>
+ fn({
+ query: async <T extends Record<string, unknown>>(text: string, values?: Array<unknown>) => {
+ const result = await db.query(text, values);
+ this.record(text, result.rows.length);
+ return result as unknown as Awaited<ReturnType<PostgresQueryable['query']>> & {rows: Array<T>};
+ },
+ }),
+ );
+ }
+
+ kvTable(): string {
+ return this.table;
+ }
+
+ reset(): void {
+ this.rowsRead = 0;
+ this.queries = 0;
+ }
+}
+
+function tableSpecs(): Array<KvTableSpec<Row>> {
+ const seen = new Set<string>();
+ const specs: Array<KvTableSpec<Row>> = [];
+ for (const value of Object.values(Tables as Record<string, unknown>)) {
+ if (!value || typeof value !== 'object') continue;
+ const candidate = value as Partial<KvTableSpec<Row>> & {selectCql?: unknown};
+ if (typeof candidate.name !== 'string') continue;
+ if (typeof candidate.selectCql !== 'function') continue;
+ if (!Array.isArray(candidate.columns) || !Array.isArray(candidate.primaryKey)) continue;
+ if (!Array.isArray(candidate.partitionKey)) continue;
+ if (seen.has(candidate.name)) continue;
+ seen.add(candidate.name);
+ specs.push({
+ name: candidate.name,
+ columns: candidate.columns as ReadonlyArray<string>,
+ primaryKey: candidate.primaryKey as ReadonlyArray<string>,
+ partitionKey: candidate.partitionKey as ReadonlyArray<string>,
+ });
+ }
+ return specs.sort((left, right) => (left.name < right.name ? -1 : 1));
+}
+
+function keyDomains(spec: KvTableSpec<Row>): Array<ReadonlyArray<CassandraParam>> {
+ return (spec.primaryKey as ReadonlyArray<string>).map((column, index) => {
+ const domain = keyDomain(spec.name, column);
+ const size = index === 0 ? 4 : index === 1 ? 3 : 2;
+ return domain.slice(0, Math.min(size, domain.length));
+ });
+}
+
+function sortableColumn(spec: KvTableSpec<Row>): string {
+ const primaryKey = spec.primaryKey as ReadonlyArray<string>;
+ const nonKey = (spec.columns as ReadonlyArray<string>).filter((column) => !primaryKey.includes(column));
+ return nonKey[0] ?? primaryKey[primaryKey.length - 1]!;
+}
+
+function buildRows(spec: KvTableSpec<Row>): Array<Row> {
+ const primaryKey = spec.primaryKey as ReadonlyArray<string>;
+ const domains = keyDomains(spec);
+ let combos: Array<Array<CassandraParam>> = [[]];
+ for (const domain of domains) {
+ const next: Array<Array<CassandraParam>> = [];
+ for (const combo of combos) {
+ for (const value of domain) next.push([...combo, value]);
+ }
+ combos = next.slice(0, MAX_ROWS_PER_TABLE * 4);
+ }
+ combos = combos.slice(0, MAX_ROWS_PER_TABLE);
+ const sortable = sortableColumn(spec);
+ const sortValues = sortableDomain(spec.name, sortable);
+ return combos.map((combo, index) => {
+ const row: Row = {};
+ for (const column of spec.columns as ReadonlyArray<string>) {
+ row[column] = wildValue(spec.name, column, index);
+ }
+ row[sortable] = sortValues[index % sortValues.length]!;
+ for (let position = 0; position < primaryKey.length; position += 1) {
+ row[primaryKey[position]!] = combo[position]!;
+ }
+ return row;
+ });
+}
+
+function eq(col: string, param: string): AnyWhere {
+ return {kind: 'eq', col, param} as AnyWhere;
+}
+
+function inClause(col: string, param: string): AnyWhere {
+ return {kind: 'in', col, param} as AnyWhere;
+}
+
+function cmp(kind: 'lt' | 'lte' | 'gt' | 'gte' | 'tokenGt', col: string, param: string): AnyWhere {
+ return {kind, col, param} as AnyWhere;
+}
+
+function selectMeta(spec: KvTableSpec<Row>, where: Array<AnyWhere>, extra: Partial<AnyMeta> = {}): AnyMeta {
+ return {action: 'select', table: spec, where, columns: spec.columns, ...extra} as AnyMeta;
+}
+
+function buildShapes(spec: KvTableSpec<Row>, rows: Array<Row>): Array<Shape> {
+ const primaryKey = spec.primaryKey as ReadonlyArray<string>;
+ const partition = spec.partitionKey as ReadonlyArray<string>;
+ const columns = spec.columns as ReadonlyArray<string>;
+ const domains = keyDomains(spec);
+ const sortable = sortableColumn(spec);
+ const sample = rows[Math.floor(rows.length / 2)]!;
+ const shapes: Array<Shape> = [];
+ const pkParams = (row: Row, count: number): CassandraParams => {
+ const params: CassandraParams = {};
+ for (const column of primaryKey.slice(0, count)) params[column] = row[column] as CassandraParam;
+ return params;
+ };
+ const pkWhere = (count: number): Array<AnyWhere> => primaryKey.slice(0, count).map((column) => eq(column, column));
+
+ shapes.push({
+ name: 'pk-eq-full-hit',
+ meta: selectMeta(spec, pkWhere(primaryKey.length)),
+ params: pkParams(sample, primaryKey.length),
+ });
+ shapes.push({
+ name: 'pk-eq-full-miss',
+ meta: selectMeta(spec, pkWhere(primaryKey.length)),
+ params: Object.fromEntries(primaryKey.map((column) => [column, '__absent__' as CassandraParam])) as CassandraParams,
+ });
+
+ for (let count = 1; count < primaryKey.length; count += 1) {
+ shapes.push({name: `prefix-eq-${count}`, meta: selectMeta(spec, pkWhere(count)), params: pkParams(sample, count)});
+ for (const kind of ['lt', 'gt', 'lte', 'gte'] as const) {
+ const params = pkParams(sample, count);
+ params[primaryKey[count]!] = sample[primaryKey[count]!] as CassandraParam;
+ shapes.push({
+ name: `prefix-eq-${count}-${kind}`,
+ meta: selectMeta(spec, [...pkWhere(count), cmp(kind, primaryKey[count]!, primaryKey[count]!)]),
+ params,
+ });
+ }
+ }
+
+ for (let position = 0; position < primaryKey.length; position += 1) {
+ const pinColumn = primaryKey[position]!;
+ const params = pkParams(sample, position);
+ params[pinColumn] = domains[position]!.slice(0, 2) as unknown as CassandraParam;
+ shapes.push({
+ name: `in-at-${position}`,
+ meta: selectMeta(spec, [...pkWhere(position), inClause(pinColumn, pinColumn)]),
+ params,
+ });
+ const dupParams = pkParams(sample, position);
+ const first = domains[position]![0]!;
+ dupParams[pinColumn] = [first, first, domains[position]![1] ?? first] as unknown as CassandraParam;
+ shapes.push({
+ name: `in-dup-at-${position}`,
+ meta: selectMeta(spec, [...pkWhere(position), inClause(pinColumn, pinColumn)]),
+ params: dupParams,
+ });
+ const emptyParams = pkParams(sample, position);
+ emptyParams[pinColumn] = [] as unknown as CassandraParam;
+ shapes.push({
+ name: `in-empty-at-${position}`,
+ meta: selectMeta(spec, [...pkWhere(position), inClause(pinColumn, pinColumn)]),
+ params: emptyParams,
+ });
+ }
+
+ shapes.push({
+ name: 'in-set-at-0',
+ meta: selectMeta(spec, [inClause(primaryKey[0]!, primaryKey[0]!)]),
+ params: {[primaryKey[0]!]: new Set(domains[0]!.slice(0, 2))},
+ });
+
+ const inAllParams: CassandraParams = {};
+ const inAllWhere = primaryKey.map((column, index) => {
+ inAllParams[column] = domains[index]!.slice(0, 2) as unknown as CassandraParam;
+ return inClause(column, column);
+ });
+ shapes.push({name: 'in-all-pk', meta: selectMeta(spec, inAllWhere), params: inAllParams});
+
+ if (partition.length > 0) {
+ const partParams: CassandraParams = {};
+ const partWhere = partition.map((column) => {
+ partParams[column] = sample[column] as CassandraParam;
+ return eq(column, column);
+ });
+ shapes.push({name: 'partition-eq', meta: selectMeta(spec, partWhere), params: partParams});
+ shapes.push({
+ name: 'partition-in-0',
+ meta: selectMeta(spec, [inClause(partition[0]!, partition[0]!), ...partWhere.slice(1)]),
+ params: {
+ ...partParams,
+ [partition[0]!]: [sample[partition[0]!], rows[0]![partition[0]!]] as unknown as CassandraParam,
+ },
+ });
+ }
+
+ const nonKey = columns.filter((column) => !primaryKey.includes(column));
+ if (nonKey.length > 0) {
+ shapes.push({
+ name: 'nonkey-eq',
+ meta: selectMeta(spec, [eq(nonKey[0]!, nonKey[0]!)]),
+ params: {[nonKey[0]!]: sample[nonKey[0]!] as CassandraParam},
+ });
+ shapes.push({
+ name: 'prefix-eq-1-nonkey-eq',
+ meta: selectMeta(spec, [...pkWhere(1), eq(nonKey[0]!, nonKey[0]!)]),
+ params: {...pkParams(sample, 1), [nonKey[0]!]: sample[nonKey[0]!] as CassandraParam},
+ });
+ }
+
+ shapes.push({
+ name: 'tokenGt-0',
+ meta: selectMeta(spec, [cmp('tokenGt', primaryKey[0]!, primaryKey[0]!)]),
+ params: {[primaryKey[0]!]: sample[primaryKey[0]!] as CassandraParam},
+ });
+
+ if (primaryKey.length >= 2) {
+ shapes.push({
+ name: 'tupleGt',
+ meta: selectMeta(spec, [
+ {kind: 'tupleGt', cols: [primaryKey[0]!, primaryKey[1]!], params: [primaryKey[0]!, primaryKey[1]!]} as AnyWhere,
+ ]),
+ params: {
+ [primaryKey[0]!]: sample[primaryKey[0]!] as CassandraParam,
+ [primaryKey[1]!]: sample[primaryKey[1]!] as CassandraParam,
+ },
+ });
+ }
+
+ shapes.push({name: 'no-where', meta: selectMeta(spec, []), params: {}});
+
+ for (const direction of ['ASC', 'DESC'] as const) {
+ shapes.push({
+ name: `order-${direction}`,
+ meta: selectMeta(spec, [], {orderBy: {col: sortable, direction}}),
+ params: {},
+ });
+ shapes.push({
+ name: `prefix-eq-1-order-${direction}`,
+ meta: selectMeta(spec, pkWhere(1), {orderBy: {col: sortable, direction}}),
+ params: pkParams(sample, 1),
+ });
+ }
+
+ for (const limit of [1, 3, 1000]) {
+ shapes.push({name: `limit-${limit}`, meta: selectMeta(spec, [], {limit}), params: {}});
+ shapes.push({
+ name: `prefix-eq-1-limit-${limit}`,
+ meta: selectMeta(spec, pkWhere(1), {limit}),
+ params: pkParams(sample, 1),
+ });
+ }
+
+ shapes.push({
+ name: 'prefix-eq-1-order-limit',
+ meta: selectMeta(spec, pkWhere(1), {orderBy: {col: sortable, direction: 'DESC'}, limit: 2}),
+ params: pkParams(sample, 1),
+ });
+
+ const subset = [...new Set([columns[0]!, columns[columns.length - 1]!, sortable])];
+ shapes.push({
+ name: 'columns-subset',
+ meta: selectMeta(spec, pkWhere(1), {columns: subset as ReadonlyArray<string>}),
+ params: pkParams(sample, 1),
+ });
+ shapes.push({
+ name: 'columns-subset-order',
+ meta: selectMeta(spec, [], {columns: subset as ReadonlyArray<string>, orderBy: {col: sortable}}),
+ params: {},
+ });
+
+ return shapes;
+}
+
+const COUNT_SHAPE_NAMES = [
+ 'pk-eq-full-hit',
+ 'pk-eq-full-miss',
+ 'prefix-eq-1',
+ 'in-at-0',
+ 'in-dup-at-0',
+ 'in-empty-at-0',
+ 'in-set-at-0',
+ 'in-all-pk',
+ 'partition-eq',
+ 'partition-in-0',
+ 'nonkey-eq',
+ 'prefix-eq-1-nonkey-eq',
+ 'no-where',
+ 'tokenGt-0',
+ 'tupleGt',
+];
+
+const DELETE_SHAPE_NAMES = [
+ 'pk-eq-full-hit',
+ 'prefix-eq-1',
+ 'in-at-0',
+ 'in-dup-at-0',
+ 'in-empty-at-0',
+ 'in-all-pk',
+ 'partition-eq',
+ 'nonkey-eq',
+ 'tokenGt-0',
+ 'no-where',
+];
+
+function derivedShapes(
+ spec: KvTableSpec<Row>,
+ shapes: Array<Shape>,
+ action: 'count' | 'delete',
+ names: ReadonlyArray<string>,
+): Array<Shape> {
+ return shapes
+ .filter((shape) => names.includes(shape.name))
+ .map((shape) => ({
+ name: `${action}:${shape.name}`,
+ meta: {action, table: spec, where: shape.meta.where} as AnyMeta,
+ params: shape.params,
+ }));
+}
+
+const AdversarialSingle = defineTable<{k: unknown; a: unknown; b: unknown}, 'k'>({
+ name: 'kvdiff_adversarial_single',
+ columns: ['k', 'a', 'b'],
+ primaryKey: ['k'],
+});
+
+const AdversarialWide = defineTable<
+ {k0: unknown; k1: unknown; k2: unknown; k3: unknown; k4: unknown; k5: unknown; v: unknown},
+ 'k0' | 'k1' | 'k2' | 'k3' | 'k4' | 'k5'
+>({
+ name: 'kvdiff_adversarial_wide',
+ columns: ['k0', 'k1', 'k2', 'k3', 'k4', 'k5', 'v'],
+ primaryKey: ['k0', 'k1', 'k2', 'k3', 'k4', 'k5'],
+});
+
+const AdversarialSplit = defineTable<{p: unknown; c: unknown; v: unknown}, 'p' | 'c', 'p'>({
+ name: 'kvdiff_adversarial_split',
+ columns: ['p', 'c', 'v'],
+ primaryKey: ['p', 'c'],
+ partitionKey: ['p'],
+});
+
+const AdversarialTrailingPartition = defineTable<{p: unknown; c: unknown; v: unknown}, 'p' | 'c', 'c'>({
+ name: 'kvdiff_adversarial_trailing',
+ columns: ['p', 'c', 'v'],
+ primaryKey: ['p', 'c'],
+ partitionKey: ['c'],
+});
+
+const ADVERSARIAL_KEY_VALUES: ReadonlyArray<CassandraParam> = [
+ null,
+ '',
+ 'a',
+ 'a"b',
+ 'a\\b',
+ 'a\u0001b',
+ 'a\u001fb',
+ 'a b',
+ 'a\u007fb',
+ '\u{1f600}',
+ '\ufffd',
+ '2020-01-01',
+ 1n,
+ 2n,
+ 9n,
+ 10n,
+ 11n,
+ 99999999999999999999n,
+ 0,
+ 1,
+ 9,
+ 10,
+ true,
+ false,
+ new Date('2020-01-01T00:00:00.000Z'),
+ Buffer.from([0x1f]),
+ Buffer.from([0x20]),
+ cassandra.types.LocalDate.fromString('2020-01-01'),
+];
+
+const AdversarialRows: Record<string, Array<Row>> = {};
+
+function adversarialCorpus(): void {
+ AdversarialRows[AdversarialSingle.name] = ADVERSARIAL_KEY_VALUES.map((value, index) => ({
+ k: value,
+ a: WILD_DOMAIN[index % WILD_DOMAIN.length]!,
+ b: index,
+ }));
+ const splitRows: Array<Row> = [];
+ for (const p of ADVERSARIAL_KEY_VALUES.slice(0, 10)) {
+ for (const c of ADVERSARIAL_KEY_VALUES.slice(0, 6)) splitRows.push({p, c, v: `${String(p)}|${String(c)}`});
+ }
+ AdversarialRows[AdversarialSplit.name] = splitRows;
+ AdversarialRows[AdversarialTrailingPartition.name] = splitRows.map((row) => ({...row}));
+ const wideRows: Array<Row> = [];
+ for (const k0 of ADVERSARIAL_KEY_VALUES.slice(0, 4)) {
+ for (const k1 of ADVERSARIAL_KEY_VALUES.slice(6, 9)) {
+ for (const k2 of ADVERSARIAL_KEY_VALUES.slice(12, 14)) {
+ wideRows.push({k0, k1, k2, k3: null, k4: '', k5: 1n, v: 'x'});
+ }
+ }
+ }
+ AdversarialRows[AdversarialWide.name] = wideRows;
+}
+
+function adversarialShapes(): Array<Shape> {
+ const single = AdversarialSingle as unknown as KvTableSpec<Row>;
+ const wide = AdversarialWide as unknown as KvTableSpec<Row>;
+ const split = AdversarialSplit as unknown as KvTableSpec<Row>;
+ const trailing = AdversarialTrailingPartition as unknown as KvTableSpec<Row>;
+ const shapes: Array<Shape> = [];
+ for (const value of ADVERSARIAL_KEY_VALUES) {
+ shapes.push({name: `single-eq-${String(value)}`, meta: selectMeta(single, [eq('k', 'k')]), params: {k: value}});
+ shapes.push({name: `split-eq-p-${String(value)}`, meta: selectMeta(split, [eq('p', 'p')]), params: {p: value}});
+ shapes.push({
+ name: `split-eq-p-eq-c-${String(value)}`,
+ meta: selectMeta(split, [eq('p', 'p'), eq('c', 'c')]),
+ params: {p: value, c: value},
+ });
+ shapes.push({
+ name: `trailing-eq-c-${String(value)}`,
+ meta: selectMeta(trailing, [eq('c', 'c')]),
+ params: {c: value},
+ });
+ shapes.push({
+ name: `trailing-eq-p-${String(value)}`,
+ meta: selectMeta(trailing, [eq('p', 'p')]),
+ params: {p: value},
+ });
+ }
+ shapes.push({
+ name: 'single-eq-localdate-vs-string',
+ meta: selectMeta(single, [eq('k', 'k')]),
+ params: {k: cassandra.types.LocalDate.fromString('2020-01-01')},
+ });
+ shapes.push({
+ name: 'single-eq-string-vs-localdate',
+ meta: selectMeta(single, [eq('k', 'k')]),
+ params: {k: '2020-01-01'},
+ });
+ shapes.push({name: 'single-eq-nan', meta: selectMeta(single, [eq('k', 'k')]), params: {k: Number.NaN}});
+ shapes.push({
+ name: 'single-eq-infinity',
+ meta: selectMeta(single, [eq('k', 'k')]),
+ params: {k: Number.POSITIVE_INFINITY},
+ });
+ shapes.push({name: 'single-eq-null', meta: selectMeta(single, [eq('k', 'k')]), params: {k: null}});
+ shapes.push({name: 'single-eq-object', meta: selectMeta(single, [eq('k', 'k')]), params: {k: {a: 1}}});
+ shapes.push({name: 'single-eq-set', meta: selectMeta(single, [eq('k', 'k')]), params: {k: new Set([1, 'a'])}});
+ shapes.push({
+ name: 'single-in-all',
+ meta: selectMeta(single, [inClause('k', 'k')]),
+ params: {k: [...ADVERSARIAL_KEY_VALUES] as unknown as CassandraParam},
+ });
+ shapes.push({
+ name: 'single-in-empty',
+ meta: selectMeta(single, [inClause('k', 'k')]),
+ params: {k: [] as unknown as CassandraParam},
+ });
+ shapes.push({name: 'single-in-empty-set', meta: selectMeta(single, [inClause('k', 'k')]), params: {k: new Set()}});
+ shapes.push({
+ name: 'single-in-huge',
+ meta: selectMeta(single, [inClause('k', 'k')]),
+ params: {k: [...Array.from({length: 4000}, (_unused, index) => BigInt(index)), 1n] as unknown as CassandraParam},
+ });
+ shapes.push({
+ name: 'split-in-p-cartesian',
+ meta: selectMeta(split, [inClause('p', 'p'), inClause('c', 'c')]),
+ params: {
+ p: ADVERSARIAL_KEY_VALUES.slice(0, 10) as unknown as CassandraParam,
+ c: ADVERSARIAL_KEY_VALUES.slice(0, 6) as unknown as CassandraParam,
+ },
+ });
+ shapes.push({
+ name: 'split-in-p-over-cap',
+ meta: selectMeta(split, [inClause('p', 'p'), inClause('c', 'c')]),
+ params: {
+ p: Array.from({length: 40}, (_unused, index) => BigInt(index)) as unknown as CassandraParam,
+ c: Array.from({length: 40}, (_unused, index) => BigInt(index)) as unknown as CassandraParam,
+ },
+ });
+ shapes.push({
+ name: 'split-eq-p-in-c-empty',
+ meta: selectMeta(split, [eq('p', 'p'), inClause('c', 'c')]),
+ params: {p: ADVERSARIAL_KEY_VALUES[2]!, c: [] as unknown as CassandraParam},
+ });
+ shapes.push({
+ name: 'wide-prefix-eq-3',
+ meta: selectMeta(wide, [eq('k0', 'k0'), eq('k1', 'k1'), eq('k2', 'k2')]),
+ params: {k0: ADVERSARIAL_KEY_VALUES[0]!, k1: ADVERSARIAL_KEY_VALUES[6]!, k2: ADVERSARIAL_KEY_VALUES[12]!},
+ });
+ shapes.push({
+ name: 'wide-prefix-eq-1',
+ meta: selectMeta(wide, [eq('k0', 'k0')]),
+ params: {k0: ADVERSARIAL_KEY_VALUES[0]!},
+ });
+ shapes.push({
+ name: 'wide-gap-k0-k2',
+ meta: selectMeta(wide, [eq('k0', 'k0'), eq('k2', 'k2')]),
+ params: {k0: ADVERSARIAL_KEY_VALUES[0]!, k2: ADVERSARIAL_KEY_VALUES[12]!},
+ });
+ shapes.push({
+ name: 'wide-full-pk',
+ meta: selectMeta(wide, [
+ eq('k0', 'k0'),
+ eq('k1', 'k1'),
+ eq('k2', 'k2'),
+ eq('k3', 'k3'),
+ eq('k4', 'k4'),
+ eq('k5', 'k5'),
+ ]),
+ params: {
+ k0: ADVERSARIAL_KEY_VALUES[0]!,
+ k1: ADVERSARIAL_KEY_VALUES[6]!,
+ k2: ADVERSARIAL_KEY_VALUES[12]!,
+ k3: null,
+ k4: '',
+ k5: 1n,
+ },
+ });
+ shapes.push({
+ name: 'trailing-partition-in',
+ meta: selectMeta(trailing, [inClause('c', 'c')]),
+ params: {c: ADVERSARIAL_KEY_VALUES.slice(0, 4) as unknown as CassandraParam},
+ });
+ shapes.push({name: 'single-missing-param', meta: selectMeta(single, [eq('k', 'k')]), params: {}});
+ shapes.push({name: 'single-in-missing-param', meta: selectMeta(single, [inClause('k', 'k')]), params: {}});
+ shapes.push({name: 'single-no-where', meta: selectMeta(single, []), params: {}});
+ shapes.push({name: 'split-no-where', meta: selectMeta(split, []), params: {}});
+ return shapes;
+}
+
+function keysetEligible(shape: Shape): boolean {
+ if (shape.meta.action !== 'select') return false;
+ if (shape.meta.orderBy !== undefined || shape.meta.limit !== undefined) return false;
+ const plan = buildCandidatePlan(shape.meta, shape.params);
+ return (
+ plan.exact &&
+ (plan.candidates.kind === 'none' || plan.candidates.kind === 'range' || plan.candidates.kind === 'scan')
+ );
+}
+
+const HEAP_ORDERINGS: ReadonlyArray<string> = [
+ 'row_key COLLATE "C" ASC',
+ 'row_key COLLATE "C" DESC',
+ "md5(row_key || 'a')",
+ "md5(row_key || 'b')",
+ "md5(row_key || 'c')",
+ "md5(row_key || 'd')",
+ "md5(row_key || 'e')",
+ "md5(row_key || 'f')",
+];
+
+const suite = dockerAvailable ? describe : describe.skip;
+
+suite('PostgresKvQueryExecutor differential', () => {
+ let raw: IPostgresClient;
+ let readLegacyClient: CountingClient;
+ let readNextClient: CountingClient;
+ let mutLegacyClient: CountingClient;
+ let mutNextClient: CountingClient;
+ let icuLegacyClient: CountingClient;
+ let icuNextClient: CountingClient;
+ let readLegacy: LegacyPostgresKvQueryExecutor;
+ let readNext: PostgresKvQueryExecutor;
+ let mutLegacy: LegacyPostgresKvQueryExecutor;
+ let mutNext: PostgresKvQueryExecutor;
+ let icuLegacy: LegacyPostgresKvQueryExecutor;
+ let icuNext: PostgresKvQueryExecutor;
+ const specs = tableSpecs();
+ const adversarialSpecs = [
+ AdversarialSingle as unknown as KvTableSpec<Row>,
+ AdversarialSplit as unknown as KvTableSpec<Row>,
+ AdversarialTrailingPartition as unknown as KvTableSpec<Row>,
+ AdversarialWide as unknown as KvTableSpec<Row>,
+ ];
+ const corpus = new Map<string, Array<Row>>();
+ const shapeCache = new Map<string, Array<Shape>>();
+ const unstableShapes: Array<string> = [];
+ const reproducibleOrders: Array<string> = [];
+ const traffic = {legacy: 0, next: 0, shapes: 0, improved: 0};
+ const keysetOrderDeltas: Array<string> = [];
+ const legacyThrowDeltas: Array<string> = [];
+
+ function shapesFor(spec: KvTableSpec<Row>): Array<Shape> {
+ const cached = shapeCache.get(spec.name);
+ if (cached) return cached;
+ const built = buildShapes(spec, corpus.get(spec.name)!);
+ shapeCache.set(spec.name, built);
+ return built;
+ }
+
+ async function freePort(): Promise<number> {
+ return new Promise((resolve, reject) => {
+ const server = createServer();
+ server.on('error', reject);
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address();
+ if (typeof address === 'string' || address === null) {
+ reject(new Error('Could not allocate a port'));
+ return;
+ }
+ const port = address.port;
+ server.close(() => resolve(port));
+ });
+ });
+ }
+
+ async function sleep(ms: number): Promise<void> {
+ await new Promise((resolve) => setTimeout(resolve, ms));
+ }
+
+ async function seedTable(spec: KvTableSpec<Row>, rows: ReadonlyArray<Row>, db: PostgresQueryable): Promise<void> {
+ const meta = {action: 'upsert', table: spec} as AnyMeta;
+ for (const row of rows) {
+ await mutLegacy.executeQuery({cql: `__seed__:${spec.name}`, params: row as CassandraParams, kvMeta: meta}, db);
+ }
+ }
+
+ async function restoreOne(kv: string, tableName: string, ordering: string): Promise<void> {
+ await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [tableName]);
+ await raw.query(`INSERT INTO ${kv} SELECT * FROM ${SEED_TABLE} WHERE table_name = $1 ORDER BY ${ordering}`, [
+ tableName,
+ ]);
+ }
+
+ async function restoreBoth(tableName: string): Promise<void> {
+ await restoreOne(LEGACY_TABLE, tableName, HEAP_ORDERINGS[0]!);
+ await restoreOne(NEXT_TABLE, tableName, HEAP_ORDERINGS[0]!);
+ }
+
+ async function tableState(kv: string, tableName: string): Promise<string> {
+ const result = await raw.query<{
+ row_key: string;
+ partition_key: string;
+ row_data: string;
+ expires_at: Date | null;
+ }>(
+ `SELECT row_key, partition_key, row_data::text AS row_data, expires_at FROM ${kv} WHERE table_name = $1 ORDER BY row_key COLLATE "C"`,
+ [tableName],
+ );
+ return JSON.stringify(
+ result.rows.map((row) => [row.row_key, row.partition_key, row.row_data, row.expires_at?.toISOString() ?? null]),
+ );
+ }
+
+ async function run(executor: AnyExecutor, client: CountingClient, shape: Shape): Promise<Outcome> {
+ client.reset();
+ try {
+ const rows = await executor.executeQuery({
+ cql: `__diff__:${shape.name}`,
+ params: shape.params,
+ kvMeta: shape.meta,
+ });
+ return {fingerprint: fingerprint(rows), rowsRead: client.rowsRead, threw: null};
+ } catch (error) {
+ return {fingerprint: '', rowsRead: client.rowsRead, threw: (error as Error).message};
+ }
+ }
+
+ async function legacyUnderHeapOrder(
+ kv: string,
+ executor: LegacyPostgresKvQueryExecutor,
+ tableName: string,
+ shape: Shape,
+ ordering: string,
+ ): Promise<string> {
+ await restoreOne(kv, tableName, ordering);
+ let observed = '';
+ await raw.transaction(async (db) => {
+ await db.query('SET LOCAL enable_indexscan = off');
+ await db.query('SET LOCAL enable_indexonlyscan = off');
+ await db.query('SET LOCAL enable_bitmapscan = off');
+ const rows = await executor.executeQuery(
+ {cql: `__probe__:${shape.name}`, params: shape.params, kvMeta: shape.meta},
+ db,
+ );
+ observed = fingerprint(rows);
+ });
+ return observed;
+ }
+
+ async function legacyIsOrderUnstable(
+ kv: string,
+ executor: LegacyPostgresKvQueryExecutor,
+ tableName: string,
+ shape: Shape,
+ candidate: string,
+ ): Promise<boolean> {
+ const observed = new Set<string>();
+ for (const ordering of HEAP_ORDERINGS) {
+ observed.add(await legacyUnderHeapOrder(kv, executor, tableName, shape, ordering));
+ }
+ await restoreOne(kv, tableName, HEAP_ORDERINGS[0]!);
+ if (observed.size === 1) return false;
+ if (observed.has(candidate)) reproducibleOrders.push(`${tableName} ${shape.name}`);
+ return true;
+ }
+
+ async function compareRead(
+ mismatches: Array<Mismatch>,
+ kv: string,
+ executor: LegacyPostgresKvQueryExecutor,
+ tableName: string,
+ shape: Shape,
+ left: Outcome,
+ right: Outcome,
+ ): Promise<void> {
+ if (left.threw !== null && right.threw === null) {
+ legacyThrowDeltas.push(`${tableName} ${shape.name}: legacy threw "${left.threw}" -> next ${right.fingerprint}`);
+ return;
+ }
+ if (left.threw === null && right.threw !== null) {
+ mismatches.push({
+ kind: 'throw',
+ table: tableName,
+ shape: shape.name,
+ legacy: left.fingerprint,
+ next: right.threw,
+ });
+ return;
+ }
+ if (left.threw !== null && right.threw !== null) {
+ if (left.threw !== right.threw) {
+ mismatches.push({kind: 'throw', table: tableName, shape: shape.name, legacy: left.threw, next: right.threw});
+ }
+ return;
+ }
+ traffic.legacy += left.rowsRead;
+ traffic.next += right.rowsRead;
+ traffic.shapes += 1;
+ if (right.rowsRead < left.rowsRead) traffic.improved += 1;
+ if (left.fingerprint === right.fingerprint) {
+ if (right.rowsRead > left.rowsRead) {
+ mismatches.push({
+ kind: 'rowsread',
+ table: tableName,
+ shape: shape.name,
+ legacy: String(left.rowsRead),
+ next: String(right.rowsRead),
+ });
+ }
+ return;
+ }
+ const leftRows = JSON.parse(left.fingerprint) as Array<unknown>;
+ const rightRows = JSON.parse(right.fingerprint) as Array<unknown>;
+ const sameMultiset = multisetFingerprint(leftRows) === multisetFingerprint(rightRows);
+ if (!sameMultiset && leftRows.length !== rightRows.length) {
+ mismatches.push({
+ kind: 'rowset',
+ table: tableName,
+ shape: shape.name,
+ legacy: left.fingerprint.slice(0, 400),
+ next: right.fingerprint.slice(0, 400),
+ });
+ return;
+ }
+ if (!(await legacyIsOrderUnstable(kv, executor, tableName, shape, right.fingerprint))) {
+ mismatches.push({
+ kind: sameMultiset ? 'order' : 'rowset',
+ table: tableName,
+ shape: shape.name,
+ legacy: left.fingerprint.slice(0, 400),
+ next: right.fingerprint.slice(0, 400),
+ });
+ return;
+ }
+ unstableShapes.push(`${tableName} ${shape.name}`);
+ }
+
+ function report(mismatches: Array<Mismatch>): string {
+ const byKind = new Map<string, number>();
+ for (const mismatch of mismatches) byKind.set(mismatch.kind, (byKind.get(mismatch.kind) ?? 0) + 1);
+ const summary = [...byKind.entries()].map(([kind, count]) => `${kind}=${count}`).join(' ');
+ const detail = mismatches
+ .slice(0, 25)
+ .map(
+ (mismatch) =>
+ `[${mismatch.kind}] ${mismatch.table} ${mismatch.shape}\n legacy: ${mismatch.legacy}\n next: ${mismatch.next}`,
+ )
+ .join('\n');
+ return `${mismatches.length} mismatches (${summary})\n${detail}`;
+ }
+
+ beforeAll(async () => {
+ const port = await freePort();
+ execFileSync(
+ 'docker',
+ [
+ 'run',
+ '-d',
+ '--name',
+ CONTAINER,
+ '-e',
+ 'POSTGRES_USER=fluxer',
+ '-e',
+ 'POSTGRES_PASSWORD=fluxer',
+ '-e',
+ 'POSTGRES_DB=fluxer',
+ '-p',
+ `127.0.0.1:${port}:5432`,
+ POSTGRES_IMAGE,
+ '-c',
+ 'fsync=off',
+ '-c',
+ 'synchronous_commit=off',
+ '-c',
+ 'full_page_writes=off',
+ ],
+ {stdio: 'ignore'},
+ );
+ let ready = false;
+ for (let attempt = 0; attempt < 180 && !ready; attempt += 1) {
+ await sleep(500);
+ const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+ stdio: 'ignore',
+ });
+ if (probe.status !== 0) continue;
+ try {
+ await initPostgres({url: `postgres://fluxer:[email protected]:${port}/fluxer`, maxConnections: 8});
+ await getDefaultPostgresClient().query('SELECT 1');
+ ready = true;
+ } catch {
+ await shutdownPostgres().catch(() => {});
+ }
+ }
+ if (!ready) throw new Error('Throwaway postgres did not become ready');
+ raw = getDefaultPostgresClient();
+ readLegacyClient = new CountingClient(raw, NEXT_TABLE);
+ readNextClient = new CountingClient(raw, NEXT_TABLE);
+ mutLegacyClient = new CountingClient(raw, LEGACY_TABLE);
+ mutNextClient = new CountingClient(raw, NEXT_TABLE);
+ icuLegacyClient = new CountingClient(raw, ICU_TABLE);
+ icuNextClient = new CountingClient(raw, ICU_TABLE);
+ await ensurePostgresKvSchema(mutLegacyClient);
+ await ensurePostgresKvSchema(mutNextClient);
+ await raw.query(`CREATE COLLATION ${ICU_COLLATION} (provider = icu, locale = 'en-US')`);
+ await ensurePostgresKvSchema(icuNextClient);
+ for (const column of ['row_key', 'partition_key']) {
+ await raw.query(`ALTER TABLE ${ICU_TABLE} ALTER COLUMN ${column} TYPE text COLLATE ${ICU_COLLATION}`);
+ }
+ readLegacy = new LegacyPostgresKvQueryExecutor(readLegacyClient);
+ readNext = new PostgresKvQueryExecutor(readNextClient);
+ mutLegacy = new LegacyPostgresKvQueryExecutor(mutLegacyClient);
+ mutNext = new PostgresKvQueryExecutor(mutNextClient);
+ icuLegacy = new LegacyPostgresKvQueryExecutor(icuLegacyClient);
+ icuNext = new PostgresKvQueryExecutor(icuNextClient);
+
+ adversarialCorpus();
+ for (const spec of specs) corpus.set(spec.name, buildRows(spec));
+ for (const spec of adversarialSpecs) corpus.set(spec.name, AdversarialRows[spec.name]!);
+
+ await raw.transaction(async (db) => {
+ for (const spec of [...specs, ...adversarialSpecs]) await seedTable(spec, corpus.get(spec.name)!, db);
+ });
+ await raw.query(
+ `UPDATE ${LEGACY_TABLE} SET expires_at = now() - interval '1 hour' WHERE abs(hashtext(row_key)) % 7 = 3`,
+ );
+ await raw.query(
+ `UPDATE ${LEGACY_TABLE} SET expires_at = now() + interval '1 hour' WHERE abs(hashtext(row_key)) % 7 = 5`,
+ );
+ await raw.query(`CREATE TABLE ${SEED_TABLE} AS TABLE ${LEGACY_TABLE}`);
+ await raw.query(`CREATE INDEX ${SEED_TABLE}_table_idx ON ${SEED_TABLE} (table_name)`);
+ await raw.query(`TRUNCATE ${LEGACY_TABLE}`);
+ for (const kv of [LEGACY_TABLE, NEXT_TABLE, ICU_TABLE]) {
+ await raw.query(`INSERT INTO ${kv} SELECT * FROM ${SEED_TABLE} ORDER BY table_name, row_key COLLATE "C"`);
+ }
+ }, 900_000);
+
+ afterAll(async () => {
+ await shutdownPostgres().catch(() => {});
+ spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+ });
+
+ it('seeds an identical corpus into both kv tables', async () => {
+ expect(specs.length).toBeGreaterThan(150);
+ const seeded = await raw.query<{count: string}>(`SELECT count(*) AS count FROM ${SEED_TABLE}`);
+ expect(Number(seeded.rows[0]!.count)).toBeGreaterThan(specs.length * 4);
+ const expired = await raw.query<{count: string}>(
+ `SELECT count(*) AS count FROM ${SEED_TABLE} WHERE expires_at IS NOT NULL AND expires_at <= now()`,
+ );
+ expect(Number(expired.rows[0]!.count)).toBeGreaterThan(0);
+ const future = await raw.query<{count: string}>(
+ `SELECT count(*) AS count FROM ${SEED_TABLE} WHERE expires_at > now()`,
+ );
+ expect(Number(future.rows[0]!.count)).toBeGreaterThan(0);
+ for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
+ const drift = await raw.query<{count: string}>(
+ `SELECT count(*) AS count FROM (
+ SELECT table_name, partition_key, row_key, row_data, expires_at FROM ${kv}
+ EXCEPT
+ SELECT table_name, partition_key, row_key, row_data, expires_at FROM ${SEED_TABLE}
+ ) AS diff`,
+ );
+ expect(drift.rows[0]!.count).toBe('0');
+ }
+ });
+
+ it('writes byte-identical stored rows through both executors', async () => {
+ const mismatches: Array<Mismatch> = [];
+ for (const spec of [...specs, ...adversarialSpecs]) {
+ const writeName = `w_${spec.name}`;
+ const writeMeta = {action: 'upsert', table: {...spec, name: writeName}} as AnyMeta;
+ for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
+ await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [writeName]);
+ }
+ for (const row of corpus.get(spec.name)!.slice(0, 4)) {
+ await mutLegacy.executeQuery({cql: '__w__', params: row as CassandraParams, kvMeta: writeMeta});
+ await mutNext.executeQuery({cql: '__w__', params: row as CassandraParams, kvMeta: writeMeta});
+ }
+ const left = await tableState(LEGACY_TABLE, writeName);
+ const right = await tableState(NEXT_TABLE, writeName);
+ if (left !== right) {
+ mismatches.push({
+ kind: 'state',
+ table: spec.name,
+ shape: 'upsert',
+ legacy: left.slice(0, 400),
+ next: right.slice(0, 400),
+ });
+ }
+ for (const kv of [LEGACY_TABLE, NEXT_TABLE]) {
+ await raw.query(`DELETE FROM ${kv} WHERE table_name = $1`, [writeName]);
+ }
+ }
+ expect(mismatches.length, report(mismatches)).toBe(0);
+ }, 900_000);
+
+ it('returns identical select results for every generated shape', async () => {
+ const mismatches: Array<Mismatch> = [];
+ let shapeCount = 0;
+ for (const spec of specs) {
+ for (const shape of shapesFor(spec)) {
+ shapeCount += 1;
+ const left = await run(readLegacy, readLegacyClient, shape);
+ const right = await run(readNext, readNextClient, shape);
+ await compareRead(mismatches, NEXT_TABLE, readLegacy, spec.name, shape, left, right);
+ }
+ }
+ expect(shapeCount).toBeGreaterThan(3000);
+ console.log(`select sweep: tables=${specs.length} shapes=${shapeCount}`);
+ expect(mismatches.length, report(mismatches)).toBe(0);
+ }, 1_800_000);
+
+ it('returns identical count results for every generated shape', async () => {
+ const mismatches: Array<Mismatch> = [];
+ for (const spec of specs) {
+ for (const shape of derivedShapes(spec, shapesFor(spec), 'count', COUNT_SHAPE_NAMES)) {
+ const left = await run(readLegacy, readLegacyClient, shape);
+ const right = await run(readNext, readNextClient, shape);
+ await compareRead(mismatches, NEXT_TABLE, readLegacy, spec.name, shape, left, right);
+ }
+ }
+ expect(mismatches.length, report(mismatches)).toBe(0);
+ }, 1_800_000);
+
+ it('leaves identical stored state after every generated delete shape', async () => {
+ const mismatches: Array<Mismatch> = [];
+ for (const spec of specs) {
+ for (const shape of derivedShapes(spec, shapesFor(spec), 'delete', DELETE_SHAPE_NAMES)) {
+ await restoreBoth(spec.name);
+ const left = await run(mutLegacy, mutLegacyClient, shape);
+ const right = await run(mutNext, mutNextClient, shape);
+ if ((left.threw === null) !== (right.threw === null)) {
+ if (left.threw !== null && right.threw === null) {
+ legacyThrowDeltas.push(`${spec.name} ${shape.name}: legacy threw "${left.threw}"`);
+ } else {
+ mismatches.push({
+ kind: 'throw',
+ table: spec.name,
+ shape: shape.name,
+ legacy: left.threw ?? 'ok',
+ next: right.threw ?? 'ok',
+ });
+ continue;
+ }
+ }
+ const leftState = await tableState(LEGACY_TABLE, spec.name);
+ const rightState = await tableState(NEXT_TABLE, spec.name);
+ if (leftState !== rightState) {
+ mismatches.push({
+ kind: 'state',
+ table: spec.name,
+ shape: shape.name,
+ legacy: leftState.slice(0, 400),
+ next: rightState.slice(0, 400),
+ });
+ }
+ traffic.legacy += left.rowsRead;
+ traffic.next += right.rowsRead;
+ traffic.shapes += 1;
+ if (right.rowsRead < left.rowsRead) traffic.improved += 1;
+ if (right.rowsRead > left.rowsRead) {
+ mismatches.push({
+ kind: 'rowsread',
+ table: spec.name,
+ shape: shape.name,
+ legacy: String(left.rowsRead),
+ next: String(right.rowsRead),
+ });
+ }
+ }
+ await restoreBoth(spec.name);
+ }
+ expect(mismatches.length, report(mismatches)).toBe(0);
+ }, 1_800_000);
+
+ it('pages over identical row multisets with identical page boundaries', async () => {
+ const mismatches: Array<Mismatch> = [];
+ for (const spec of specs) {
+ const primaryKey = spec.primaryKey as ReadonlyArray<string>;
+ const rows = corpus.get(spec.name)!;
+ const sample = rows[Math.floor(rows.length / 2)]!;
+ const sortable = sortableColumn(spec);
+ const pagedShapes: Array<Shape> = [
+ {name: 'paged-no-where', meta: selectMeta(spec, []), params: {}},
+ {
+ name: 'paged-prefix-eq-1',
+ meta: selectMeta(spec, [eq(primaryKey[0]!, primaryKey[0]!)]),
+ params: {[primaryKey[0]!]: sample[primaryKey[0]!] as CassandraParam},
+ },
+ {
+ name: 'paged-nonkey-eq',
+ meta: selectMeta(spec, [eq(sortable, sortable)]),
+ params: {[sortable]: sample[sortable] as CassandraParam},
+ },
+ ];
+ for (const shape of pagedShapes) {
+ for (const pageSize of [2, 5]) {
+ const collect = async (executor: AnyExecutor) => {
+ const pages: Array<Array<unknown>> = [];
+ let pageState: string | null = null;
+ try {
+ for (let guard = 0; guard < 500; guard += 1) {
+ const page: {rows: Array<unknown>; pageState: string | null} = await executor.executePagedQuery(
+ {cql: `__page__:${shape.name}`, params: shape.params, kvMeta: shape.meta},
+ {pageSize, pageState},
+ );
+ pages.push(page.rows);
+ pageState = page.pageState;
+ if (pageState === null) break;
+ }
+ } catch (error) {
+ return {pages, error: (error as Error).message};
+ }
+ return {pages, error: null as string | null};
+ };
+ const left = await collect(readLegacy);
+ const right = await collect(readNext);
+ const label = `${shape.name}/size${pageSize}`;
+ if (left.error !== right.error) {
+ mismatches.push({
+ kind: 'throw',
+ table: spec.name,
+ shape: label,
+ legacy: left.error ?? 'ok',
+ next: right.error ?? 'ok',
+ });
+ continue;
+ }
+ if (left.error !== null) continue;
+ const leftSizes = left.pages.map((page) => page.length).join(',');
+ const rightSizes = right.pages.map((page) => page.length).join(',');
+ if (leftSizes !== rightSizes) {
+ mismatches.push({
+ kind: 'rowset',
+ table: spec.name,
+ shape: label,
+ legacy: `sizes=${leftSizes}`,
+ next: `sizes=${rightSizes}`,
+ });
+ continue;
+ }
+ const leftAll = left.pages.flat();
+ const rightAll = right.pages.flat();
+ if (multisetFingerprint(leftAll) !== multisetFingerprint(rightAll)) {
+ mismatches.push({
+ kind: 'rowset',
+ table: spec.name,
+ shape: label,
+ legacy: fingerprint(leftAll).slice(0, 400),
+ next: fingerprint(rightAll).slice(0, 400),
+ });
+ continue;
+ }
+ if (fingerprint(leftAll) === fingerprint(rightAll)) continue;
+ if (keysetEligible(shape)) {
+ keysetOrderDeltas.push(`${spec.name} ${label}`);
+ continue;
+ }
+ if (await legacyIsOrderUnstable(NEXT_TABLE, readLegacy, spec.name, shape, fingerprint(rightAll))) {
+ unstableShapes.push(`${spec.name} ${label}`);
+ continue;
+ }
+ mismatches.push({
+ kind: 'order',
+ table: spec.name,
+ shape: label,
+ legacy: fingerprint(leftAll).slice(0, 400),
+ next: fingerprint(rightAll).slice(0, 400),
+ });
+ }
+ }
+ }
+ expect(mismatches.length, report(mismatches)).toBe(0);
+ }, 1_800_000);
+
+ it('returns identical results on a database whose row_key collation is linguistic', async () => {
+ const mismatches: Array<Mismatch> = [];
+ const ordering = await raw.query<{icu: boolean; c: boolean}>(
+ `SELECT (E'"a"\u001f"b"' COLLATE ${ICU_COLLATION}) < (E'"a"\u0020' COLLATE ${ICU_COLLATION}) AS icu,
+ (E'"a"\u001f"b"' COLLATE "C") < (E'"a"\u0020' COLLATE "C") AS c`,
+ );
+ expect(ordering.rows[0]!.c).toBe(true);
+ expect(ordering.rows[0]!.icu).toBe(false);
+ for (const spec of [...specs, ...adversarialSpecs]) {
+ const shapes = spec.name.startsWith('kvdiff_') ? [] : shapesFor(spec);
+ for (const shape of shapes) {
+ const left = await run(icuLegacy, icuLegacyClient, shape);
+ const right = await run(icuNext, icuNextClient, shape);
+ await compareRead(mismatches, ICU_TABLE, icuLegacy, spec.name, shape, left, right);
+ }
+ for (const shape of derivedShapes(
+ spec,
+ spec.name.startsWith('kvdiff_') ? [] : shapesFor(spec),
+ 'count',
+ COUNT_SHAPE_NAMES,
+ )) {
+ const left = await run(icuLegacy, icuLegacyClient, shape);
+ const right = await run(icuNext, icuNextClient, shape);
+ await compareRead(mismatches, ICU_TABLE, icuLegacy, spec.name, shape, left, right);
+ }
+ }
+ expect(mismatches.length, report(mismatches)).toBe(0);
+ }, 1_800_000);
+
+ it('accepts a legacy offset page token and reproduces the legacy page exactly', async () => {
+ const mismatches: Array<Mismatch> = [];
+ for (const spec of specs.slice(0, 40)) {
+ const shape: Shape = {name: 'legacy-token', meta: selectMeta(spec, []), params: {}};
+ const query = {cql: '__legacytoken__', params: shape.params, kvMeta: shape.meta};
+ const first = await readLegacy.executePagedQuery(query, {pageSize: 3});
+ if (first.pageState === null) continue;
+ const legacySecond = await readLegacy.executePagedQuery(query, {pageSize: 3, pageState: first.pageState});
+ const nextSecond = await readNext.executePagedQuery(query, {pageSize: 3, pageState: first.pageState});
+ if (fingerprint(legacySecond.rows) !== fingerprint(nextSecond.rows)) {
+ mismatches.push({
+ kind: 'rowset',
+ table: spec.name,
+ shape: 'legacy-token',
+ legacy: fingerprint(legacySecond.rows).slice(0, 400),
+ next: fingerprint(nextSecond.rows).slice(0, 400),
+ });
+ }
+ }
+ expect(mismatches.length, report(mismatches)).toBe(0);
+ }, 900_000);
+
+ it('agrees on adversarial key encodings, cross-type params and degenerate IN lists', async () => {
+ const mismatches: Array<Mismatch> = [];
+ const shapes = adversarialShapes();
+ const derived = shapes.map((shape) => ({
+ name: `count:${shape.name}`,
+ meta: {action: 'count', table: shape.meta.table, where: shape.meta.where} as AnyMeta,
+ params: shape.params,
+ }));
+ for (const shape of [...shapes, ...derived]) {
+ const left = await run(readLegacy, readLegacyClient, shape);
+ const right = await run(readNext, readNextClient, shape);
+ await compareRead(mismatches, NEXT_TABLE, readLegacy, shape.meta.table.name, shape, left, right);
+ }
+ for (const shape of shapes) {
+ const deleteShape: Shape = {
+ name: `delete:${shape.name}`,
+ meta: {action: 'delete', table: shape.meta.table, where: shape.meta.where} as AnyMeta,
+ params: shape.params,
+ };
+ await restoreBoth(shape.meta.table.name);
+ const left = await run(mutLegacy, mutLegacyClient, deleteShape);
+ const right = await run(mutNext, mutNextClient, deleteShape);
+ if ((left.threw === null) !== (right.threw === null)) {
+ if (left.threw !== null && right.threw === null) {
+ legacyThrowDeltas.push(`${shape.meta.table.name} ${deleteShape.name}: legacy threw "${left.threw}"`);
+ } else {
+ mismatches.push({
+ kind: 'throw',
+ table: shape.meta.table.name,
+ shape: deleteShape.name,
+ legacy: left.threw ?? 'ok',
+ next: right.threw ?? 'ok',
+ });
+ continue;
+ }
+ }
+ const leftState = await tableState(LEGACY_TABLE, shape.meta.table.name);
+ const rightState = await tableState(NEXT_TABLE, shape.meta.table.name);
+ if (leftState !== rightState) {
+ mismatches.push({
+ kind: 'state',
+ table: shape.meta.table.name,
+ shape: deleteShape.name,
+ legacy: leftState.slice(0, 400),
+ next: rightState.slice(0, 400),
+ });
+ }
+ }
+ for (const spec of adversarialSpecs) await restoreBoth(spec.name);
+
+ const singleName = AdversarialSingle.name;
+ const groundTruth = await raw.query<{count: string}>(
+ `SELECT count(*) AS count FROM ${NEXT_TABLE} WHERE table_name = $1 AND (expires_at IS NULL OR expires_at > now())`,
+ [singleName],
+ );
+ const pushedCount = await readNext.executeQuery<{count: number}>({
+ cql: '__groundtruth__',
+ params: {},
+ kvMeta: {action: 'count', table: AdversarialSingle as unknown as KvTableSpec<Row>, where: []} as AnyMeta,
+ });
+ expect(pushedCount[0]!.count).toBe(Number(groundTruth.rows[0]!.count));
+
+ expect(mismatches.length, report(mismatches)).toBe(0);
+ }, 1_800_000);
+
+ it('records every accepted delta and nothing else', () => {
+ for (const entry of unstableShapes) {
+ expect(entry, `unstable shape without a table name: ${entry}`).toContain(' ');
+ }
+ for (const entry of keysetOrderDeltas) {
+ expect(entry, `keyset delta without a table name: ${entry}`).toContain(' ');
+ }
+ for (const entry of legacyThrowDeltas) {
+ expect(entry, `unexpected legacy throw delta: ${entry}`).toContain('legacy threw');
+ }
+ expect(traffic.shapes).toBeGreaterThan(3000);
+ expect(traffic.next).toBeLessThan(traffic.legacy);
+ console.log(
+ `row payload traffic: shapes=${traffic.shapes} legacyRows=${traffic.legacy} nextRows=${traffic.next} reduction=${(100 - (traffic.next / traffic.legacy) * 100).toFixed(1)}% shapesImproved=${traffic.improved}`,
+ );
+ console.log(
+ `accepted deltas: order-unstable=${unstableShapes.length} (legacy-reproducible=${reproducibleOrders.length}) keyset-order=${keysetOrderDeltas.length} legacy-throw=${legacyThrowDeltas.length}`,
+ );
+ console.log(`order-unstable sample: ${unstableShapes.slice(0, 5).join(' | ')}`);
+ console.log(`keyset-order sample: ${keysetOrderDeltas.slice(0, 5).join(' | ')}`);
+ console.log(`legacy-throw: ${[...new Set(legacyThrowDeltas.map((entry) => entry.split(': ')[1]))].join(' | ')}`);
+ });
+});
diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
index 8ae05ca42..43912e71c 100644
--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
@@ -2,11 +2,14 @@
import {type IPostgresClient, type PostgresQueryable, quoteIdentifier} from '@pkgs/postgres/src/Client';
import cassandra from 'cassandra-driver';
+import {Logger} from '../Logger';
import {getKvMeta, getTableMetadata} from './CassandraMetaRegistry';
import type {CassandraParams, ColumnName, KvQueryMeta, PreparedQuery, WhereExpr} from './CassandraTypes';
type Row = Record<string, unknown>;
type EqWhereExpr = Extract<WhereExpr<Row>, {kind: 'eq'}>;
+type InWhereExpr = Extract<WhereExpr<Row>, {kind: 'in'}>;
+type PinnedWhereExpr = EqWhereExpr | InWhereExpr;
interface StoredRow {
row_key: string;
@@ -17,7 +20,40 @@ interface PageState {
offset: number;
}
+export type CandidatePlan =
+ | {kind: 'none'}
+ | {kind: 'rowKeys'; rowKeys: Array<string>}
+ | {kind: 'range'; lowerBound: string; upperBound: string}
+ | {kind: 'ranges'; lowerBounds: Array<string>; upperBounds: Array<string>}
+ | {kind: 'partitionKeys'; partitionKeys: Array<string>}
+ | {kind: 'scan'};
+
+export interface QueryPlan {
+ candidates: CandidatePlan;
+ exact: boolean;
+}
+
+interface QueryShape {
+ leadingClauses: ReadonlyArray<PinnedWhereExpr>;
+ partitionClauses: ReadonlyArray<PinnedWhereExpr> | null;
+ inClauses: ReadonlyArray<InWhereExpr>;
+ whereColumns: ReadonlyArray<string>;
+ requiredColumns: ReadonlyArray<string> | null;
+ clauseCount: number;
+ summary: string;
+}
+
+export interface PlanFragments {
+ predicate: string;
+ params: Array<unknown>;
+}
+
const VALUE_SEPARATOR = '\u001f';
+const KEY_RANGE_UPPER = ' ';
+const MAX_ROW_KEY_COMBINATIONS = 32_768;
+const MAX_PREFIX_RANGES = 256;
+const FULL_SCAN_LOG_INTERVAL_MS = 60_000;
+const FULL_SCAN_LOG_KEY_LIMIT = 1024;
const ENCODED_TYPE_KEY = '__fluxer_type';
const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56;
const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';
@@ -93,11 +129,24 @@ function decodeRow(value: unknown): Row {
return decoded;
}
+function decodeRowColumns(value: unknown, columns: ReadonlyArray<string>): Row {
+ if (!isPlainObject(value)) {
+ throw new Error('Postgres KV row payload is not an object');
+ }
+ const decoded: Row = {};
+ for (const column of columns) {
+ if (column in value) {
+ decoded[column] = decodeValue(value[column]);
+ }
+ }
+ return decoded;
+}
+
function valueKey(value: unknown): string {
return JSON.stringify(encodeValue(value));
}
-function keyFromColumns(columns: ReadonlyArray<string>, row: Row): string {
+export function keyFromColumns(columns: ReadonlyArray<string>, row: Row): string {
return columns.map((column) => valueKey(row[column])).join(VALUE_SEPARATOR);
}
@@ -137,10 +186,6 @@ function rowKeyFromParams(meta: KvQueryMeta, params: CassandraParams): string {
return rowKey(meta, paramsRow(params, (meta.pkColumns ?? meta.table.primaryKey) as ReadonlyArray<string>));
}
-function partitionKeyFromParams(meta: KvQueryMeta, params: CassandraParams): string {
- return partitionKey(meta, paramsRow(params, meta.table.partitionKey as ReadonlyArray<string>));
-}
-
function compareValues(left: unknown, right: unknown): number {
if (typeof left === 'bigint' || typeof right === 'bigint') {
const l = typeof left === 'bigint' ? left : BigInt(left as number | string);
@@ -160,8 +205,8 @@ function valuesEqual(left: unknown, right: unknown): boolean {
if (left == null && right == null) return true;
if (left instanceof Date && right instanceof Date) return left.getTime() === right.getTime();
if (Buffer.isBuffer(left) && Buffer.isBuffer(right)) return left.equals(right);
- if (left?.constructor?.name === 'LocalDate' || right?.constructor?.name === 'LocalDate') {
- return left?.toString() === right?.toString();
+ if (left?.constructor?.name === 'LocalDate' && right?.constructor?.name === 'LocalDate') {
+ return left.toString() === right.toString();
}
return left === right;
}
@@ -170,7 +215,11 @@ function getParam(params: CassandraParams, param: string): unknown {
return params[param];
}
-function matchesWhere(row: Row, where: ReadonlyArray<WhereExpr<Row>> | undefined, params: CassandraParams): boolean {
+export function matchesWhere(
+ row: Row,
+ where: ReadonlyArray<WhereExpr<Row>> | undefined,
+ params: CassandraParams,
+): boolean {
for (const clause of where ?? []) {
switch (clause.kind) {
case 'eq':
@@ -238,39 +287,269 @@ function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
});
}
-function equalityParam(where: ReadonlyArray<WhereExpr<Row>> | undefined, column: string): string | null {
- const clause = (where ?? []).find((entry) => entry.kind === 'eq' && entry.col === column);
- return clause && clause.kind === 'eq' ? clause.param : null;
+function whereClauses(meta: KvQueryMeta): ReadonlyArray<WhereExpr<Row>> {
+ return (meta.where ?? []) as ReadonlyArray<WhereExpr<Row>>;
}
-function inParam(where: ReadonlyArray<WhereExpr<Row>> | undefined, column: string): string | null {
- const clause = (where ?? []).find((entry) => entry.kind === 'in' && entry.col === column);
- return clause && clause.kind === 'in' ? clause.param : null;
+function pinnedClause(where: ReadonlyArray<WhereExpr<Row>>, column: string): PinnedWhereExpr | null {
+ for (const clause of where) {
+ if ((clause.kind === 'eq' || clause.kind === 'in') && clause.col === column) return clause;
+ }
+ return null;
}
-function fullRowKeysFromWhere(meta: KvQueryMeta, params: CassandraParams): Array<string> | null {
- const pk = meta.table.primaryKey as ReadonlyArray<string>;
- const eqParams = pk.map((column) => equalityParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, column));
- if (eqParams.every((param) => param !== null)) {
- const row: Row = {};
- for (let i = 0; i < pk.length; i += 1) row[pk[i]!] = params[eqParams[i]!];
- return [rowKey(meta, row)];
+function clauseColumns(clause: WhereExpr<Row>): ReadonlyArray<string> {
+ return clause.kind === 'tupleGt' ? (clause.cols as ReadonlyArray<string>) : [clause.col as string];
+}
+
+function describeClause(clause: WhereExpr<Row>): string {
+ return `${clauseColumns(clause).join('+')} ${clause.kind}`;
+}
+
+function requiredColumnsFor(meta: KvQueryMeta, whereColumns: ReadonlyArray<string>): ReadonlyArray<string> | null {
+ if (!meta.columns) return null;
+ const required = new Set<string>(meta.columns as ReadonlyArray<string>);
+ for (const column of whereColumns) required.add(column);
+ if (meta.orderBy) required.add(meta.orderBy.col as string);
+ for (const column of meta.table.primaryKey as ReadonlyArray<string>) required.add(column);
+ for (const column of meta.table.columns as ReadonlyArray<string>) {
+ if (!required.has(column)) return [...required];
}
- if (pk.length === 1) {
- const param = inParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, pk[0]!);
- if (param) {
- const values = params[param] as ReadonlyArray<unknown> | Set<unknown>;
- const haystack = values instanceof Set ? [...values] : values;
- return haystack.map((value) => rowKey(meta, {[pk[0]!]: value}));
+ return null;
+}
+
+const QUERY_SHAPES = new WeakMap<KvQueryMeta, QueryShape>();
+
+function queryShape(meta: KvQueryMeta): QueryShape {
+ const cached = QUERY_SHAPES.get(meta);
+ if (cached) return cached;
+ const where = whereClauses(meta);
+ const leadingClauses: Array<PinnedWhereExpr> = [];
+ for (const column of meta.table.primaryKey as ReadonlyArray<string>) {
+ const clause = pinnedClause(where, column);
+ if (!clause) break;
+ leadingClauses.push(clause);
+ }
+ const partitionColumns = meta.table.partitionKey as ReadonlyArray<string>;
+ const partitionClauses: Array<PinnedWhereExpr> = [];
+ for (const column of partitionColumns) {
+ const clause = pinnedClause(where, column);
+ if (!clause) {
+ partitionClauses.length = 0;
+ break;
}
- }
+ partitionClauses.push(clause);
+ }
+ const whereColumns = [...new Set(where.flatMap(clauseColumns))];
+ const shape: QueryShape = {
+ leadingClauses,
+ partitionClauses:
+ partitionColumns.length > 0 && partitionClauses.length === partitionColumns.length ? partitionClauses : null,
+ inClauses: where.filter((clause): clause is InWhereExpr => clause.kind === 'in'),
+ whereColumns,
+ requiredColumns: requiredColumnsFor(meta, whereColumns),
+ clauseCount: where.length,
+ summary: where.map(describeClause).join(', '),
+ };
+ QUERY_SHAPES.set(meta, shape);
+ return shape;
+}
+
+function clauseValues(clause: PinnedWhereExpr, params: CassandraParams): Array<unknown> | null {
+ if (clause.kind === 'eq') return [getParam(params, clause.param)];
+ const values = getParam(params, clause.param);
+ if (values === null || values === undefined) return [];
+ if (values instanceof Set) return [...values];
+ if (Array.isArray(values)) return [...values];
return null;
}
-function hasFullPartition(meta: KvQueryMeta): boolean {
- return meta.table.partitionKey.every((column) =>
- equalityParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, column),
- );
+function isKeyComparableValue(value: unknown): boolean {
+ if (value === null || value === undefined) return true;
+ if (typeof value === 'string' || typeof value === 'boolean' || typeof value === 'bigint') return true;
+ if (typeof value === 'number') return Number.isFinite(value);
+ if (Buffer.isBuffer(value)) return true;
+ if (value instanceof Date) return !Number.isNaN(value.getTime());
+ return false;
+}
+
+function keySegments(values: ReadonlyArray<unknown>): Array<string> | null {
+ const segments: Array<string> = [];
+ for (const value of values) {
+ let segment: string;
+ try {
+ segment = valueKey(value);
+ } catch {
+ return null;
+ }
+ if (typeof segment !== 'string') return null;
+ segments.push(segment);
+ }
+ return segments;
+}
+
+function combinationCount(segmentLists: ReadonlyArray<ReadonlyArray<string>>): number {
+ let total = 1;
+ for (const segments of segmentLists) total *= segments.length;
+ return total;
+}
+
+function keyPrefixes(segmentLists: ReadonlyArray<ReadonlyArray<string>>): Array<string> {
+ let prefixes: Array<string> | null = null;
+ for (const segments of segmentLists) {
+ const next: Array<string> = [];
+ for (const segment of segments) {
+ if (prefixes === null) {
+ next.push(segment);
+ continue;
+ }
+ for (const prefix of prefixes) next.push(`${prefix}${VALUE_SEPARATOR}${segment}`);
+ }
+ prefixes = next;
+ }
+ return [...new Set(prefixes ?? [])];
+}
+
+function keyRangeLowerBound(prefix: string): string {
+ return `${prefix}${VALUE_SEPARATOR}`;
+}
+
+function keyRangeUpperBound(prefix: string): string {
+ return `${prefix}${KEY_RANGE_UPPER}`;
+}
+
+interface PinnedColumn {
+ values: Array<unknown>;
+ segments: Array<string>;
+}
+
+function pinnedColumns(clauses: ReadonlyArray<PinnedWhereExpr>, params: CassandraParams): Array<PinnedColumn> {
+ const pinned: Array<PinnedColumn> = [];
+ for (const clause of clauses) {
+ const values = clauseValues(clause, params);
+ if (values === null || values.length === 0) break;
+ const segments = keySegments(values);
+ if (segments === null) break;
+ pinned.push({values, segments});
+ }
+ return pinned;
+}
+
+function planIsExact(meta: KvQueryMeta, shape: QueryShape, pinned: ReadonlyArray<PinnedColumn>): boolean {
+ if (pinned.length !== shape.clauseCount) return false;
+ if (meta.limit !== undefined || meta.orderBy !== undefined) return false;
+ for (const column of pinned) {
+ for (const value of column.values) {
+ if (!isKeyComparableValue(value)) return false;
+ }
+ }
+ return true;
+}
+
+export function buildCandidatePlan(meta: KvQueryMeta, params: CassandraParams): QueryPlan {
+ const shape = queryShape(meta);
+ for (const clause of shape.inClauses) {
+ const values = clauseValues(clause, params);
+ if (values !== null && values.length === 0) {
+ return {candidates: {kind: 'none'}, exact: true};
+ }
+ }
+ const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
+ const pinned = pinnedColumns(shape.leadingClauses, params);
+ let leading = pinned.length;
+ while (leading > 0) {
+ const segmentLists = pinned.slice(0, leading).map((column) => column.segments);
+ const cap = leading === primaryKey.length ? MAX_ROW_KEY_COMBINATIONS : MAX_PREFIX_RANGES;
+ if (combinationCount(segmentLists) <= cap) break;
+ leading -= 1;
+ }
+ if (leading > 0) {
+ const columns = pinned.slice(0, leading);
+ const prefixes = keyPrefixes(columns.map((column) => column.segments));
+ const exact = planIsExact(meta, shape, columns);
+ if (leading === primaryKey.length) {
+ return {candidates: {kind: 'rowKeys', rowKeys: prefixes}, exact};
+ }
+ if (prefixes.length === 1) {
+ const prefix = prefixes[0]!;
+ return {
+ candidates: {kind: 'range', lowerBound: keyRangeLowerBound(prefix), upperBound: keyRangeUpperBound(prefix)},
+ exact,
+ };
+ }
+ return {
+ candidates: {
+ kind: 'ranges',
+ lowerBounds: prefixes.map(keyRangeLowerBound),
+ upperBounds: prefixes.map(keyRangeUpperBound),
+ },
+ exact,
+ };
+ }
+ if (shape.partitionClauses) {
+ const partition = pinnedColumns(shape.partitionClauses, params);
+ if (
+ partition.length === shape.partitionClauses.length &&
+ combinationCount(partition.map((column) => column.segments)) <= MAX_ROW_KEY_COMBINATIONS
+ ) {
+ return {
+ candidates: {kind: 'partitionKeys', partitionKeys: keyPrefixes(partition.map((column) => column.segments))},
+ exact: planIsExact(meta, shape, partition),
+ };
+ }
+ }
+ return {candidates: {kind: 'scan'}, exact: planIsExact(meta, shape, [])};
+}
+
+export function planFragments(plan: CandidatePlan): PlanFragments {
+ switch (plan.kind) {
+ case 'rowKeys':
+ return {predicate: ' AND kv.row_key = ANY($2::text[])', params: [plan.rowKeys]};
+ case 'range':
+ return {
+ predicate: ' AND kv.row_key COLLATE "C" >= $2 AND kv.row_key COLLATE "C" < $3',
+ params: [plan.lowerBound, plan.upperBound],
+ };
+ case 'ranges': {
+ const params: Array<unknown> = [];
+ const arms = plan.lowerBounds.map((lowerBound, index) => {
+ params.push(lowerBound, plan.upperBounds[index]);
+ return `(kv.row_key COLLATE "C" >= $${params.length} AND kv.row_key COLLATE "C" < $${params.length + 1})`;
+ });
+ return {predicate: ` AND (${arms.join(' OR ')})`, params};
+ }
+ case 'partitionKeys':
+ return plan.partitionKeys.length === 1
+ ? {predicate: ' AND kv.partition_key = $2', params: [plan.partitionKeys[0]]}
+ : {predicate: ' AND kv.partition_key = ANY($2::text[])', params: [plan.partitionKeys]};
+ default:
+ return {predicate: '', params: []};
+ }
+}
+
+function logWarn(details: Record<string, unknown>, message: string): void {
+ try {
+ Logger.warn(details, message);
+ } catch {}
+}
+
+function logError(details: Record<string, unknown>, message: string): void {
+ try {
+ Logger.error(details, message);
+ } catch {}
+}
+
+const fullScanLoggedAt = new Map<string, number>();
+
+function logFullScan(meta: KvQueryMeta): void {
+ const shape = queryShape(meta);
+ const key = `${meta.table.name}|${meta.action}|${shape.summary}`;
+ const now = Date.now();
+ const last = fullScanLoggedAt.get(key);
+ if (last !== undefined && now - last < FULL_SCAN_LOG_INTERVAL_MS) return;
+ if (fullScanLoggedAt.size >= FULL_SCAN_LOG_KEY_LIMIT) fullScanLoggedAt.clear();
+ fullScanLoggedAt.set(key, now);
+ logWarn({table: meta.table.name, action: meta.action, where: shape.summary || 'none'}, 'Postgres KV full table scan');
}
function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
@@ -355,6 +634,24 @@ function parseEqWhere(whereSql: string, cql: string): ReadonlyArray<EqWhereExpr>
});
}
+async function repairInvalidPostgresKvIndexes(db: PostgresQueryable, kvTable: string): Promise<void> {
+ const invalid = await db.query<{index_name: string; index_def: string}>(
+ `SELECT cls.relname AS index_name, pg_get_indexdef(idx.indexrelid) AS index_def
+FROM pg_index idx
+JOIN pg_class cls ON cls.oid = idx.indexrelid
+WHERE idx.indrelid = to_regclass($1)
+ AND NOT idx.indisvalid
+ AND NOT idx.indisprimary
+ AND NOT idx.indisunique`,
+ [kvTable],
+ );
+ for (const row of invalid.rows) {
+ logError({table: kvTable, index: row.index_name}, 'Postgres KV index is invalid, rebuilding it');
+ await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(row.index_name)}`);
+ await db.query(row.index_def);
+ }
+}
+
export async function ensurePostgresKvSchema(client: IPostgresClient): Promise<void> {
const kvTable = client.kvTable();
const table = quoteIdentifier(kvTable);
@@ -387,12 +684,22 @@ CREATE TABLE IF NOT EXISTS ${table} (
await db.query(
`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,
);
- await db.query(`
+ await repairInvalidPostgresKvIndexes(db, kvTable);
+ const pending = await db.query(`
+SELECT 1
+FROM ${table}
+WHERE table_name = 'messages'
+ AND partition_key = row_key
+ AND split_part(row_key, chr(31), 3) <> ''
+LIMIT 1`);
+ if (pending.rows.length > 0) {
+ await db.query(`
UPDATE ${table}
SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)
WHERE table_name = 'messages'
AND partition_key = row_key
AND split_part(row_key, chr(31), 3) <> ''`);
+ }
await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`);
});
}
@@ -434,9 +741,9 @@ export class PostgresKvQueryExecutor {
const meta = this.meta(query);
switch (meta.action) {
case 'select':
- return (await this.select(meta, query.params, db)) as Array<T>;
+ return (await this.select(meta, query.params, buildCandidatePlan(meta, query.params), db)) as Array<T>;
case 'count':
- return [{count: (await this.select(meta, query.params, db)).length}] as Array<T>;
+ return (await this.count(meta, query.params, db)) as Array<T>;
case 'upsert':
return (await this.upsert(meta, query.params, db)) as Array<T>;
case 'insert':
@@ -495,42 +802,51 @@ export class PostgresKvQueryExecutor {
return meta;
}
- private async candidates(
- meta: KvQueryMeta,
- params: CassandraParams,
- db: PostgresQueryable,
- ): Promise<Array<StoredRow>> {
- const rowKeys = fullRowKeysFromWhere(meta, params);
- if (rowKeys) {
- const result = await db.query<StoredRow>(
- `SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND row_key = ANY($2::text[]) AND (expires_at IS NULL OR expires_at > now())`,
- [meta.table.name, rowKeys],
- );
- return result.rows;
- }
- if (hasFullPartition(meta)) {
- const result = await db.query<StoredRow>(
- `SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND partition_key = $2 AND (expires_at IS NULL OR expires_at > now())`,
- [meta.table.name, partitionKeyFromParams(meta, params)],
- );
- return result.rows;
- }
+ private async candidates(meta: KvQueryMeta, plan: QueryPlan, db: PostgresQueryable): Promise<Array<StoredRow>> {
+ if (plan.candidates.kind === 'none') return [];
+ if (plan.candidates.kind === 'scan') logFullScan(meta);
+ const fragments = planFragments(plan.candidates);
const result = await db.query<StoredRow>(
- `SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND (expires_at IS NULL OR expires_at > now())`,
- [meta.table.name],
+ `SELECT kv.row_key, kv.row_data FROM ${this.table} kv WHERE kv.table_name = $1${fragments.predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
+ [meta.table.name, ...fragments.params],
);
return result.rows;
}
- private async select(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
- let rows = (await this.candidates(meta, params, db))
- .map((stored) => decodeRow(stored.row_data))
+ private matchingRows(meta: KvQueryMeta, stored: ReadonlyArray<StoredRow>, params: CassandraParams): Array<Row> {
+ const required = queryShape(meta).requiredColumns;
+ return stored
+ .map((entry) => (required ? decodeRowColumns(entry.row_data, required) : decodeRow(entry.row_data)))
.filter((row) => matchesWhere(row, meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params));
+ }
+
+ private async select(
+ meta: KvQueryMeta,
+ params: CassandraParams,
+ plan: QueryPlan,
+ db: PostgresQueryable,
+ ): Promise<Array<Row>> {
+ let rows = this.matchingRows(meta, await this.candidates(meta, plan, db), params);
rows = sortRows(meta, rows);
if (typeof meta.limit === 'number') rows = rows.slice(0, meta.limit);
return rows.map((row) => projectRow(row, meta.columns as ReadonlyArray<string> | undefined));
}
+ private async count(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
+ const plan = buildCandidatePlan(meta, params);
+ if (!plan.exact) {
+ return [{count: (await this.select(meta, params, plan, db)).length}];
+ }
+ if (plan.candidates.kind === 'none') return [{count: 0}];
+ if (plan.candidates.kind === 'scan') logFullScan(meta);
+ const fragments = planFragments(plan.candidates);
+ const result = await db.query<{count: string}>(
+ `SELECT count(*) AS count FROM ${this.table} kv WHERE kv.table_name = $1${fragments.predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
+ [meta.table.name, ...fragments.params],
+ );
+ return [{count: Number(result.rows[0]?.count ?? 0)}];
+ }
+
private async upsert(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
const incoming = rowFromParams(meta, params);
const key = rowKey(meta, incoming);
@@ -587,10 +903,26 @@ DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = EXCLUDED.row_da
}
private async delete(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<void> {
- const rows = await this.candidates(meta, params, db);
+ const plan = buildCandidatePlan(meta, params);
+ if (plan.candidates.kind === 'none') return;
+ if (plan.exact) {
+ if (plan.candidates.kind === 'scan') logFullScan(meta);
+ const fragments = planFragments(plan.candidates);
+ await db.query(
+ `DELETE FROM ${this.table} kv WHERE kv.table_name = $1${fragments.predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
+ [meta.table.name, ...fragments.params],
+ );
+ return;
+ }
+ const whereColumns = queryShape(meta).whereColumns;
+ const rows = await this.candidates(meta, plan, db);
const matchingKeys = rows
.filter((stored) =>
- matchesWhere(decodeRow(stored.row_data), meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params),
+ matchesWhere(
+ decodeRowColumns(stored.row_data, whereColumns),
+ meta.where as ReadonlyArray<WhereExpr<Row>> | undefined,
+ params,
+ ),
)
.map((stored) => stored.row_key);
if (matchingKeys.length === 0) return;
diff --git a/fluxer_api/src/api/database/PostgresKvQueryPlan.test.ts b/fluxer_api/src/api/database/PostgresKvQueryPlan.test.ts
new file mode 100644
index 000000000..9c16ae3bf
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvQueryPlan.test.ts
@@ -0,0 +1,441 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import cassandra from 'cassandra-driver';
+import {describe, expect, it} from 'vitest';
+import {defineTable} from './CassandraTableDsl';
+import type {CassandraParams, PreparedQuery} from './CassandraTypes';
+import {buildCandidatePlan, type CandidatePlan, keyFromColumns, matchesWhere} from './PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+
+const SEPARATOR = '\u001f';
+
+const TRICKY_STRINGS = [
+ '',
+ 'a',
+ 'a"b',
+ 'a\\b',
+ 'a\u0001b',
+ 'a\u001fb',
+ 'a b',
+ 'a\u007fb',
+ '\u{1f600}',
+ '\ufffd',
+ 'a\u0000b',
+ 'b',
+];
+
+function byteCompare(left: string, right: string): number {
+ return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8'));
+}
+
+function withinRange(rowKey: string, lowerBound: string, upperBound: string): boolean {
+ return byteCompare(rowKey, lowerBound) >= 0 && byteCompare(rowKey, upperBound) < 0;
+}
+
+function planAccepts(plan: CandidatePlan, rowKey: string, partitionKey: string): boolean {
+ switch (plan.kind) {
+ case 'none':
+ return false;
+ case 'rowKeys':
+ return plan.rowKeys.includes(rowKey);
+ case 'range':
+ return withinRange(rowKey, plan.lowerBound, plan.upperBound);
+ case 'ranges':
+ return plan.lowerBounds.some((lower, index) => withinRange(rowKey, lower, plan.upperBounds[index]!));
+ case 'partitionKeys':
+ return plan.partitionKeys.includes(partitionKey);
+ case 'scan':
+ return true;
+ }
+}
+
+function matchingRangeCount(plan: CandidatePlan, rowKey: string): number {
+ if (plan.kind !== 'ranges') return 0;
+ return plan.lowerBounds.filter((lower, index) => withinRange(rowKey, lower, plan.upperBounds[index]!)).length;
+}
+
+function expectKeyShape(key: string, separators: number): void {
+ let seen = 0;
+ for (const byte of Buffer.from(key, 'utf8')) {
+ if (byte === 0x1f) {
+ seen += 1;
+ continue;
+ }
+ expect(byte).toBeGreaterThanOrEqual(0x20);
+ }
+ expect(seen).toBe(separators);
+}
+
+function planOf(query: PreparedQuery) {
+ const meta = query.kvMeta;
+ if (!meta) {
+ throw new Error('Query is missing kv metadata');
+ }
+ return {meta, plan: buildCandidatePlan(meta, query.params)};
+}
+
+function checkQuery(query: PreparedQuery, rows: Array<Row>): CandidatePlan {
+ const {meta, plan} = planOf(query);
+ const primaryKey = meta.table.primaryKey as ReadonlyArray<string>;
+ const partitionKey = meta.table.partitionKey as ReadonlyArray<string>;
+ const accepted: Array<Row> = [];
+ const matched: Array<Row> = [];
+ for (const row of rows) {
+ const rowKey = keyFromColumns(primaryKey, row);
+ expectKeyShape(rowKey, primaryKey.length - 1);
+ expect(matchingRangeCount(plan.candidates, rowKey)).toBeLessThanOrEqual(1);
+ if (planAccepts(plan.candidates, rowKey, keyFromColumns(partitionKey, row))) accepted.push(row);
+ if (matchesWhere(row, meta.where, query.params)) matched.push(row);
+ }
+ for (const row of matched) {
+ expect(accepted).toContain(row);
+ }
+ if (plan.exact) {
+ expect(accepted).toEqual(matched);
+ }
+ return plan.candidates;
+}
+
+interface SingleRow {
+ a: string;
+ payload: string;
+}
+
+const SingleKey = defineTable<SingleRow, 'a'>({
+ name: 'kv_plan_single',
+ columns: ['a', 'payload'],
+ primaryKey: ['a'],
+});
+
+const singleRows: Array<Row> = TRICKY_STRINGS.map((a) => ({a, payload: 'p'}));
+
+interface PairRow {
+ a: string;
+ b: bigint;
+ payload: string;
+}
+
+const PairKey = defineTable<PairRow, 'a' | 'b'>({
+ name: 'kv_plan_pair',
+ columns: ['a', 'b', 'payload'],
+ primaryKey: ['a', 'b'],
+});
+
+const pairRows: Array<Row> = TRICKY_STRINGS.flatMap((a) =>
+ [-1n, 0n, 9n, 10n, 1000n].map((b) => ({a, b, payload: 'p'})),
+);
+
+interface TripleRow {
+ a: string;
+ b: bigint;
+ c: boolean;
+ payload: string;
+}
+
+const TripleKey = defineTable<TripleRow, 'a' | 'b' | 'c'>({
+ name: 'kv_plan_triple',
+ columns: ['a', 'b', 'c', 'payload'],
+ primaryKey: ['a', 'b', 'c'],
+});
+
+const tripleRows: Array<Row> = TRICKY_STRINGS.flatMap((a) =>
+ [9n, 10n].flatMap((b) => [true, false].map((c) => ({a, b, c, payload: 'p'}))),
+);
+
+interface WideRow {
+ a: string;
+ b: bigint;
+ c: boolean;
+ d: Date | null;
+ e: Buffer;
+ f: number;
+ payload: string;
+}
+
+const WideKey = defineTable<WideRow, 'a' | 'b' | 'c' | 'd' | 'e' | 'f'>({
+ name: 'kv_plan_wide',
+ columns: ['a', 'b', 'c', 'd', 'e', 'f', 'payload'],
+ primaryKey: ['a', 'b', 'c', 'd', 'e', 'f'],
+});
+
+const wideRows: Array<Row> = ['a', 'a"b', '\u{1f600}'].flatMap((a) =>
+ [9n, 10n].flatMap((b) =>
+ [true, false].flatMap((c) =>
+ [null, new Date('2020-01-01T00:00:00.000Z')].flatMap((d) =>
+ [Buffer.from('one'), Buffer.from([0x00, 0x1f, 0xff])].flatMap((e) =>
+ [1, 2].map((f) => ({a, b, c, d, e, f, payload: 'p'})),
+ ),
+ ),
+ ),
+ ),
+);
+
+interface PartitionedRow {
+ a: string;
+ b: string;
+ payload: string;
+}
+
+const PartitionedKey = defineTable<PartitionedRow, 'a' | 'b', 'b'>({
+ name: 'kv_plan_partitioned',
+ columns: ['a', 'b', 'payload'],
+ primaryKey: ['a', 'b'],
+ partitionKey: ['b'],
+});
+
+const partitionedRows: Array<Row> = TRICKY_STRINGS.flatMap((a) => TRICKY_STRINGS.map((b) => ({a, b, payload: 'p'})));
+
+describe('PostgresKvQueryExecutor candidate planner', () => {
+ it('pins a single-column primary key by equality', () => {
+ for (const a of TRICKY_STRINGS) {
+ expect(checkQuery(SingleKey.select({where: SingleKey.where.eq('a')}).bind({a}), singleRows).kind).toBe('rowKeys');
+ }
+ });
+
+ it('pins a single-column primary key by IN', () => {
+ const query = SingleKey.select({where: SingleKey.where.in('a', 'ids')}).bind({
+ ids: ['a', 'a\u0001b', 'a"b', 'a'],
+ });
+ expect(checkQuery(query, singleRows).kind).toBe('rowKeys');
+ });
+
+ it('returns nothing for an empty IN list', () => {
+ for (const params of [{ids: []}, {ids: new Set<string>()}, {}]) {
+ const query = SingleKey.select({where: SingleKey.where.in('a', 'ids')}).bind(params as CassandraParams);
+ expect(checkQuery(query, singleRows).kind).toBe('none');
+ }
+ });
+
+ it('ranges over the leading column of a composite primary key', () => {
+ for (const a of TRICKY_STRINGS) {
+ expect(checkQuery(PairKey.select({where: PairKey.where.eq('a')}).bind({a}), pairRows).kind).toBe('range');
+ }
+ });
+
+ it('ranges over the leading column when a comparison follows', () => {
+ const query = PairKey.select({where: [PairKey.where.eq('a'), PairKey.where.lt('b')]}).bind({
+ a: 'a\u0001b',
+ b: 10n,
+ });
+ expect(checkQuery(query, pairRows).kind).toBe('range');
+ });
+
+ it('emits one range per value when the leading column uses IN', () => {
+ const query = PairKey.select({where: PairKey.where.in('a', 'ids')}).bind({
+ ids: ['a', 'a\u0001b', 'a\u001fb', 'a'],
+ });
+ expect(checkQuery(query, pairRows).kind).toBe('ranges');
+ });
+
+ it('pins the full primary key across a cartesian product of IN lists', () => {
+ const query = PairKey.select({where: [PairKey.where.in('a', 'ids'), PairKey.where.in('b', 'bs')]}).bind({
+ ids: ['a', 'a\u007fb'],
+ bs: [9n, 10n],
+ });
+ expect(checkQuery(query, pairRows).kind).toBe('rowKeys');
+ });
+
+ it('ranges over a two-column prefix of a three-column primary key', () => {
+ for (const a of TRICKY_STRINGS) {
+ const query = TripleKey.select({where: [TripleKey.where.eq('a'), TripleKey.where.eq('b')]}).bind({a, b: 9n});
+ expect(checkQuery(query, tripleRows).kind).toBe('range');
+ }
+ });
+
+ it('falls back to a scan when the leading column is unpinned', () => {
+ const query = TripleKey.select({where: [TripleKey.where.eq('b'), TripleKey.where.eq('c')]}).bind({b: 9n, c: true});
+ expect(checkQuery(query, tripleRows).kind).toBe('scan');
+ });
+
+ it('pins every column of a six-column primary key', () => {
+ const query = WideKey.select({
+ where: [
+ WideKey.where.eq('a'),
+ WideKey.where.eq('b'),
+ WideKey.where.eq('c'),
+ WideKey.where.eq('d'),
+ WideKey.where.eq('e'),
+ WideKey.where.eq('f'),
+ ],
+ }).bind({a: '\u{1f600}', b: 10n, c: false, d: null, e: Buffer.from([0x00, 0x1f, 0xff]), f: 2});
+ expect(checkQuery(query, wideRows).kind).toBe('rowKeys');
+ });
+
+ it('stops at the first gap in the primary key', () => {
+ const query = WideKey.select({
+ where: [WideKey.where.eq('a'), WideKey.where.eq('b'), WideKey.where.eq('f')],
+ }).bind({a: 'a', b: 9n, f: 1});
+ expect(checkQuery(query, wideRows).kind).toBe('range');
+ });
+
+ it('uses the partition key when it is not a prefix of the primary key', () => {
+ for (const b of TRICKY_STRINGS) {
+ const query = PartitionedKey.select({where: PartitionedKey.where.eq('b')}).bind({b});
+ expect(checkQuery(query, partitionedRows).kind).toBe('partitionKeys');
+ }
+ });
+
+ it('uses the partition key for an IN on the partition column', () => {
+ const query = PartitionedKey.select({where: PartitionedKey.where.in('b', 'bs')}).bind({
+ bs: new Set(['a', 'a\u0001b', 'a\\b']),
+ });
+ expect(checkQuery(query, partitionedRows).kind).toBe('partitionKeys');
+ });
+
+ it('scans a table with no predicate', () => {
+ expect(checkQuery(SingleKey.select().bind({}), singleRows).kind).toBe('scan');
+ });
+});
+
+describe('PostgresKvQueryExecutor plan exactness', () => {
+ it('is exact when the key predicate consumes every clause', () => {
+ expect(planOf(PairKey.select({where: PairKey.where.eq('a')}).bind({a: 'a'})).plan.exact).toBe(true);
+ expect(planOf(SingleKey.select().bind({})).plan.exact).toBe(true);
+ expect(planOf(PartitionedKey.select({where: PartitionedKey.where.eq('b')}).bind({b: 'a'})).plan.exact).toBe(true);
+ });
+
+ it('is inexact when a clause is left for the JavaScript filter', () => {
+ const bounded = PairKey.select({where: [PairKey.where.eq('a'), PairKey.where.lt('b')]}).bind({a: 'a', b: 9n});
+ expect(planOf(bounded).plan.exact).toBe(false);
+ expect(planOf(TripleKey.select({where: TripleKey.where.eq('b')}).bind({b: 9n})).plan.exact).toBe(false);
+ });
+
+ it('is inexact when a limit or an order is present', () => {
+ expect(planOf(PairKey.select({where: PairKey.where.eq('a'), limit: 5}).bind({a: 'a'})).plan.exact).toBe(false);
+ const ordered = PairKey.select({where: PairKey.where.eq('a'), orderBy: {col: 'b'}}).bind({a: 'a'});
+ expect(planOf(ordered).plan.exact).toBe(false);
+ });
+
+ it('is inexact when a pinned value has no stable key encoding', () => {
+ expect(planOf(SingleKey.select({where: SingleKey.where.eq('a')}).bind({a: {}})).plan.exact).toBe(false);
+ expect(planOf(SingleKey.select({where: SingleKey.where.eq('a')}).bind({a: Number.NaN})).plan.exact).toBe(false);
+ });
+
+ it('is exact for an empty IN list regardless of the other clauses', () => {
+ const query = PairKey.select({where: [PairKey.where.in('a', 'ids'), PairKey.where.lt('b')], limit: 3}).bind({
+ ids: [],
+ b: 9n,
+ });
+ expect(planOf(query).plan.exact).toBe(true);
+ });
+});
+
+describe('PostgresKvQueryExecutor plan totality', () => {
+ it('leaves a column unpinned when its value has no key encoding at all', () => {
+ const invalid = PairKey.select({where: PairKey.where.eq('a')}).bind({a: new Date('nope')});
+ const {plan} = planOf(invalid);
+ expect(plan.candidates.kind).toBe('scan');
+ expect(plan.exact).toBe(false);
+ });
+
+ it('leaves a trailing column unpinned without losing the leading prefix', () => {
+ const query = PairKey.select({where: [PairKey.where.eq('a'), PairKey.where.eq('b')]}).bind({
+ a: 'a',
+ b: new Date('nope') as never,
+ });
+ const {plan} = planOf(query);
+ expect(plan.candidates.kind).toBe('range');
+ expect(plan.exact).toBe(false);
+ });
+
+ it('does not treat a scalar bound to an IN parameter as an iterable', () => {
+ const query = SingleKey.select({where: SingleKey.where.in('a', 'ids')}).bind({ids: 'ab' as never});
+ const {plan} = planOf(query);
+ expect(plan.candidates.kind).toBe('scan');
+ expect(plan.exact).toBe(false);
+ });
+});
+
+describe('PostgresKvQueryExecutor plan size caps', () => {
+ it('keeps a full primary key product as row keys rather than demoting it to ranges', () => {
+ const ids = Array.from({length: 30}, (_, i) => `id${i}`);
+ const seqs = Array.from({length: 30}, (_, i) => BigInt(i));
+ const query = PairKey.select({where: [PairKey.where.in('a', 'ids'), PairKey.where.in('b', 'seqs')]}).bind({
+ ids,
+ seqs,
+ });
+ const {plan} = planOf(query);
+ expect(plan.candidates.kind).toBe('rowKeys');
+ expect(plan.exact).toBe(true);
+ if (plan.candidates.kind !== 'rowKeys') return;
+ expect(plan.candidates.rowKeys.length).toBe(900);
+ });
+
+ it('caps the number of prefix ranges and degrades to a scan instead', () => {
+ const under = TripleKey.select({where: TripleKey.where.in('a', 'ids')}).bind({
+ ids: Array.from({length: 256}, (_, i) => `id${i}`),
+ });
+ expect(planOf(under).plan.candidates.kind).toBe('ranges');
+ const over = TripleKey.select({where: TripleKey.where.in('a', 'ids')}).bind({
+ ids: Array.from({length: 257}, (_, i) => `id${i}`),
+ });
+ const {plan} = planOf(over);
+ expect(plan.candidates.kind).toBe('scan');
+ expect(plan.exact).toBe(false);
+ });
+
+ it('enforces the cap even when the trailing pinned column is an equality', () => {
+ const ids = Array.from({length: 40}, (_, i) => `id${i}`);
+ const query = WideKey.select({
+ where: [WideKey.where.in('a', 'ids'), WideKey.where.in('b', 'seqs'), WideKey.where.eq('c')],
+ }).bind({ids, seqs: Array.from({length: 40}, (_, i) => BigInt(i)), c: true});
+ const {plan} = planOf(query);
+ expect(plan.candidates.kind).toBe('ranges');
+ expect(plan.exact).toBe(false);
+ if (plan.candidates.kind !== 'ranges') return;
+ expect(plan.candidates.lowerBounds.length).toBe(40);
+ });
+});
+
+describe('PostgresKvQueryExecutor prefix bounds', () => {
+ it('brackets exactly the rows sharing the pinned prefix', () => {
+ const {plan} = planOf(PairKey.select({where: PairKey.where.eq('a')}).bind({a: 'a'}));
+ expect(plan.candidates.kind).toBe('range');
+ if (plan.candidates.kind !== 'range') return;
+ expect(plan.candidates.lowerBound).toBe(`"a"${SEPARATOR}`);
+ expect(plan.candidates.upperBound).toBe('"a" ');
+ expect(byteCompare(plan.candidates.lowerBound, plan.candidates.upperBound)).toBeLessThan(0);
+ });
+});
+
+describe('PostgresKvQueryExecutor key equality', () => {
+ const eqWhere = [{kind: 'eq', col: 'a', param: 'a'}] as never;
+ const corpus: Array<[string, unknown]> = [
+ ['null', null],
+ ['undefined', undefined],
+ ['empty-string', ''],
+ ['date-string', '2020-01-01'],
+ ['days-string', '18262'],
+ ['zero', 0],
+ ['neg-zero', -0],
+ ['number', 18262],
+ ['bigint', 18262n],
+ ['true', true],
+ ['false', false],
+ ['date', new Date('2020-01-01T00:00:00.000Z')],
+ ['buffer', Buffer.from('2020-01-01')],
+ ['local-date', cassandra.types.LocalDate.fromString('2020-01-01')],
+ ['other-local-date', cassandra.types.LocalDate.fromString('2021-06-05')],
+ ];
+
+ it('accepts an equality exactly when both sides share a row key', () => {
+ const deltas: Array<string> = [];
+ for (const [storedName, stored] of corpus) {
+ for (const [paramName, param] of corpus) {
+ const accepted = matchesWhere({a: stored}, eqWhere, {a: param} as CassandraParams);
+ const sameKey = keyFromColumns(['a'], {a: stored}) === keyFromColumns(['a'], {a: param});
+ if (accepted !== sameKey)
+ deltas.push(`stored=${storedName} param=${paramName} match=${accepted} key=${sameKey}`);
+ }
+ }
+ expect(deltas).toEqual([]);
+ });
+
+ it('does not equate a LocalDate with its string rendering', () => {
+ const localDate = cassandra.types.LocalDate.fromString('2020-01-01');
+ expect(matchesWhere({a: localDate}, eqWhere, {a: localDate} as CassandraParams)).toBe(true);
+ expect(matchesWhere({a: localDate}, eqWhere, {a: '2020-01-01'} as CassandraParams)).toBe(false);
+ expect(matchesWhere({a: '2020-01-01'}, eqWhere, {a: localDate} as CassandraParams)).toBe(false);
+ });
+});
diff --git a/fluxer_api/src/api/database/PostgresKvUpgradeSafety.test.ts b/fluxer_api/src/api/database/PostgresKvUpgradeSafety.test.ts
new file mode 100644
index 000000000..2fb9dba06
--- /dev/null
+++ b/fluxer_api/src/api/database/PostgresKvUpgradeSafety.test.ts
@@ -0,0 +1,484 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import {createServer} from 'node:net';
+import {
+ getDefaultPostgresClient,
+ type IPostgresClient,
+ initPostgres,
+ type PostgresQueryable,
+ shutdownPostgres,
+} from '@pkgs/postgres/src/Client';
+import {afterAll, beforeAll, describe, expect, it} from 'vitest';
+import {GuildMembers, ReadStates, Users} from '../Tables';
+import {LegacyPostgresKvQueryExecutor, legacyEnsurePostgresKvSchema} from './__testref__/LegacyPostgresKvQueryExecutor';
+import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+
+const POSTGRES_IMAGE = 'postgres:16-alpine';
+const CONTAINER = `fluxer-kvupgrade-${process.pid.toString(36)}-${Date.now().toString(36)}`;
+const KV = 'kv_upgrade';
+const dockerUp = spawnSync('docker', ['version'], {stdio: 'ignore'}).status === 0;
+
+let PORT = 0;
+
+async function sleep(ms: number): Promise<void> {
+ await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function freePort(): Promise<number> {
+ return new Promise((resolve, reject) => {
+ const server = createServer();
+ server.on('error', reject);
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address();
+ if (typeof address === 'string' || address === null) {
+ reject(new Error('no port'));
+ return;
+ }
+ const port = address.port;
+ server.close(() => resolve(port));
+ });
+ });
+}
+
+if (dockerUp) {
+ beforeAll(async () => {
+ PORT = await freePort();
+ execFileSync(
+ 'docker',
+ [
+ 'run',
+ '-d',
+ '--name',
+ CONTAINER,
+ '-e',
+ 'POSTGRES_USER=fluxer',
+ '-e',
+ 'POSTGRES_PASSWORD=fluxer',
+ '-e',
+ 'POSTGRES_DB=fluxer',
+ '-p',
+ `127.0.0.1:${PORT}:5432`,
+ POSTGRES_IMAGE,
+ '-c',
+ 'fsync=off',
+ '-c',
+ 'synchronous_commit=off',
+ ],
+ {stdio: 'ignore'},
+ );
+ for (let attempt = 0; attempt < 180; attempt += 1) {
+ await sleep(500);
+ const probe = spawnSync('docker', ['exec', CONTAINER, 'pg_isready', '-U', 'fluxer', '-d', 'fluxer'], {
+ stdio: 'ignore',
+ });
+ if (probe.status !== 0) continue;
+ try {
+ await initPostgres({url: `postgres://fluxer:[email protected]:${PORT}/fluxer`, maxConnections: 8});
+ await getDefaultPostgresClient().query('SELECT 1');
+ return;
+ } catch {
+ await shutdownPostgres().catch(() => {});
+ }
+ }
+ throw new Error('postgres not ready');
+ }, 900_000);
+
+ afterAll(async () => {
+ await shutdownPostgres().catch(() => {});
+ spawnSync('docker', ['rm', '-f', CONTAINER], {stdio: 'ignore'});
+ });
+}
+
+class TableClient implements IPostgresClient {
+ constructor(
+ private readonly inner: IPostgresClient,
+ private readonly name: string,
+ ) {}
+ query: IPostgresClient['query'] = (text, values) => this.inner.query(text, values);
+ async connect() {
+ await this.inner.connect();
+ }
+ async shutdown() {}
+ isConnected() {
+ return this.inner.isConnected();
+ }
+ async transaction<T>(fn: (c: PostgresQueryable) => Promise<T>) {
+ return this.inner.transaction(fn);
+ }
+ kvTable() {
+ return this.name;
+ }
+}
+
+function fp(rows: ReadonlyArray<Row>): string {
+ return rows
+ .map((r) =>
+ Object.entries(r)
+ .map(([k, v]) => `${k}=${String(v)}`)
+ .join('|'),
+ )
+ .join(';');
+}
+
+function pq(cql: string, params: Record<string, unknown>) {
+ return {cql, params} as never;
+}
+
+const suite = dockerUp ? describe : describe.skip;
+
+suite('postgres kv upgrade safety', () => {
+ let raw: IPostgresClient;
+ let client: TableClient;
+ let legacy: LegacyPostgresKvQueryExecutor;
+ let next: PostgresKvQueryExecutor;
+
+ const usersScan = () => pq(Users.selectCql(), {});
+
+ beforeAll(async () => {
+ await initPostgres({url: `postgres://fluxer:[email protected]:${PORT}/fluxer`, maxConnections: 8});
+ raw = getDefaultPostgresClient();
+ await raw.query(`DROP TABLE IF EXISTS ${KV}`);
+ client = new TableClient(raw, KV);
+ await legacyEnsurePostgresKvSchema(client);
+ legacy = new LegacyPostgresKvQueryExecutor(client);
+ next = new PostgresKvQueryExecutor(client);
+ for (let i = 0; i < 40; i += 1) {
+ await legacy.executeQuery(
+ Users.upsertAll({user_id: BigInt(1000 + i), username: `u${i}`, discriminator: `${i}`} as never) as never,
+ );
+ }
+ for (let g = 0; g < 3; g += 1) {
+ for (let u = 0; u < 12; u += 1) {
+ await legacy.executeQuery(
+ GuildMembers.upsertAll({guild_id: BigInt(500 + g), user_id: BigInt(1000 + u)} as never) as never,
+ );
+ }
+ }
+ for (let u = 0; u < 5; u += 1) {
+ for (let c = 0; c < 4; c += 1) {
+ await legacy.executeQuery(
+ ReadStates.upsertAll({user_id: BigInt(1000 + u), channel_id: BigInt(7000 + c)} as never) as never,
+ );
+ }
+ }
+ }, 300_000);
+
+ afterAll(async () => {
+ await shutdownPostgres().catch(() => {});
+ });
+
+ it('reads data written by the old image identically', async () => {
+ const shapes = [
+ {name: 'users scan', q: usersScan()},
+ {
+ name: 'guild members by guild',
+ q: pq(GuildMembers.selectCql({where: GuildMembers.where.eq('guild_id')}), {guild_id: 501n}),
+ },
+ {
+ name: 'read states by user',
+ q: pq(ReadStates.selectCql({where: ReadStates.where.eq('user_id')}), {user_id: 1002n}),
+ },
+ {name: 'user by id', q: pq(Users.selectCql({where: Users.where.eq('user_id')}), {user_id: 1005n})},
+ ];
+ for (const shape of shapes) {
+ const a = await legacy.executeQuery(shape.q);
+ const b = await next.executeQuery(shape.q);
+ expect(fp(b), shape.name).toBe(fp(a));
+ expect(a.length, shape.name).toBeGreaterThan(0);
+ }
+ });
+
+ async function drain(order: Array<'legacy' | 'next'>, pageSize: number) {
+ const keys: Array<string> = [];
+ let state: string | null = null;
+ let pages = 0;
+ try {
+ for (;;) {
+ const who = order[pages % order.length]!;
+ const exec = who === 'legacy' ? legacy : next;
+ const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery(usersScan(), {
+ pageSize,
+ pageState: state,
+ });
+ pages += 1;
+ for (const row of page.rows) keys.push(String(row.user_id));
+ state = page.pageState;
+ if (!state) break;
+ if (pages > 200) throw new Error('runaway paging loop');
+ }
+ } catch (error) {
+ return {keys, error: (error as Error).message, pages};
+ }
+ return {keys, error: null, pages};
+ }
+
+ it('pages consistently with only the old image running', async () => {
+ const r = await drain(['legacy'], 7);
+ expect(r.error).toBeNull();
+ expect(r.keys.length).toBe(40);
+ expect(new Set(r.keys).size).toBe(40);
+ });
+
+ it('pages consistently with only the new image running', async () => {
+ const r = await drain(['next'], 7);
+ expect(r.error).toBeNull();
+ expect(r.keys.length).toBe(40);
+ expect(new Set(r.keys).size).toBe(40);
+ });
+
+ it('ROLLING RESTART: an old replica consumes a token minted by the new one', async () => {
+ const first = await next.executePagedQuery(usersScan(), {pageSize: 7, pageState: null});
+ expect(first.pageState).toBeTruthy();
+ const token = JSON.parse(Buffer.from(first.pageState!, 'base64url').toString('utf8'));
+ let failure: string | null = null;
+ let rows = 0;
+ try {
+ const second = await legacy.executePagedQuery(usersScan(), {pageSize: 7, pageState: first.pageState});
+ rows = second.rows.length;
+ } catch (error) {
+ failure = (error as Error).message;
+ }
+ console.log(`NEW->OLD token=${JSON.stringify(token)} legacy=${failure ?? `${rows} rows`}`);
+ expect(failure).toBeNull();
+ });
+
+ it('ROLLING RESTART: alternating old and new replicas drain the scan', async () => {
+ const a = await drain(['next', 'legacy'], 7);
+ const b = await drain(['legacy', 'next'], 7);
+ console.log(`new-first err=${a.error} pages=${a.pages} total=${a.keys.length} unique=${new Set(a.keys).size}`);
+ console.log(`old-first err=${b.error} pages=${b.pages} total=${b.keys.length} unique=${new Set(b.keys).size}`);
+ expect(a.error).toBeNull();
+ expect(new Set(a.keys).size).toBe(40);
+ expect(b.error).toBeNull();
+ expect(new Set(b.keys).size).toBe(40);
+ });
+
+ it('a dual {offset, after} token would survive both directions', async () => {
+ const first = await next.executePagedQuery(usersScan(), {pageSize: 7, pageState: null});
+ const cursor = JSON.parse(Buffer.from(first.pageState!, 'base64url').toString('utf8')) as {after: string};
+ const dual = Buffer.from(JSON.stringify({offset: first.rows.length, after: cursor.after})).toString('base64url');
+ const viaOld = await legacy.executePagedQuery(usersScan(), {pageSize: 7, pageState: dual});
+ const viaNew = await next.executePagedQuery(usersScan(), {pageSize: 7, pageState: dual});
+ console.log(`dual token -> old replica: ${viaOld.rows.length} rows, new replica: ${viaNew.rows.length} rows`);
+ expect(viaOld.rows.length).toBe(7);
+ expect(viaNew.rows.length).toBe(7);
+ });
+
+ it('deletes exactly the same rows as the old image', async () => {
+ const seed = async (exec: LegacyPostgresKvQueryExecutor | PostgresKvQueryExecutor, base: bigint) => {
+ for (let c = 0; c < 5; c += 1) {
+ await exec.executeQuery(ReadStates.upsertAll({user_id: base, channel_id: BigInt(9000 + c)} as never) as never);
+ }
+ };
+ await seed(legacy, 4001n);
+ await seed(legacy, 4002n);
+ await legacy.executeQuery(
+ ReadStates.delete({where: ReadStates.where.eq('user_id')}).bind({user_id: 4001n}) as never,
+ );
+ await next.executeQuery(ReadStates.delete({where: ReadStates.where.eq('user_id')}).bind({user_id: 4002n}) as never);
+ const left = await raw.query<{row_key: string}>(
+ `SELECT row_key FROM ${KV} WHERE table_name = 'read_states' AND row_key LIKE '%400%' ORDER BY row_key`,
+ );
+ console.log(`rows left after legacy delete + new delete: ${left.rows.length}`);
+ expect(left.rows.length).toBe(0);
+ });
+
+ it('writes byte-identical row_key and partition_key to the old image', async () => {
+ const OLD = `${KV}_w_old`;
+ const NEW = `${KV}_w_new`;
+ await raw.query(`DROP TABLE IF EXISTS ${OLD}`);
+ await raw.query(`DROP TABLE IF EXISTS ${NEW}`);
+ const oldClient = new TableClient(raw, OLD);
+ const newClient = new TableClient(raw, NEW);
+ await legacyEnsurePostgresKvSchema(oldClient);
+ await ensurePostgresKvSchema(newClient);
+ const oldExec = new LegacyPostgresKvQueryExecutor(oldClient);
+ const newExec = new PostgresKvQueryExecutor(newClient);
+ for (let i = 0; i < 30; i += 1) {
+ const row = {user_id: BigInt(9000 + i), channel_id: BigInt(500 + i)} as never;
+ await oldExec.executeQuery(ReadStates.upsertAll(row) as never);
+ await newExec.executeQuery(ReadStates.upsertAll(row) as never);
+ const g = {guild_id: BigInt(70 + i), user_id: BigInt(9000 + i)} as never;
+ await oldExec.executeQuery(GuildMembers.upsertAll(g) as never);
+ await newExec.executeQuery(GuildMembers.upsertAll(g) as never);
+ }
+ const diff = await raw.query<{n: string}>(`
+ SELECT count(*) AS n FROM (
+ (SELECT table_name, partition_key, row_key FROM ${OLD}
+ EXCEPT SELECT table_name, partition_key, row_key FROM ${NEW})
+ UNION ALL
+ (SELECT table_name, partition_key, row_key FROM ${NEW}
+ EXCEPT SELECT table_name, partition_key, row_key FROM ${OLD})
+ ) d`);
+ const schemaDiff = await raw.query<{n: string}>(`
+ SELECT count(*) AS n FROM (
+ (SELECT replace(indexdef, '${OLD}', 'KV') FROM pg_indexes WHERE tablename = '${OLD}'
+ EXCEPT SELECT replace(indexdef, '${NEW}', 'KV') FROM pg_indexes WHERE tablename = '${NEW}')
+ UNION ALL
+ (SELECT replace(indexdef, '${NEW}', 'KV') FROM pg_indexes WHERE tablename = '${NEW}'
+ EXCEPT SELECT replace(indexdef, '${OLD}', 'KV') FROM pg_indexes WHERE tablename = '${OLD}')
+ ) d`);
+ console.log(`key diffs=${diff.rows[0]!.n} index-definition diffs=${schemaDiff.rows[0]!.n}`);
+ expect(Number(diff.rows[0]!.n)).toBe(0);
+ expect(Number(schemaDiff.rows[0]!.n)).toBe(0);
+ });
+
+ it('survives three simultaneous boots (two api replicas and a worker)', async () => {
+ const BOOT = `${KV}_boot`;
+ await raw.query(`DROP TABLE IF EXISTS ${BOOT}`);
+ const results = await Promise.allSettled([
+ ensurePostgresKvSchema(new TableClient(raw, BOOT)),
+ ensurePostgresKvSchema(new TableClient(raw, BOOT)),
+ ensurePostgresKvSchema(new TableClient(raw, BOOT)),
+ ]);
+ const failures = results.filter((r) => r.status === 'rejected');
+ console.log(`concurrent boots rejected: ${failures.length}`);
+ for (const f of failures) console.log(` ${(f as PromiseRejectedResult).reason}`);
+ expect(failures.length).toBe(0);
+ }, 120_000);
+
+ it('backfills the messages partition key once and then skips the update', async () => {
+ const BACKFILL = 'kv_backfill';
+ const legacyKey = `"c"${String.fromCharCode(31)}"b"${String.fromCharCode(31)}"m"`;
+ const backfillClient = new TableClient(raw, BACKFILL);
+ await raw.query(`DROP TABLE IF EXISTS ${BACKFILL}`);
+ await ensurePostgresKvSchema(backfillClient);
+ await raw.query(
+ `INSERT INTO ${BACKFILL} (table_name, partition_key, row_key, row_data) VALUES ('messages', $1, $1, '{}'::jsonb)`,
+ [legacyKey],
+ );
+ const pendingBefore = await raw.query(
+ `SELECT 1 FROM ${BACKFILL} WHERE table_name = 'messages' AND partition_key = row_key AND split_part(row_key, chr(31), 3) <> ''`,
+ );
+ expect(pendingBefore.rows.length).toBe(1);
+ await ensurePostgresKvSchema(backfillClient);
+ const after = await raw.query<{partition_key: string}>(`SELECT partition_key FROM ${BACKFILL}`);
+ expect(after.rows[0]?.partition_key).toBe(`"c"${String.fromCharCode(31)}"b"`);
+ const pendingAfter = await raw.query(
+ `SELECT 1 FROM ${BACKFILL} WHERE table_name = 'messages' AND partition_key = row_key AND split_part(row_key, chr(31), 3) <> ''`,
+ );
+ expect(pendingAfter.rows.length).toBe(0);
+ await ensurePostgresKvSchema(backfillClient);
+ const stable = await raw.query<{partition_key: string}>(`SELECT partition_key FROM ${BACKFILL}`);
+ expect(stable.rows[0]?.partition_key).toBe(`"c"${String.fromCharCode(31)}"b"`);
+ await raw.query(`DROP TABLE IF EXISTS ${BACKFILL}`);
+ }, 120_000);
+
+ it('boot does not repair an invalid C-collation index', async () => {
+ const q = pq(ReadStates.selectCql({where: ReadStates.where.eq('user_id')}), {user_id: 1002n});
+ const before = await next.executeQuery(q);
+ await raw.query(`UPDATE pg_index SET indisvalid = false WHERE indexrelid = '${KV}_row_key_c_idx'::regclass`);
+ await ensurePostgresKvSchema(client);
+ const check = await raw.query<{indisvalid: boolean}>(
+ `SELECT indisvalid FROM pg_index WHERE indexrelid = '${KV}_row_key_c_idx'::regclass`,
+ );
+ const after = await next.executeQuery(q);
+ await raw.query(`UPDATE pg_index SET indisvalid = true WHERE indexrelid = '${KV}_row_key_c_idx'::regclass`);
+ console.log(`index valid after boot: ${check.rows[0]?.indisvalid}`);
+ expect(String(after.map((r) => Object.values(r).map(String).join(',')))).toBe(
+ String(before.map((r) => Object.values(r).map(String).join(','))),
+ );
+ expect(check.rows[0]?.indisvalid).toBe(true);
+ });
+
+ describe('multi-range plan shape on a populated upgraded database', () => {
+ const PERF = 'kv_ranges_perf';
+ const SEP = String.fromCharCode(31);
+ const BASE = 1456074443984486400n;
+ let perf: IPostgresClient;
+
+ beforeAll(async () => {
+ const db = getDefaultPostgresClient();
+ await db.query(`DROP TABLE IF EXISTS ${PERF}`);
+ perf = new TableClient(db, PERF);
+ await ensurePostgresKvSchema(perf);
+ await db.query(
+ `INSERT INTO ${PERF} (table_name, partition_key, row_key, row_data)
+SELECT 'push_subscriptions',
+ format('{"__fluxer_type":"bigint","value":"%s"}%s{"__fluxer_type":"bigint","value":"%s"}', u, chr(31), s),
+ format('{"__fluxer_type":"bigint","value":"%s"}%s{"__fluxer_type":"bigint","value":"%s"}', u, chr(31), s),
+ jsonb_build_object(
+ 'user_id', jsonb_build_object('__fluxer_type', 'bigint', 'value', u::text),
+ 'subscription_id', jsonb_build_object('__fluxer_type', 'bigint', 'value', s::text),
+ 'endpoint', repeat('e', 300))
+FROM generate_series($1::bigint, $1::bigint + 19999) u, generate_series(1, 3) s`,
+ [BASE.toString()],
+ );
+ await db.query(`ANALYZE ${PERF}`);
+ }, 300_000);
+
+ it('emits a multi-range plan that is not slower than the tier-3 scan it replaces', async () => {
+ const {PushSubscriptions} = await import('../Tables');
+ const {buildCandidatePlan, planFragments} = await import('./PostgresKvQueryExecutor');
+ const {getKvMeta} = await import('./CassandraMetaRegistry');
+ const db = getDefaultPostgresClient();
+ const explain = async (sql: string, params: Array<unknown>) => {
+ const r = await db.query<Record<string, string>>(`EXPLAIN (ANALYZE, BUFFERS) ${sql}`, params);
+ const plan = r.rows.map((row) => row['QUERY PLAN']!).join('\n');
+ return {plan, ms: Number(/Execution Time: ([\d.]+)/.exec(plan)?.[1] ?? 'NaN')};
+ };
+ const cql = PushSubscriptions.selectCql({where: PushSubscriptions.where.in('user_id', 'user_ids')});
+ const meta = getKvMeta(cql)!;
+ const scan = await explain(
+ `SELECT kv.row_key, kv.row_data FROM ${PERF} kv WHERE kv.table_name = $1 AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
+ ['push_subscriptions'],
+ );
+ const deltas: Array<string> = [];
+ for (const size of [5, 10, 25, 50, 100]) {
+ const params = {user_ids: Array.from({length: size}, (_, i) => BASE + BigInt(i * 3))};
+ const plan = buildCandidatePlan(meta, params as never);
+ expect(plan.candidates.kind).toBe('ranges');
+ const fragments = planFragments(plan.candidates);
+ expect(fragments.predicate).not.toContain('unnest');
+ const pushed = await explain(
+ `SELECT kv.row_key, kv.row_data FROM ${PERF} kv WHERE kv.table_name = $1${fragments.predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now())`,
+ ['push_subscriptions', ...fragments.params],
+ );
+ const verdict = `n=${size} pushdown=${pushed.ms.toFixed(2)}ms scan=${scan.ms.toFixed(2)}ms`;
+ console.log(verdict);
+ if (pushed.ms > scan.ms) deltas.push(`${verdict}\n${pushed.plan}`);
+ }
+ expect(deltas.join('\n')).toBe('');
+ const overCap = buildCandidatePlan(meta, {
+ user_ids: Array.from({length: 257}, (_, i) => BASE + BigInt(i * 3)),
+ } as never);
+ expect(overCap.candidates.kind).toBe('scan');
+ expect(SEP).toBe('\u001f');
+ }, 300_000);
+ });
+});
+
+describe('candidate plan shapes reached by real queries', () => {
+ it('classifies the push fan-out and webhook lookups', async () => {
+ const {PushSubscriptions, Webhooks} = await import('../Tables');
+ const {buildCandidatePlan} = await import('./PostgresKvQueryExecutor');
+ const {getKvMeta} = await import('./CassandraMetaRegistry');
+ const cases = [
+ {
+ name: 'push_subscriptions IN(user_id) x100',
+ cql: PushSubscriptions.selectCql({where: PushSubscriptions.where.in('user_id', 'user_ids')}),
+ params: {user_ids: Array.from({length: 100}, (_, i) => BigInt(1000 + i))},
+ },
+ {
+ name: 'webhooks IN(webhook_id) x100',
+ cql: Webhooks.selectCql({where: Webhooks.where.in('webhook_id', 'webhook_ids')}),
+ params: {webhook_ids: Array.from({length: 100}, (_, i) => BigInt(1000 + i))},
+ },
+ ];
+ for (const c of cases) {
+ const meta = getKvMeta(c.cql)!;
+ const plan = buildCandidatePlan(meta, c.params as never);
+ const size =
+ plan.candidates.kind === 'ranges'
+ ? plan.candidates.lowerBounds.length
+ : plan.candidates.kind === 'rowKeys'
+ ? plan.candidates.rowKeys.length
+ : 0;
+ console.log(`${c.name} -> kind=${plan.candidates.kind} n=${size} exact=${plan.exact}`);
+ }
+ expect(true).toBe(true);
+ });
+});
diff --git a/fluxer_api/src/api/database/__testref__/LegacyPostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/__testref__/LegacyPostgresKvQueryExecutor.ts
new file mode 100644
index 000000000..95639ca6c
--- /dev/null
+++ b/fluxer_api/src/api/database/__testref__/LegacyPostgresKvQueryExecutor.ts
@@ -0,0 +1,561 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {type IPostgresClient, type PostgresQueryable, quoteIdentifier} from '@pkgs/postgres/src/Client';
+import cassandra from 'cassandra-driver';
+import {getKvMeta, getTableMetadata} from '../CassandraMetaRegistry';
+import type {CassandraParams, ColumnName, KvQueryMeta, PreparedQuery, WhereExpr} from '../CassandraTypes';
+import {matchesWhere} from '../PostgresKvQueryExecutor';
+
+type Row = Record<string, unknown>;
+type EqWhereExpr = Extract<WhereExpr<Row>, {kind: 'eq'}>;
+
+interface StoredRow {
+ row_key: string;
+ row_data: unknown;
+}
+
+interface PageState {
+ offset: number;
+}
+
+const VALUE_SEPARATOR = '\u001f';
+const ENCODED_TYPE_KEY = '__fluxer_type';
+const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56;
+const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';
+
+function normalizeCql(cql: string): string {
+ return cql.replace(/\s+/g, ' ').trim();
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+ return value !== null && typeof value === 'object' && Object.getPrototypeOf(value) === Object.prototype;
+}
+
+function encodeValue(value: unknown): unknown {
+ if (value === null || value === undefined) return null;
+ if (typeof value === 'bigint') return {[ENCODED_TYPE_KEY]: 'bigint', value: value.toString()};
+ if (value instanceof Date) return {[ENCODED_TYPE_KEY]: 'date', value: value.toISOString()};
+ if (Buffer.isBuffer(value)) return {[ENCODED_TYPE_KEY]: 'buffer', value: value.toString('base64')};
+ if (value instanceof Set) return {[ENCODED_TYPE_KEY]: 'set', value: [...value.values()].map(encodeValue)};
+ if (value instanceof Map) {
+ return {
+ [ENCODED_TYPE_KEY]: 'map',
+ value: [...value.entries()].map(([key, entry]) => [encodeValue(key), encodeValue(entry)]),
+ };
+ }
+ if (typeof value === 'object' && value.constructor?.name === 'LocalDate') {
+ return {[ENCODED_TYPE_KEY]: 'local_date', value: value.toString()};
+ }
+ if (Array.isArray(value)) return value.map(encodeValue);
+ if (isPlainObject(value)) {
+ const encoded: Record<string, unknown> = {};
+ for (const [key, entry] of Object.entries(value)) {
+ encoded[key] = encodeValue(entry);
+ }
+ return encoded;
+ }
+ return value;
+}
+
+function decodeValue(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(decodeValue);
+ if (!isPlainObject(value)) return value;
+ const encodedType = value[ENCODED_TYPE_KEY];
+ if (encodedType === 'bigint') return BigInt(String(value.value));
+ if (encodedType === 'date') return new Date(String(value.value));
+ if (encodedType === 'buffer') return Buffer.from(String(value.value), 'base64');
+ if (encodedType === 'set') return new Set(((value.value as Array<unknown>) ?? []).map(decodeValue));
+ if (encodedType === 'map') {
+ return new Map(
+ ((value.value as Array<[unknown, unknown]>) ?? []).map(([key, entry]) => [decodeValue(key), decodeValue(entry)]),
+ );
+ }
+ if (encodedType === 'local_date') return cassandra.types.LocalDate.fromString(String(value.value));
+ const decoded: Record<string, unknown> = {};
+ for (const [key, entry] of Object.entries(value)) {
+ decoded[key] = decodeValue(entry);
+ }
+ return decoded;
+}
+
+function encodeRow(row: Row): Record<string, unknown> {
+ const encoded: Record<string, unknown> = {};
+ for (const [key, value] of Object.entries(row)) {
+ encoded[key] = encodeValue(value);
+ }
+ return encoded;
+}
+
+function decodeRow(value: unknown): Row {
+ const decoded = decodeValue(value);
+ if (!isPlainObject(decoded)) {
+ throw new Error('Postgres KV row payload is not an object');
+ }
+ return decoded;
+}
+
+function valueKey(value: unknown): string {
+ return JSON.stringify(encodeValue(value));
+}
+
+function keyFromColumns(columns: ReadonlyArray<string>, row: Row): string {
+ return columns.map((column) => valueKey(row[column])).join(VALUE_SEPARATOR);
+}
+
+function paramsRow(params: CassandraParams, columns: ReadonlyArray<string>): Row {
+ const row: Row = {};
+ for (const column of columns) {
+ if (!(column in params)) {
+ throw new Error(`Missing Postgres KV key parameter: ${column}`);
+ }
+ row[column] = params[column];
+ }
+ return row;
+}
+
+function rowFromParams(meta: KvQueryMeta, params: CassandraParams): Row {
+ const row: Row = {};
+ for (const column of meta.table.columns) {
+ if (column in params) {
+ row[column] = params[column];
+ }
+ }
+ if (meta.nowColumn) {
+ row[meta.nowColumn] = new Date();
+ }
+ return row;
+}
+
+function rowKey(meta: KvQueryMeta, row: Row): string {
+ return keyFromColumns(meta.table.primaryKey as ReadonlyArray<string>, row);
+}
+
+function partitionKey(meta: KvQueryMeta, row: Row): string {
+ return keyFromColumns(meta.table.partitionKey as ReadonlyArray<string>, row);
+}
+
+function rowKeyFromParams(meta: KvQueryMeta, params: CassandraParams): string {
+ return rowKey(meta, paramsRow(params, (meta.pkColumns ?? meta.table.primaryKey) as ReadonlyArray<string>));
+}
+
+function partitionKeyFromParams(meta: KvQueryMeta, params: CassandraParams): string {
+ return partitionKey(meta, paramsRow(params, meta.table.partitionKey as ReadonlyArray<string>));
+}
+
+function compareValues(left: unknown, right: unknown): number {
+ if (typeof left === 'bigint' || typeof right === 'bigint') {
+ const l = typeof left === 'bigint' ? left : BigInt(left as number | string);
+ const r = typeof right === 'bigint' ? right : BigInt(right as number | string);
+ if (l === r) return 0;
+ return l < r ? -1 : 1;
+ }
+ const l = left instanceof Date ? left.getTime() : left?.constructor?.name === 'LocalDate' ? left.toString() : left;
+ const r =
+ right instanceof Date ? right.getTime() : right?.constructor?.name === 'LocalDate' ? right.toString() : right;
+ if (Buffer.isBuffer(l) && Buffer.isBuffer(r)) return Buffer.compare(l, r);
+ if (l === r) return 0;
+ return (l as number | string) < (r as number | string) ? -1 : 1;
+}
+
+function projectRow(row: Row, columns: ReadonlyArray<string> | undefined): Row {
+ if (!columns) return {...row};
+ const projected: Row = {};
+ for (const column of columns) {
+ projected[column] = row[column];
+ }
+ return projected;
+}
+
+function sortRows(meta: KvQueryMeta, rows: Array<Row>): Array<Row> {
+ if (meta.orderBy) {
+ const direction = meta.orderBy.direction === 'DESC' ? -1 : 1;
+ return rows.sort((left, right) => compareValues(left[meta.orderBy!.col], right[meta.orderBy!.col]) * direction);
+ }
+ return rows.sort((left, right) => {
+ for (const column of meta.table.primaryKey) {
+ const cmp = compareValues(left[column], right[column]);
+ if (cmp !== 0) return cmp;
+ }
+ return 0;
+ });
+}
+
+function equalityParam(where: ReadonlyArray<WhereExpr<Row>> | undefined, column: string): string | null {
+ const clause = (where ?? []).find((entry) => entry.kind === 'eq' && entry.col === column);
+ return clause && clause.kind === 'eq' ? clause.param : null;
+}
+
+function inParam(where: ReadonlyArray<WhereExpr<Row>> | undefined, column: string): string | null {
+ const clause = (where ?? []).find((entry) => entry.kind === 'in' && entry.col === column);
+ return clause && clause.kind === 'in' ? clause.param : null;
+}
+
+function fullRowKeysFromWhere(meta: KvQueryMeta, params: CassandraParams): Array<string> | null {
+ const pk = meta.table.primaryKey as ReadonlyArray<string>;
+ const eqParams = pk.map((column) => equalityParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, column));
+ if (eqParams.every((param) => param !== null)) {
+ const row: Row = {};
+ for (let i = 0; i < pk.length; i += 1) row[pk[i]!] = params[eqParams[i]!];
+ return [rowKey(meta, row)];
+ }
+ if (pk.length === 1) {
+ const param = inParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, pk[0]!);
+ if (param) {
+ const values = params[param] as ReadonlyArray<unknown> | Set<unknown>;
+ const haystack = values instanceof Set ? [...values] : values;
+ return haystack.map((value) => rowKey(meta, {[pk[0]!]: value}));
+ }
+ }
+ return null;
+}
+
+function hasFullPartition(meta: KvQueryMeta): boolean {
+ return meta.table.partitionKey.every((column) =>
+ equalityParam(meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, column),
+ );
+}
+
+function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | null | undefined {
+ const ttlParam = meta.ttlParamName;
+ if (!ttlParam) return undefined;
+ const ttlRaw = params[ttlParam];
+ if (typeof ttlRaw !== 'number') {
+ throw new Error(`TTL parameter ${ttlParam} must be a number`);
+ }
+ return new Date(Date.now() + ttlRaw * 1000);
+}
+
+function encodePageState(pageState: PageState): string {
+ return Buffer.from(JSON.stringify(pageState)).toString('base64url');
+}
+
+function decodePageState(pageState: string | null | undefined): PageState {
+ if (!pageState) return {offset: 0};
+ const decoded = JSON.parse(Buffer.from(pageState, 'base64url').toString('utf8')) as PageState;
+ if (!Number.isInteger(decoded.offset) || decoded.offset < 0) {
+ throw new Error('Invalid Postgres KV page state');
+ }
+ return decoded;
+}
+
+function parseRawMeta(cql: string): KvQueryMeta<Row> | null {
+ const normalized = normalizeCql(cql).replace(/;$/, '');
+ const update =
+ /^UPDATE\s+([A-Za-z0-9_]+)(?:\s+USING\s+(?:TIMESTAMP|TTL)\s+:[A-Za-z0-9_]+)?\s+SET\s+(.+?)\s+WHERE\s+(.+)$/iu.exec(
+ normalized,
+ );
+ if (update) {
+ const table = tableSpec(update[1]!);
+ const patchKeys = update[2]!.split(',').map((part) => {
+ const [column, value] = part.trim().split(/\s*=\s*/u);
+ if (!column || !value?.startsWith(':')) {
+ throw new Error(`Postgres KV raw UPDATE only supports parameter assignments: ${cql}`);
+ }
+ return column;
+ });
+ const where = parseEqWhere(update[3]!, cql);
+ return {
+ action: 'patch',
+ table,
+ where,
+ patchKeys,
+ pkColumns: where.map((clause) => clause.col),
+ };
+ }
+ const select =
+ /^SELECT\s+(.+?)\s+FROM\s+([A-Za-z0-9_]+)(?:\s+WHERE\s+(.+?))?(?:\s+ALLOW\s+FILTERING)?(?:\s+LIMIT\s+(\d+))?$/iu.exec(
+ normalized,
+ );
+ if (select) {
+ const table = tableSpec(select[2]!);
+ return {
+ action: 'select',
+ table,
+ columns: select[1]!.split(',').map((part) => part.trim() as ColumnName<Row>),
+ where: select[3] ? parseEqWhere(select[3], cql) : [],
+ limit: select[4] ? Number.parseInt(select[4], 10) : undefined,
+ };
+ }
+ return null;
+}
+
+function tableSpec(tableName: string): KvQueryMeta<Row>['table'] {
+ const table = getTableMetadata(tableName);
+ if (!table) {
+ throw new Error(`Postgres KV metadata is missing for table: ${tableName}`);
+ }
+ return table;
+}
+
+function parseEqWhere(whereSql: string, cql: string): ReadonlyArray<EqWhereExpr> {
+ return whereSql.split(/\s+AND\s+/iu).map((part) => {
+ const match = /^\s*([A-Za-z0-9_]+)\s*=\s*:([A-Za-z0-9_]+)\s*$/u.exec(part.trim());
+ if (!match) {
+ throw new Error(`Postgres KV raw WHERE only supports equality predicates: ${cql}`);
+ }
+ return {kind: 'eq', col: match[1]! as ColumnName<Row>, param: match[2]!};
+ });
+}
+
+export async function legacyEnsurePostgresKvSchema(client: IPostgresClient): Promise<void> {
+ const kvTable = client.kvTable();
+ const table = quoteIdentifier(kvTable);
+ await client.transaction(async (db) => {
+ await db.query("SELECT set_config('statement_timeout', $1, true)", [POSTGRES_KV_SCHEMA_LOCK_TIMEOUT]);
+ await db.query('SELECT pg_advisory_xact_lock($1, hashtext($2))', [POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, kvTable]);
+ await db.query("SELECT set_config('statement_timeout', '0', true)");
+ await db.query(`
+CREATE TABLE IF NOT EXISTS ${table} (
+ table_name text NOT NULL,
+ partition_key text NOT NULL,
+ row_key text NOT NULL,
+ row_data jsonb NOT NULL,
+ expires_at timestamptz,
+ updated_at timestamptz NOT NULL DEFAULT now(),
+ PRIMARY KEY (table_name, row_key)
+)`);
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_partition_row_idx`)} ON ${table} (table_name, partition_key, row_key)`,
+ );
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_row_key_c_idx`)} ON ${table} (table_name, row_key COLLATE "C")`,
+ );
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_expires_idx`)} ON ${table} (expires_at) WHERE expires_at IS NOT NULL`,
+ );
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_messages_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'messages'`,
+ );
+ await db.query(
+ `CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,
+ );
+ await db.query(`
+UPDATE ${table}
+SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)
+WHERE table_name = 'messages'
+ AND partition_key = row_key
+ AND split_part(row_key, chr(31), 3) <> ''`);
+ await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`);
+ });
+}
+
+export async function legacyPruneExpiredPostgresKvRows(client: IPostgresClient, batchSize = 5000): Promise<number> {
+ if (!Number.isInteger(batchSize) || batchSize <= 0) {
+ throw new Error('Postgres KV prune batch size must be a positive integer');
+ }
+ const table = quoteIdentifier(client.kvTable());
+ const result = await client.query(
+ `
+WITH expired AS (
+ SELECT table_name, row_key
+ FROM ${table}
+ WHERE expires_at IS NOT NULL AND expires_at <= now()
+ ORDER BY expires_at
+ LIMIT $1
+ FOR UPDATE SKIP LOCKED
+)
+DELETE FROM ${table} kv
+USING expired
+WHERE kv.table_name = expired.table_name AND kv.row_key = expired.row_key`,
+ [batchSize],
+ );
+ return result.rowCount ?? 0;
+}
+
+export class LegacyPostgresKvQueryExecutor {
+ private readonly table: string;
+
+ constructor(private readonly client: IPostgresClient) {
+ this.table = quoteIdentifier(client.kvTable());
+ }
+
+ async executeQuery<T = Row, P extends CassandraParams = CassandraParams>(
+ query: PreparedQuery<P>,
+ db: PostgresQueryable = this.client,
+ ): Promise<Array<T>> {
+ const meta = this.meta(query);
+ switch (meta.action) {
+ case 'select':
+ return (await this.select(meta, query.params, db)) as Array<T>;
+ case 'count':
+ return [{count: (await this.select(meta, query.params, db)).length}] as Array<T>;
+ case 'upsert':
+ return (await this.upsert(meta, query.params, db)) as Array<T>;
+ case 'insert':
+ return (await this.upsert(meta, query.params, db)) as Array<T>;
+ case 'patch':
+ await this.patch(meta, query.params, db);
+ return [];
+ case 'delete':
+ await this.delete(meta, query.params, db);
+ return [];
+ case 'batch':
+ return [];
+ default: {
+ const _exhaustive: never = meta.action;
+ throw new Error(`Unsupported Postgres KV action: ${_exhaustive}`);
+ }
+ }
+ }
+
+ async executePagedQuery<T = Row, P extends CassandraParams = CassandraParams>(
+ query: PreparedQuery<P>,
+ options: {pageSize: number; pageState?: string | null},
+ ): Promise<{rows: Array<T>; pageState: string | null}> {
+ const state = decodePageState(options.pageState);
+ const rows = await this.executeQuery<T, P>(query);
+ const pageRows = rows.slice(state.offset, state.offset + options.pageSize);
+ const nextOffset = state.offset + pageRows.length;
+ return {
+ rows: pageRows,
+ pageState: nextOffset < rows.length ? encodePageState({offset: nextOffset}) : null,
+ };
+ }
+
+ async executeBatch(
+ queries: Array<{query: string; params: object; meta?: KvQueryMeta}>,
+ atomic = true,
+ ): Promise<void> {
+ if (atomic) {
+ await this.client.transaction(async (db) => {
+ for (const query of queries) {
+ await this.executeQuery({cql: query.query, params: query.params as CassandraParams, kvMeta: query.meta}, db);
+ }
+ });
+ return;
+ }
+ for (const query of queries) {
+ await this.executeQuery({cql: query.query, params: query.params as CassandraParams, kvMeta: query.meta});
+ }
+ }
+
+ private meta(query: PreparedQuery): KvQueryMeta<Row> {
+ const meta = (query.kvMeta ?? getKvMeta(query.cql) ?? parseRawMeta(query.cql)) as KvQueryMeta<Row> | null;
+ if (!meta) {
+ throw new Error(`Postgres KV does not understand query: ${query.cql}`);
+ }
+ return meta;
+ }
+
+ private async candidates(
+ meta: KvQueryMeta,
+ params: CassandraParams,
+ db: PostgresQueryable,
+ ): Promise<Array<StoredRow>> {
+ const rowKeys = fullRowKeysFromWhere(meta, params);
+ if (rowKeys) {
+ const result = await db.query<StoredRow>(
+ `SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND row_key = ANY($2::text[]) AND (expires_at IS NULL OR expires_at > now())`,
+ [meta.table.name, rowKeys],
+ );
+ return result.rows;
+ }
+ if (hasFullPartition(meta)) {
+ const result = await db.query<StoredRow>(
+ `SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND partition_key = $2 AND (expires_at IS NULL OR expires_at > now())`,
+ [meta.table.name, partitionKeyFromParams(meta, params)],
+ );
+ return result.rows;
+ }
+ const result = await db.query<StoredRow>(
+ `SELECT row_key, row_data FROM ${this.table} WHERE table_name = $1 AND (expires_at IS NULL OR expires_at > now())`,
+ [meta.table.name],
+ );
+ return result.rows;
+ }
+
+ private async select(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
+ let rows = (await this.candidates(meta, params, db))
+ .map((stored) => decodeRow(stored.row_data))
+ .filter((row) => matchesWhere(row, meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params));
+ rows = sortRows(meta, rows);
+ if (typeof meta.limit === 'number') rows = rows.slice(0, meta.limit);
+ return rows.map((row) => projectRow(row, meta.columns as ReadonlyArray<string> | undefined));
+ }
+
+ private async upsert(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<Array<Row>> {
+ const incoming = rowFromParams(meta, params);
+ const key = rowKey(meta, incoming);
+ const existing = await this.getRow(meta, key, db);
+ if (meta.ifNotExists && existing) {
+ return [{'[applied]': false}];
+ }
+ if (meta.ifNotExists) {
+ await db.query(
+ `DELETE FROM ${this.table} WHERE table_name = $1 AND row_key = $2 AND expires_at IS NOT NULL AND expires_at <= now()`,
+ [meta.table.name, key],
+ );
+ }
+ const next = {...(existing ?? {}), ...incoming};
+ const expiresAt = ttlExpiresAt(meta, params) ?? null;
+ const result = await db.query(
+ `INSERT INTO ${this.table} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
+VALUES ($1, $2, $3, $4::jsonb, $5, now())
+ON CONFLICT (table_name, row_key)
+DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = EXCLUDED.row_data, expires_at = EXCLUDED.expires_at, updated_at = now()
+WHERE NOT $6`,
+ [
+ meta.table.name,
+ partitionKey(meta, next),
+ key,
+ JSON.stringify(encodeRow(next)),
+ expiresAt,
+ meta.ifNotExists === true,
+ ],
+ );
+ if (meta.ifNotExists) {
+ return [{'[applied]': result.rowCount === 1}];
+ }
+ return [];
+ }
+
+ private async patch(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<void> {
+ const key = rowKeyFromParams(meta, params);
+ const stored = await this.getStoredRow(meta, key, db);
+ const base = stored?.row ?? paramsRow(params, (meta.pkColumns ?? meta.table.primaryKey) as ReadonlyArray<string>);
+ const next = {...base};
+ for (const column of meta.patchKeys ?? []) {
+ next[column] = column in params ? params[column] : null;
+ }
+ const ttl = ttlExpiresAt(meta, params);
+ const expiresAt = ttl === undefined ? (stored?.expiresAt ?? null) : ttl;
+ await db.query(
+ `INSERT INTO ${this.table} (table_name, partition_key, row_key, row_data, expires_at, updated_at)
+VALUES ($1, $2, $3, $4::jsonb, $5, now())
+ON CONFLICT (table_name, row_key)
+DO UPDATE SET partition_key = EXCLUDED.partition_key, row_data = EXCLUDED.row_data, expires_at = EXCLUDED.expires_at, updated_at = now()`,
+ [meta.table.name, partitionKey(meta, next), key, JSON.stringify(encodeRow(next)), expiresAt ?? null],
+ );
+ }
+
+ private async delete(meta: KvQueryMeta, params: CassandraParams, db: PostgresQueryable): Promise<void> {
+ const rows = await this.candidates(meta, params, db);
+ const matchingKeys = rows
+ .filter((stored) =>
+ matchesWhere(decodeRow(stored.row_data), meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params),
+ )
+ .map((stored) => stored.row_key);
+ if (matchingKeys.length === 0) return;
+ await db.query(`DELETE FROM ${this.table} WHERE table_name = $1 AND row_key = ANY($2::text[])`, [
+ meta.table.name,
+ matchingKeys,
+ ]);
+ }
+
+ private async getStoredRow(
+ meta: KvQueryMeta,
+ key: string,
+ db: PostgresQueryable,
+ ): Promise<{row: Row; expiresAt: Date | null} | null> {
+ const result = await db.query<StoredRow & {expires_at: Date | null}>(
+ `SELECT row_key, row_data, expires_at FROM ${this.table} WHERE table_name = $1 AND row_key = $2 AND (expires_at IS NULL OR expires_at > now()) LIMIT 1`,
+ [meta.table.name, key],
+ );
+ const row = result.rows[0];
+ return row ? {row: decodeRow(row.row_data), expiresAt: row.expires_at ?? null} : null;
+ }
+
+ private async getRow(meta: KvQueryMeta, key: string, db: PostgresQueryable): Promise<Row | null> {
+ return (await this.getStoredRow(meta, key, db))?.row ?? null;
+ }
+}