A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(api): page postgres kv full scans by keyset (#3465)
Hampuse8dff6a8a167
diff --git a/fluxer_api/src/api/database/PostgresKvKeysetScan.test.ts b/fluxer_api/src/api/database/PostgresKvKeysetScan.test.tsnew file mode 100644index 000000000..89ba1fb45--- /dev/null+++ b/fluxer_api/src/api/database/PostgresKvKeysetScan.test.ts@@ -0,0 +1,334 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {defineTable} from '@app/api/database/CassandraTableDsl';+import type {PreparedQuery} from '@app/api/database/CassandraTypes';+import {keyFromColumns, PostgresKvQueryExecutor} from '@app/api/database/PostgresKvQueryExecutor';+import {dockerAvailable, startPostgresKv, stopPostgresKv} from '@app/api/test/DockerTestContainer';+import type {IPostgresClient} from '@pkgs/postgres/src/Client';+import {afterAll, beforeAll, beforeEach, describe, expect, it} from 'vitest';++const KV_TABLE = 'kv_keyset_scan';++type Row = Record<string, unknown>;++interface UserRow {+ user_id: bigint;+ name: string;+ kind: number;+}++interface BucketRow {+ channel_id: bigint;+ bucket: number;+ note: string;+}++interface NameRow {+ name: string;+ value: number;+}++const KeysetUsers = defineTable<UserRow, 'user_id'>({+ name: 'keyset_users',+ columns: ['user_id', 'name', 'kind'],+ primaryKey: ['user_id'],+});++const KeysetBuckets = defineTable<BucketRow, 'channel_id' | 'bucket', 'channel_id'>({+ name: 'keyset_buckets',+ columns: ['channel_id', 'bucket', 'note'],+ primaryKey: ['channel_id', 'bucket'],+ partitionKey: ['channel_id'],+});++const KeysetNames = defineTable<NameRow, 'name'>({+ name: 'keyset_names',+ columns: ['name', 'value'],+ primaryKey: ['name'],+});++interface Statement {+ text: string;+ rows: number;+}++function recordingClient(client: IPostgresClient, statements: Array<Statement>): IPostgresClient {+ return {+ query: (async (text: string, values?: Array<unknown>, name?: string) => {+ const result = await client.query(text, values, name);+ statements.push({text, rows: result.rows.length});+ return result;+ }) as IPostgresClient['query'],+ connect: () => client.connect(),+ shutdown: () => client.shutdown(),+ transaction: (fn) => client.transaction(fn),+ kvTable: () => client.kvTable(),+ };+}++function pageState(state: Row): string {+ return Buffer.from(JSON.stringify(state)).toString('base64url');+}++function readPageState(state: string): Row {+ return JSON.parse(Buffer.from(state, 'base64url').toString('utf8')) as Row;+}++function userId(index: number): bigint {+ return 1_000_000_000_000_000_000n + BigInt(index) * 7_919n;+}++describe.skipIf(!dockerAvailable)('Postgres KV keyset scans', () => {+ let raw: IPostgresClient;+ let statements: Array<Statement>;+ let executor: PostgresKvQueryExecutor;+ let postgresContainer: string;++ beforeAll(async () => {+ const started = await startPostgresKv({namePrefix: 'fluxer-kvkeyset', kvTable: KV_TABLE, maxConnections: 4});+ postgresContainer = started.container;+ raw = started.client;+ }, 900_000);++ beforeEach(async () => {+ await raw.query(`DELETE FROM ${KV_TABLE}`);+ statements = [];+ executor = new PostgresKvQueryExecutor(recordingClient(raw, statements));+ });++ afterAll(async () => {+ await stopPostgresKv(postgresContainer);+ });++ async function seed(queries: ReadonlyArray<PreparedQuery>): Promise<void> {+ await executor.executeBatch(queries.map((query) => ({query: query.cql, params: query.params, meta: query.kvMeta})));+ }++ async function seedUsers(count: number): Promise<Array<UserRow>> {+ const rows = Array.from({length: count}, (_, index) => ({+ user_id: userId((index * 37) % count),+ name: `user-${index}`,+ kind: index % 7,+ }));+ await seed(rows.map((row) => KeysetUsers.insert(row)));+ return rows;+ }++ async function seedBuckets(channels: number, buckets: number): Promise<void> {+ const queries: Array<PreparedQuery> = [];+ for (let channel = 0; channel < channels; channel += 1) {+ for (let bucket = 0; bucket < buckets; bucket += 1) {+ queries.push(+ KeysetBuckets.insert({+ channel_id: userId((channel * 13) % channels),+ bucket: bucket * 11 - 20,+ note: `note-${(channel + bucket) % 5}`,+ }),+ );+ }+ }+ await seed(queries);+ }++ async function oracle(query: PreparedQuery): Promise<Array<Row>> {+ const rows = await executor.executeQuery<Row>(query);+ statements.length = 0;+ return rows;+ }++ async function walk(+ query: PreparedQuery,+ pageSize: number,+ start: string | null = null,+ ): Promise<{pages: Array<Array<Row>>; states: Array<string>}> {+ const pages: Array<Array<Row>> = [];+ const states: Array<string> = [];+ let state = start;+ do {+ const page = await executor.executePagedQuery<Row>(query, {pageSize, pageState: state});+ pages.push(page.rows);+ state = page.pageState;+ if (state !== null) states.push(state);+ expect(pages.length).toBeLessThan(10_000);+ } while (state !== null);+ return {pages, states};+ }++ function expectBounded(batch: number): void {+ expect(statements.length).toBeGreaterThan(0);+ for (const statement of statements) {+ expect(statement.rows).toBeLessThanOrEqual(batch);+ if (statement.text.includes('row_data FROM')) expect(statement.text).toMatch(/LIMIT \$2$/);+ }+ }++ function byKey(columns: ReadonlyArray<string>, rows: ReadonlyArray<Row>): Array<Row> {+ return [...rows].sort((left, right) => {+ const l = keyFromColumns(columns, left);+ const r = keyFromColumns(columns, right);+ return l < r ? -1 : l > r ? 1 : 0;+ });+ }++ it('walks a numeric single key table in the same order without a full load', async () => {+ await seedUsers(250);+ const query = KeysetUsers.select({}).bind({});+ const expected = await oracle(query);+ expect(expected).toHaveLength(250);+ const {pages, states} = await walk(query, 40);+ expect(pages.flat()).toEqual(expected);+ expect(pages.slice(0, -1).every((page) => page.length === 40)).toBe(true);+ expect(readPageState(states[0]!)).toEqual({+ offset: 40,+ after: keyFromColumns(['user_id'], expected[39]!),+ keyed: true,+ });+ expectBounded(41);+ });++ it('walks a composite key table by row key without reloading it per page', async () => {+ await seedBuckets(30, 7);+ const query = KeysetBuckets.select({}).bind({});+ const expected = await oracle(query);+ expect(expected).toHaveLength(210);+ const {pages} = await walk(query, 25);+ const rows = pages.flat();+ expect(byKey(['channel_id', 'bucket'], rows)).toEqual(byKey(['channel_id', 'bucket'], expected));+ expect(new Set(rows.map((row) => keyFromColumns(['channel_id', 'bucket'], row))).size).toBe(210);+ const channelRuns = rows.filter((row, index) => index === 0 || rows[index - 1]!.channel_id !== row.channel_id);+ expect(channelRuns).toHaveLength(30);+ expect(statements.filter((statement) => statement.text.includes('row_data FROM'))).toHaveLength(pages.length);+ expectBounded(26);+ });++ it('walks a text key table in the same order', async () => {+ await seed(+ Array.from({length: 90}, (_, index) => KeysetNames.insert({name: `name-${(index * 31) % 90}-x`, value: index})),+ );+ const query = KeysetNames.select({}).bind({});+ const expected = await oracle(query);+ const {pages} = await walk(query, 16);+ expect(pages.flat()).toEqual(expected);+ expectBounded(17);+ });++ it('fills filtered pages from bounded batches on numeric and composite tables', async () => {+ await seedUsers(300);+ await seedBuckets(20, 9);+ const users = KeysetUsers.select({where: KeysetUsers.where.eq('kind')}).bind({kind: 3});+ const expectedUsers = await oracle(users);+ const userWalk = await walk(users, 10);+ expect(userWalk.pages.flat()).toEqual(expectedUsers);+ expect(userWalk.pages.slice(0, -1).every((page) => page.length === 10)).toBe(true);+ expectBounded(11);+ const buckets = KeysetBuckets.select({where: KeysetBuckets.where.eq('note')}).bind({note: 'note-2'});+ const expectedBuckets = await oracle(buckets);+ const bucketWalk = await walk(buckets, 6);+ expect(byKey(['channel_id', 'bucket'], bucketWalk.pages.flat())).toEqual(+ byKey(['channel_id', 'bucket'], expectedBuckets),+ );+ expect(bucketWalk.pages.slice(0, -1).every((page) => page.length === 6)).toBe(true);+ expectBounded(7);+ });++ it('skips expired rows', async () => {+ await seedUsers(60);+ await seedBuckets(6, 5);+ await raw.query(+ `UPDATE ${KV_TABLE} SET expires_at = now() - interval '1 minute' WHERE (row_data ->> 'kind')::int = 2 OR row_data ->> 'note' = 'note-1'`,+ );+ const users = KeysetUsers.select({}).bind({});+ const expectedUsers = await oracle(users);+ expect(expectedUsers.length).toBeLessThan(60);+ expect((await walk(users, 7)).pages.flat()).toEqual(expectedUsers);+ const buckets = KeysetBuckets.select({}).bind({});+ const expectedBuckets = await oracle(buckets);+ expect(expectedBuckets.length).toBeLessThan(30);+ expect(byKey(['channel_id', 'bucket'], (await walk(buckets, 7)).pages.flat())).toEqual(+ byKey(['channel_id', 'bucket'], expectedBuckets),+ );+ expectBounded(8);+ });++ it('returns one empty page for an empty table', async () => {+ for (const query of [KeysetUsers.select({}).bind({}), KeysetBuckets.select({}).bind({})]) {+ expect(await executor.executePagedQuery(query, {pageSize: 10})).toEqual({rows: [], pageState: null});+ }+ expectBounded(1);+ });++ it('never drops rows with non-numeric keys from a single key table', async () => {+ await seedUsers(40);+ await executor.executeQuery(KeysetUsers.insert({user_id: 'stray' as unknown as bigint, name: 'stray', kind: 0}));+ const {pages} = await walk(KeysetUsers.select({}).bind({}), 9);+ const names = pages.flat().map((row) => row.name);+ expect(names).toHaveLength(41);+ expect(new Set(names).size).toBe(41);+ expect(names).toContain('stray');+ expectBounded(10);+ });++ it('resumes every page state shape the previous release produced', async () => {+ await seedUsers(120);+ await seedBuckets(12, 6);+ const users = KeysetUsers.select({}).bind({});+ const expectedUsers = await oracle(users);+ const userKey = keyFromColumns(['user_id'], expectedUsers[49]!);+ for (const state of [{offset: 50, after: userKey, keyed: true}, {offset: 50, after: userKey}, {offset: 50}]) {+ expect((await walk(users, 15, pageState(state))).pages.flat()).toEqual(expectedUsers.slice(50));+ }+ const buckets = KeysetBuckets.select({}).bind({});+ const expectedBuckets = await oracle(buckets);+ const bucketKey = keyFromColumns(['channel_id', 'bucket'], expectedBuckets[29]!);+ statements.length = 0;+ const resumed = await walk(buckets, 11, pageState({offset: 30, after: bucketKey}));+ expect(byKey(['channel_id', 'bucket'], resumed.pages.flat())).toEqual(+ byKey(['channel_id', 'bucket'], expectedBuckets.slice(30)),+ );+ expect(readPageState(resumed.states[0]!)).toMatchObject({byRowKey: true, since: bucketKey});+ expectBounded(12);+ });++ it('streams limited scans in key order instead of loading the table', async () => {+ const seeded = await seedUsers(200);+ const sorted = [...seeded].sort((left, right) => (left.user_id < right.user_id ? -1 : 1));+ const first = KeysetUsers.select({limit: 20}).bind({});+ const next = KeysetUsers.select({where: KeysetUsers.where.tokenGt('user_id', 'last_user_id'), limit: 20});+ const expectedFirst = await oracle(KeysetUsers.select({}).bind({}));+ expect(await executor.executeQuery(first)).toEqual(expectedFirst.slice(0, 20));+ expectBounded(21);+ statements.length = 0;+ const rows = await executor.executeQuery(next.bind({last_user_id: sorted[99]!.user_id}));+ expect(rows).toEqual(expectedFirst.slice(100, 120));+ expect(statements.filter((statement) => statement.text.includes('row_data FROM'))).toHaveLength(1);+ expectBounded(21);+ statements.length = 0;+ expect(await executor.executeQuery(next.bind({last_user_id: sorted[195]!.user_id}))).toEqual(+ expectedFirst.slice(196),+ );+ expectBounded(21);+ });++ it('probes for non-numeric keys through the primary key while other tables hold string keys', async () => {+ await raw.query(+ `INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data) SELECT 'keyset_users', k, k, '{}'::jsonb FROM generate_series(1, 20000) g, LATERAL (SELECT '{"__fluxer_type":"bigint","value":"' || g || '"}' AS k) s`,+ );+ await raw.query(+ `INSERT INTO ${KV_TABLE} (table_name, partition_key, row_key, row_data) SELECT 'keyset_strings', g::text, '"' || lpad(g::text, 8, '0') || '"', '{}'::jsonb FROM generate_series(1, 200000) g`,+ );+ await raw.query(`ANALYZE ${KV_TABLE}`);+ await executor.executePagedQuery(KeysetUsers.select({}).bind({}), {pageSize: 10});+ const probe = statements.find((statement) => statement.text.endsWith('AS found'));+ expect(probe).toBeDefined();+ const plan = await raw.query<{'QUERY PLAN': string}>(`EXPLAIN (COSTS OFF) ${probe!.text}`, ['keyset_users']);+ expect(plan.rows.map((row) => row['QUERY PLAN']).join('\n')).not.toContain('Seq Scan');+ });++ it('keeps the sorted load when a limited scan asks for an order the key walk cannot give', async () => {+ await seedUsers(30);+ const descending = KeysetUsers.select({orderBy: {col: 'user_id', direction: 'DESC'}, limit: 5}).bind({});+ const rows = await executor.executeQuery<Row>(descending);+ const all = await oracle(KeysetUsers.select({}).bind({}));+ expect(rows).toEqual(all.slice(-5).reverse());+ });+});diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.tsindex 41f976183..271de7516 100644--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts@@ -28,6 +28,19 @@ interface PageState {offset: number;after?: string;keyed?: boolean;+ byRowKey?: boolean;+ since?: string;+}++interface ScanWalk {+ numeric: boolean;+ after: string | null;+ since: string | null;+}++interface NumericFloor {+ value: string;+ inclusive: boolean;}interface PageEntry {@@ -90,6 +103,12 @@ const EXPIRED_STORED_ROW = 'kv.expires_at IS NOT NULL AND kv.expires_at <= now()const MERGED_ROW_DATA = `CASE WHEN ${EXPIRED_STORED_ROW} THEN EXCLUDED.row_data ELSE kv.row_data || EXCLUDED.row_data END`;const KEPT_EXPIRES_AT = `CASE WHEN ${EXPIRED_STORED_ROW} THEN NULL ELSE kv.expires_at END`;const NO_EXPIRY = 'infinity';+const BIGINT_ROW_KEY_PREFIX = `{"${ENCODED_TYPE_KEY}":"bigint","value":"`;+const NON_NUMERIC_ROW_KEY_RANGES = [+ `kv.row_key COLLATE "C" < '-'`,+ `kv.row_key COLLATE "C" >= ':' AND kv.row_key COLLATE "C" < '${BIGINT_ROW_KEY_PREFIX}'`,+ `kv.row_key COLLATE "C" >= '${BIGINT_ROW_KEY_PREFIX}:'`,+];export async function postgresKvPassIsFresh(client: IPostgresClient,@@ -676,15 +695,18 @@ function logError(details: Record<string, unknown>, message: string): void {const fullScanLoggedAt = new Map<string, number>();-function logFullScan(meta: KvQueryMeta): void {+function logFullScan(meta: KvQueryMeta, walk?: 'numeric' | 'row_key'): void {const shape = queryShape(meta);- const key = `${meta.table.name}|${meta.action}|${shape.summary}`;+ const key = `${meta.table.name}|${meta.action}|${shape.summary}|${walk ?? 'load'}`;const now = Date.now();const last = fullScanLoggedAt.get(key);if (last !== undefined && now - last < FULL_SCAN_LOG_INTERVAL_MS) return;if (fullScanLoggedAt.size >= FULL_SCAN_LOG_KEY_LIMIT) fullScanLoggedAt.clear();fullScanLoggedAt.set(key, now);- logWarn({table: meta.table.name, action: meta.action, where: shape.summary || 'none'}, 'Postgres KV full table scan');+ logWarn(+ {table: meta.table.name, action: meta.action, where: shape.summary || 'none', walk: walk ?? 'load'},+ 'Postgres KV full table scan',+ );}function ttlExpiresAt(meta: KvQueryMeta, params: CassandraParams): Date | typeof NO_EXPIRY | null | undefined {@@ -715,7 +737,13 @@ function decodePageState(pageState: string | null | undefined): PageState {throw new Error('Invalid Postgres KV page state');}if (typeof decoded.after !== 'string') return {offset: decoded.offset};- return {offset: decoded.offset, after: decoded.after, keyed: decoded.keyed === true};+ return {+ offset: decoded.offset,+ after: decoded.after,+ keyed: decoded.keyed === true,+ byRowKey: decoded.byRowKey === true,+ ...(typeof decoded.since === 'string' ? {since: decoded.since} : {}),+ };}function pageableSelect(meta: KvQueryMeta, pageSize: number): boolean {@@ -734,22 +762,30 @@ function numericKeyValue(value: unknown): string | null {return null;}-function numericScanPlan(meta: KvQueryMeta, plan: QueryPlan): boolean {- return plan.exact && plan.candidates.kind === 'scan' && (meta.table.primaryKey as ReadonlyArray<string>).length === 1;+function numericRowKeyValue(key: string): string | null {+ const values = decodeRowKey(key, 1);+ return values === null ? null : numericKeyValue(values[0]);}-function numericScanKeyed(meta: KvQueryMeta, plan: QueryPlan, entries: ReadonlyArray<PageEntry>): boolean {- if (!numericScanPlan(meta, plan)) return false;- const column = (meta.table.primaryKey as ReadonlyArray<string>)[0]!;- return entries.every((entry) => numericKeyValue(entry.row[column]) !== null);+function singlePrimaryKey(meta: KvQueryMeta): boolean {+ return (meta.table.primaryKey as ReadonlyArray<string>).length === 1;}-function numericScanCursor(state: PageState): {rowKey: string; value: string} | null {- if (state.keyed !== true || state.after === undefined) return null;- const values = decodeRowKey(state.after, 1);- if (values === null) return null;- const value = numericKeyValue(values[0]);- return value === null ? null : {rowKey: state.after, value};+function walksInPrimaryKeyOrder(meta: KvQueryMeta): boolean {+ if (!singlePrimaryKey(meta)) return false;+ if (meta.orderBy === undefined) return true;+ return meta.orderBy.col === meta.table.primaryKey[0] && meta.orderBy.direction !== 'DESC';+}++function numericFloor(meta: KvQueryMeta, params: CassandraParams): NumericFloor | null {+ const column = meta.table.primaryKey[0];+ for (const clause of whereClauses(meta)) {+ if (clause.col !== column) continue;+ if (clause.kind !== 'gt' && clause.kind !== 'tokenGt' && clause.kind !== 'gte') continue;+ const value = numericKeyValue(getParam(params, clause.param));+ if (value !== null) return {value, inclusive: clause.kind === 'gte'};+ }+ return null;}function pageStart(meta: KvQueryMeta, entries: ReadonlyArray<PageEntry>, state: PageState): number {@@ -1018,10 +1054,10 @@ export class PostgresKvQueryExecutor {const meta = this.meta(query);if (pageableSelect(meta, options.pageSize) && (state.after !== undefined || state.offset === 0)) {const plan = buildCandidatePlan(meta, query.params);- const cursor = numericScanPlan(meta, plan) ? numericScanCursor(state) : null;- const page = cursor- ? await this.numericScanPage(meta, query.params, cursor, state.offset, options.pageSize)- : await this.sortedPage(meta, query.params, plan, state, options.pageSize);+ const page =+ plan.candidates.kind === 'scan'+ ? await this.scanPage(meta, query.params, await this.scanWalk(meta, state), state.offset, options.pageSize)+ : await this.sortedPage(meta, query.params, plan, state, options.pageSize);return {rows: page.rows as Array<T>, pageState: page.pageState};}const rows = await this.executeQuery<T, P>(query);@@ -1033,29 +1069,111 @@ export class PostgresKvQueryExecutor {};}- private async numericScanPage(+ private async scanWalk(meta: KvQueryMeta, state: PageState): Promise<ScanWalk> {+ if (state.byRowKey) return {numeric: false, after: state.after ?? null, since: state.since ?? null};+ const after = state.after ?? null;+ const numericAfter = after === null || numericRowKeyValue(after) !== null;+ if (state.keyed && after !== null && numericAfter && singlePrimaryKey(meta)) {+ return {numeric: true, after, since: null};+ }+ const numeric = singlePrimaryKey(meta) && !(await this.hasNonNumericRows(meta, this.client));+ if (numeric && numericAfter) return {numeric: true, after, since: null};+ return {numeric: false, after: null, since: after};+ }++ private async hasNonNumericRows(meta: KvQueryMeta, db: PostgresQueryable): Promise<boolean> {+ const probes = NON_NUMERIC_ROW_KEY_RANGES.map(+ (range) =>+ `(SELECT kv.row_key FROM ${this.table} kv WHERE kv.table_name = $1 AND ${range} AND (kv.expires_at IS NULL OR kv.expires_at > now()) AND ${NUMERIC_ROW_KEY} IS NULL ORDER BY kv.row_key COLLATE "C" LIMIT 1) IS NOT NULL`,+ );+ const result = await db.query<{found: boolean}>(`SELECT ${probes.join(' OR ')} AS found`, [meta.table.name]);+ return result.rows[0]?.found === true;+ }++ private async scanPage(meta: KvQueryMeta,params: CassandraParams,- cursor: {rowKey: string; value: string},+ walk: ScanWalk,offset: number,pageSize: number,): Promise<{rows: Array<Row>; pageState: string | null}> {- logFullScan(meta);- const result = await this.client.query<StoredRow>(- `SELECT kv.row_key, kv.row_data FROM ${this.table} kv WHERE kv.table_name = $1 AND ${NUMERIC_ROW_KEY} IS NOT NULL AND (${NUMERIC_ROW_KEY}, kv.row_key COLLATE "C") > ($2::numeric, $3) AND (kv.expires_at IS NULL OR kv.expires_at > now()) ORDER BY ${NUMERIC_ROW_KEY}, kv.row_key COLLATE "C" LIMIT $4`,- [meta.table.name, cursor.value, cursor.rowKey, pageSize + 1],- );- const entries = this.matchingEntries(meta, result.rows.slice(0, pageSize), params);- const last = entries[entries.length - 1];+ const page = await this.walkScan(meta, params, walk, pageSize, this.client);+ if (!page.more || page.after === null) return {rows: this.projected(meta, page.entries), pageState: null};+ const position = {offset: offset + page.entries.length, after: page.after};return {- rows: this.projected(meta, entries),- pageState:- result.rows.length > pageSize && last- ? encodePageState({offset: offset + entries.length, after: last.key, keyed: true})- : null,+ rows: this.projected(meta, page.entries),+ pageState: encodePageState(+ walk.numeric+ ? {...position, keyed: true}+ : {...position, byRowKey: true, ...(walk.since === null ? {} : {since: walk.since})},+ ),};}+ private async walkScan(+ meta: KvQueryMeta,+ params: CassandraParams,+ walk: ScanWalk,+ want: number,+ db: PostgresQueryable,+ ): Promise<{entries: Array<PageEntry>; after: string | null; more: boolean}> {+ logFullScan(meta, walk.numeric ? 'numeric' : 'row_key');+ const floor = walk.numeric ? numericFloor(meta, params) : null;+ const since = walk.since === null ? null : decodeRowKey(walk.since, meta.table.primaryKey.length);+ if (walk.since !== null && since === null) throw new Error('Invalid Postgres KV page state');+ const limit = want + 1;+ const entries: Array<PageEntry> = [];+ let after = walk.after;+ while (entries.length < want) {+ const batch = await this.scanBatch(meta, walk.numeric, floor, after, limit, db);+ for (let index = 0; index < batch.length; index += 1) {+ const stored = batch[index]!;+ after = stored.row_key;+ const entry = this.matchingEntry(meta, stored, params);+ if (entry && (since === null || compareRowToKeyValues(meta, entry.row, since) > 0)) entries.push(entry);+ if (entries.length === want) {+ return {entries, after, more: index < batch.length - 1 || batch.length === limit};+ }+ }+ if (batch.length < limit) return {entries, after, more: false};+ }+ return {entries, after, more: false};+ }++ private async scanBatch(+ meta: KvQueryMeta,+ numeric: boolean,+ floor: NumericFloor | null,+ after: string | null,+ limit: number,+ db: PostgresQueryable,+ ): Promise<Array<StoredRow>> {+ const params: Array<unknown> = [meta.table.name, limit];+ let predicate = '';+ if (numeric) {+ predicate = ` AND ${NUMERIC_ROW_KEY} IS NOT NULL`;+ if (floor) {+ params.push(floor.value);+ predicate += ` AND ${NUMERIC_ROW_KEY} ${floor.inclusive ? '>=' : '>'} $${params.length}::numeric`;+ }+ if (after !== null) {+ const value = numericRowKeyValue(after);+ if (value === null) throw new Error('Invalid Postgres KV page state');+ params.push(value, after);+ predicate += ` AND (${NUMERIC_ROW_KEY}, kv.row_key COLLATE "C") > ($${params.length - 1}::numeric, $${params.length})`;+ }+ } else if (after !== null) {+ params.push(after);+ predicate = ` AND kv.row_key COLLATE "C" > $${params.length}`;+ }+ const order = numeric ? `${NUMERIC_ROW_KEY}, kv.row_key COLLATE "C"` : 'kv.row_key COLLATE "C"';+ const result = await db.query<StoredRow>(+ `SELECT kv.row_key, kv.row_data FROM ${this.table} kv WHERE kv.table_name = $1${predicate} AND (kv.expires_at IS NULL OR kv.expires_at > now()) ORDER BY ${order} LIMIT $2`,+ params,+ );+ return result.rows;+ }+private async sortedPage(meta: KvQueryMeta,params: CassandraParams,@@ -1071,12 +1189,9 @@ export class PostgresKvQueryExecutor {const last = page[page.length - 1];const nextOffset = start + page.length;if (nextOffset >= entries.length || !last) return {rows: this.projected(meta, page), pageState: null};- const keyed = numericScanKeyed(meta, plan, entries);return {rows: this.projected(meta, page),- pageState: encodePageState(- keyed ? {offset: nextOffset, after: last.key, keyed: true} : {offset: nextOffset, after: last.key},- ),+ pageState: encodePageState({offset: nextOffset, after: last.key}),};}@@ -1146,17 +1261,22 @@ export class PostgresKvQueryExecutor {stored: ReadonlyArray<StoredRow>,params: CassandraParams,): Array<PageEntry> {- const required = queryShape(meta).requiredColumns;const entries: Array<PageEntry> = [];for (const entry of stored) {- const row = required ? decodeRowColumns(entry.row_data, required) : decodeRow(entry.row_data);- if (matchesWhere(row, meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params)) {- entries.push({key: entry.row_key, row});- }+ const matched = this.matchingEntry(meta, entry, params);+ if (matched) entries.push(matched);}return entries;}+ private matchingEntry(meta: KvQueryMeta, stored: StoredRow, params: CassandraParams): PageEntry | null {+ const required = queryShape(meta).requiredColumns;+ const row = required ? decodeRowColumns(stored.row_data, required) : decodeRow(stored.row_data);+ return matchesWhere(row, meta.where as ReadonlyArray<WhereExpr<Row>> | undefined, params)+ ? {key: stored.row_key, row}+ : null;+ }+private matchingRows(meta: KvQueryMeta, stored: ReadonlyArray<StoredRow>, params: CassandraParams): Array<Row> {return this.matchingEntries(meta, stored, params).map((entry) => entry.row);}@@ -1185,6 +1305,16 @@ export class PostgresKvQueryExecutor {) {return this.unorderedScan(meta, meta.limit, db);}+ if (+ typeof meta.limit === 'number' &&+ Number.isInteger(meta.limit) &&+ plan.candidates.kind === 'scan' &&+ walksInPrimaryKeyOrder(meta) &&+ !(await this.hasNonNumericRows(meta, db))+ ) {+ const walk = {numeric: true, after: null, since: null};+ return this.projected(meta, (await this.walkScan(meta, params, walk, meta.limit, db)).entries);+ }let rows = this.matchingRows(meta, await this.candidates(meta, plan, db), params);rows = sortRows(meta, rows);if (typeof meta.limit === 'number') rows = rows.slice(0, meta.limit);diff --git a/fluxer_api/src/api/database/PostgresKvUnorderedScan.test.ts b/fluxer_api/src/api/database/PostgresKvUnorderedScan.test.tsindex 558060c46..b870364de 100644--- a/fluxer_api/src/api/database/PostgresKvUnorderedScan.test.ts+++ b/fluxer_api/src/api/database/PostgresKvUnorderedScan.test.ts@@ -55,15 +55,15 @@ describe('Postgres KV unordered scan', () => {expect(rows).toEqual([{user_id: '2'}, {user_id: '1'}]);});- it('keeps the sorted full scan when order is asked for', async () => {+ it('keeps the sorted full scan when the order cannot be walked by key', async () => {const statements: Array<{text: string; values: Array<unknown> | undefined}> = [];const executor = new PostgresKvQueryExecutor(recordingClient(statements));const rows = await executor.executeQuery({cql: '__ordered_probe',params: {},- kvMeta: selectMeta({limit: 1}),+ kvMeta: selectMeta({limit: 1, orderBy: {col: 'user_id', direction: 'DESC'}}),});expect(statements[0]!.text).not.toContain('LIMIT');- expect(rows).toEqual([{user_id: '1'}]);+ expect(rows).toEqual([{user_id: '2'}]);});});