A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(kv): drop the row_key order probe that cliffed paged scans (#2306)
Hampus42df4f673144
diff --git a/fluxer_api/src/api/app/APILifecycle.ts b/fluxer_api/src/api/app/APILifecycle.tsindex 9baa77623..3707caa81 100644--- a/fluxer_api/src/api/app/APILifecycle.ts+++ b/fluxer_api/src/api/app/APILifecycle.ts@@ -134,12 +134,7 @@ export function createInitializer(config: APIConfig, logger: ILogger): () => ProsetInjectedWorkerService(new WorkerService(workerQueue, getSnowflakeService(), new JobLedgerRepository()));logger.info('JetStream worker service initialized');}- try {- await ensureDeletionQueueState(getKVAccountDeletionQueue(), logger);- } catch (error) {- logger.error({error}, 'Failed to verify KV deletion queue state');- throw error;- }+ await ensureDeletionQueueState(getKVAccountDeletionQueue(), logger);logger.info('Initializing search indexes...');let searchInitialized = false;try {diff --git a/fluxer_api/src/api/app/DeletionQueueStartup.ts b/fluxer_api/src/api/app/DeletionQueueStartup.tsindex ca313207d..8c7eeb3e1 100644--- a/fluxer_api/src/api/app/DeletionQueueStartup.ts+++ b/fluxer_api/src/api/app/DeletionQueueStartup.ts@@ -7,19 +7,23 @@ export async function ensureDeletionQueueState(deletionQueue: KVAccountDeletionQueueService,logger: ILogger,): Promise<void> {- if (!(await deletionQueue.needsRebuild())) {- logger.info('KV deletion queue state is healthy');- return;- }- const lockToken = await deletionQueue.acquireRebuildLock();- if (!lockToken) {- logger.info('Another instance is rebuilding the KV deletion queue, skipping');- return;- }- logger.info('KV deletion queue needs rebuild, rebuilding...');try {- await deletionQueue.rebuildState(lockToken);- } finally {- await deletionQueue.releaseRebuildLock(lockToken);+ if (!(await deletionQueue.needsRebuild())) {+ logger.info('KV deletion queue state is healthy');+ return;+ }+ const lockToken = await deletionQueue.acquireRebuildLock();+ if (!lockToken) {+ logger.info('Another instance is rebuilding the KV deletion queue, skipping');+ return;+ }+ logger.info('KV deletion queue needs rebuild, rebuilding...');+ try {+ await deletionQueue.rebuildState(lockToken);+ } finally {+ await deletionQueue.releaseRebuildLock(lockToken);+ }+ } catch (error) {+ logger.error({error}, 'KV deletion queue rebuild failed, continuing startup');}}diff --git a/fluxer_api/src/api/app/tests/DeletionQueueStartup.test.ts b/fluxer_api/src/api/app/tests/DeletionQueueStartup.test.tsindex b3be180ca..3d19bceb6 100644--- a/fluxer_api/src/api/app/tests/DeletionQueueStartup.test.ts+++ b/fluxer_api/src/api/app/tests/DeletionQueueStartup.test.ts@@ -72,6 +72,23 @@ describe('ensureDeletionQueueState', () => {expect(apiFailures).toEqual([]);});+ it('does not abort startup when the paged user scan fails', async () => {+ const kvClient = new MockKVProvider();+ let scans = 0;+ const repository = {+ async scanAllUsersPage() {+ scans += 1;+ throw new Error('paged user scan failed');+ },+ } as unknown as UserRepository;+ const queue = new KVAccountDeletionQueueService(kvClient, repository);++ await expect(ensureDeletionQueueState(queue, new NoopLogger())).resolves.toBeUndefined();++ expect(scans).toBe(1);+ expect(await queue.acquireRebuildLock()).not.toBeNull();+ });+it('rebuilds under the lock when no other instance holds it', async () => {const kvClient = new MockKVProvider();const queue = new KVAccountDeletionQueueService(diff --git a/fluxer_api/src/api/database/PostgresKvPagingAdversarial.test.ts b/fluxer_api/src/api/database/PostgresKvPagingAdversarial.test.tsindex e2921a06c..ef0928de8 100644--- a/fluxer_api/src/api/database/PostgresKvPagingAdversarial.test.ts+++ b/fluxer_api/src/api/database/PostgresKvPagingAdversarial.test.ts@@ -574,46 +574,60 @@ describe.skipIf(!dockerAvailable)('postgres kv paging adversarial', () => {expect(deltas, `paged order deltas:\n${deltas.join('\n')}`).toEqual([]);}, 300_000);- it('reads a page instead of the whole table for every page after the first', async () => {- await wipe(KV);- const rowCount = 40;- const pageSize = 4;- for (let i = 0; i < rowCount; i += 1) {- await upsert(next, FlatTable, {k: `key${String(i).padStart(2, '0')}`, v: `v${i}`});- }- const drain = async (build: (client: IPostgresClient) => AnyExec) => {+ it('pages a bigint scan at the same cost on both sides of a digit count boundary', async () => {+ const drain = async (ids: ReadonlyArray<bigint>) => {+ await wipe(KV);+ for (const id of ids) await upsert(next, FlatTable, {k: id, v: id.toString()});const counter = new CountingClient(raw, KV);- const exec = build(counter);- const reads: Array<number> = [];+ const exec = new PostgresKvQueryExecutor(counter);+ const seen: Array<string> = [];let pageState: string | null = null;- let seen = 0;for (let guard = 0; guard < 100; guard += 1) {- const before = counter.rowsRead;const page: {rows: Array<Row>; pageState: string | null} = await exec.executePagedQuery<Row>(- {cql: '__reads__', params: {}, kvMeta: selectMeta(FlatTable)},- {pageSize, pageState},+ {cql: '__digits__', params: {}, kvMeta: selectMeta(FlatTable)},+ {pageSize: 4, pageState},);- reads.push(counter.rowsRead - before);- seen += page.rows.length;+ for (const row of page.rows) seen.push(String(row.k));pageState = page.pageState;if (pageState === null) break;}- return {reads, seen};+ return {seen, rowsRead: counter.rowsRead};};- const legacyDrain = await drain((client) => new LegacyPostgresKvQueryExecutor(client));- const nextDrain = await drain((client) => new PostgresKvQueryExecutor(client));- const total = (reads: Array<number>) => reads.reduce((sum, count) => sum + count, 0);- expect(legacyDrain.seen).toBe(rowCount);- expect(nextDrain.seen).toBe(rowCount);- expect(nextDrain.reads.length).toBe(rowCount / pageSize);- expect(Math.min(...legacyDrain.reads), 'offset paging re-reads the whole table for every page').toBe(rowCount);+ const sameWidth = Array.from({length: 40}, (_, index) => 1_000_000_000_000_000_000n + BigInt(index));+ const straddling = Array.from({length: 40}, (_, index) => 999_999_999_999_999_980n + BigInt(index));+ const flat = await drain(sameWidth);+ const crossing = await drain(straddling);+ expect(flat.seen, 'same width scan order').toEqual(sameWidth.map(String));+ expect(crossing.seen, 'boundary crossing scan order').toEqual(straddling.map(String));expect(- Math.max(...nextDrain.reads.slice(1)),- `rows read per page: ${nextDrain.reads.join(',')}`,- ).toBeLessThanOrEqual(pageSize + 1);- expect(total(nextDrain.reads), `next=${total(nextDrain.reads)} legacy=${total(legacyDrain.reads)}`).toBeLessThan(- total(legacyDrain.reads) / 2,- );+ crossing.rowsRead,+ `same width read ${flat.rowsRead} rows, boundary crossing read ${crossing.rowsRead}`,+ ).toBe(flat.rowsRead);+ }, 300_000);++ it('keeps paging bigint keys across a digit count boundary while returned rows are deleted', async () => {+ await wipe(KV);+ const ids = Array.from({length: 30}, (_, index) => 999_999_999_999_999_985n + BigInt(index));+ for (const id of ids) await upsert(next, FlatTable, {k: id, v: id.toString()});+ const seen: Array<string> = [];+ let pageState: string | null = null;+ for (let guard = 0; guard < 100; guard += 1) {+ const page: {rows: Array<Row>; pageState: string | null} = await next.executePagedQuery<Row>(+ {cql: '__digitdrain__', params: {}, kvMeta: selectMeta(FlatTable)},+ {pageSize: 4, pageState},+ );+ for (const row of page.rows) seen.push(String(row.k));+ pageState = page.pageState;+ for (const row of page.rows) {+ await next.executeQuery({+ cql: '__digitdrain_delete__',+ params: {k: row.k} as CassandraParams,+ kvMeta: deleteMeta(FlatTable, [{kind: 'eq', col: 'k', param: 'k'} as WhereExpr<Row>]),+ });+ }+ if (pageState === null) break;+ }+ expect(seen, `returned order: ${seen.join(',')}`).toEqual(ids.map(String));}, 300_000);it('pages a prefix range whose keys sit adjacent to the range bounds', async () => {diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.tsindex 1e5aacd66..9d166714a 100644--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts@@ -19,7 +19,6 @@ interface StoredRow {interface PageState {offset: number;after?: string;- keyed?: boolean;}interface PageEntry {@@ -65,7 +64,6 @@ const VALUE_SEPARATOR = '\u001f';const KEY_RANGE_UPPER = ' ';const MAX_ROW_KEY_COMBINATIONS = 32_768;const MAX_PREFIX_RANGES = 256;-const PAGE_PROBE_BATCH = 10_000;const FULL_SCAN_LOG_INTERVAL_MS = 60_000;const FULL_SCAN_LOG_KEY_LIMIT = 1024;const ENCODED_TYPE_KEY = '__fluxer_type';@@ -671,7 +669,7 @@ function decodePageState(pageState: string | null | undefined): PageState {throw new Error('Invalid Postgres KV page state');}if (typeof decoded.after !== 'string') return {offset: decoded.offset};- return {offset: decoded.offset, after: decoded.after, keyed: decoded.keyed === true};+ return {offset: decoded.offset, after: decoded.after};}function pageableSelect(meta: KvQueryMeta, pageSize: number): boolean {@@ -684,13 +682,6 @@ function pageableSelect(meta: KvQueryMeta, pageSize: number): boolean {);}-function keysetCandidates(plan: QueryPlan): boolean {- return (- plan.exact &&- (plan.candidates.kind === 'none' || plan.candidates.kind === 'range' || plan.candidates.kind === 'scan')- );-}-function pageStart(meta: KvQueryMeta, entries: ReadonlyArray<PageEntry>, state: PageState): number {if (state.after === undefined) return state.offset;const index = entries.findIndex((entry) => entry.key === state.after);@@ -702,12 +693,6 @@ function pageStart(meta: KvQueryMeta, entries: ReadonlyArray<PageEntry>, state:return start;}-function pagedStatementName(prefix: string, plan: CandidatePlan, after: string | null): string | undefined {- const name = planStatementName(prefix, plan);- if (name === undefined) return undefined;- return after === null ? name : `${name}_after`;-}-function parseRawMeta(cql: string): KvQueryMeta<Row> | null {const normalized = normalizeCql(cql).replace(/;$/, '');const update =@@ -930,7 +915,8 @@ export class PostgresKvQueryExecutor {const state = decodePageState(options.pageState);const meta = this.meta(query);if (pageableSelect(meta, options.pageSize) && (state.after !== undefined || state.offset === 0)) {- const page = await this.selectPage(meta, query.params, state, options.pageSize);+ const plan = buildCandidatePlan(meta, query.params);+ const page = await this.sortedPage(meta, query.params, plan, state, options.pageSize);return {rows: page.rows as Array<T>, pageState: page.pageState};}const rows = await this.executeQuery<T, P>(query);@@ -942,93 +928,6 @@ export class PostgresKvQueryExecutor {};}- private async selectPage(- meta: KvQueryMeta,- params: CassandraParams,- state: PageState,- pageSize: number,- ): Promise<{rows: Array<Row>; pageState: string | null}> {- const plan = buildCandidatePlan(meta, params);- if (keysetCandidates(plan)) {- if (state.keyed === true) {- return this.keysetPage(meta, params, plan, state.after ?? null, state.offset, pageSize);- }- if (state.after === undefined && (await this.rowKeyOrderMatchesSort(meta, plan))) {- return this.keysetPage(meta, params, plan, null, 0, pageSize);- }- }- return this.sortedPage(meta, params, plan, state, pageSize);- }-- private pagedSql(- meta: KvQueryMeta,- fragments: PlanFragments,- projection: string,- after: string | null,- limit: number,- ): {text: string; values: Array<unknown>} {- const values: Array<unknown> = [meta.table.name, ...fragments.params];- let text = `SELECT ${projection} FROM ${this.table} kv WHERE kv.table_name = $1${fragments.predicate}`;- if (after !== null) {- values.push(after);- text += ` AND kv.row_key COLLATE "C" > $${values.length}`;- }- text += ' AND (kv.expires_at IS NULL OR kv.expires_at > now()) ORDER BY kv.row_key COLLATE "C"';- values.push(limit);- return {text: `${text} LIMIT $${values.length}`, values};- }-- private async rowKeyOrderMatchesSort(meta: KvQueryMeta, plan: QueryPlan): Promise<boolean> {- if (plan.candidates.kind === 'none') return true;- const fragments = planFragmentGroups(plan.candidates)[0]!;- const columns = (meta.table.primaryKey as ReadonlyArray<string>).length;- let after: string | null = null;- let previous: Array<unknown> | null = null;- for (;;) {- const sql = this.pagedSql(meta, fragments, 'kv.row_key', after, PAGE_PROBE_BATCH);- const name = pagedStatementName('kv_keys', plan.candidates, after);- const batch: Array<{row_key: string}> = (await this.client.query<{row_key: string}>(sql.text, sql.values, name))- .rows;- for (const stored of batch) {- const values = decodeRowKey(stored.row_key, columns);- if (values === null) return false;- if (previous !== null && compareKeyValues(previous, values) > 0) return false;- previous = values;- }- if (batch.length < PAGE_PROBE_BATCH) return true;- after = batch[batch.length - 1]!.row_key;- }- }-- private async keysetPage(- meta: KvQueryMeta,- params: CassandraParams,- plan: QueryPlan,- after: string | null,- offset: number,- pageSize: number,- ): Promise<{rows: Array<Row>; pageState: string | null}> {- if (plan.candidates.kind === 'none') return {rows: [], pageState: null};- if (plan.candidates.kind === 'scan') logFullScan(meta);- const fragments = planFragmentGroups(plan.candidates)[0]!;- const sql = this.pagedSql(meta, fragments, 'kv.row_key, kv.row_data', after, pageSize + 1);- const result = await this.client.query<StoredRow>(- sql.text,- sql.values,- pagedStatementName('kv_page', plan.candidates, after),- );- const stored = result.rows.slice(0, pageSize);- const entries = this.matchingEntries(meta, stored, params);- const last = entries[entries.length - 1];- return {- rows: entries.map((entry) => projectRow(entry.row, meta.columns as ReadonlyArray<string> | undefined)),- pageState:- result.rows.length > pageSize && last- ? encodePageState({offset: offset + entries.length, after: last.key, keyed: true})- : null,- };- }-private async sortedPage(meta: KvQueryMeta,params: CassandraParams,