A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(kv): mark the messages migration done instead of rescanning (#2221)
Hampus0a920def2b51
diff --git a/fluxer_api/src/api/database/PostgresKvQueryDifferential.test.ts b/fluxer_api/src/api/database/PostgresKvQueryDifferential.test.tsindex 41bc3f861..164c22066 100644--- a/fluxer_api/src/api/database/PostgresKvQueryDifferential.test.ts+++ b/fluxer_api/src/api/database/PostgresKvQueryDifferential.test.ts@@ -1082,8 +1082,8 @@ suite('PostgresKvQueryExecutor differential', () => {);await raw.query(`CREATE TABLE ${SEED_TABLE} AS TABLE ${LEGACY_TABLE}`);await raw.query(`CREATE INDEX ${SEED_TABLE}_table_idx ON ${SEED_TABLE} (table_name)`);- await raw.query(`TRUNCATE ${LEGACY_TABLE}`);for (const kv of [LEGACY_TABLE, NEXT_TABLE, ICU_TABLE]) {+ await raw.query(`TRUNCATE ${kv}`);await raw.query(`INSERT INTO ${kv} SELECT * FROM ${SEED_TABLE} ORDER BY table_name, row_key COLLATE "C"`);}}, 900_000);diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.tsindex 4b9e8c0c5..a39dcf7e4 100644--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts@@ -63,6 +63,9 @@ const FULL_SCAN_LOG_KEY_LIMIT = 1024;const ENCODED_TYPE_KEY = '__fluxer_type';const POSTGRES_KV_SCHEMA_LOCK_NAMESPACE = 0x46584b56;const POSTGRES_KV_SCHEMA_LOCK_TIMEOUT = '120s';+const POSTGRES_KV_SCHEMA_MIGRATION_TIMEOUT = '30min';+export const POSTGRES_KV_MIGRATION_TABLE = '__fluxer_schema_migrations';+const POSTGRES_KV_MESSAGES_PARTITION_MIGRATION = 'messages_partition_key_v1';const EXPIRED_STORED_ROW = 'kv.expires_at IS NOT NULL AND kv.expires_at <= now()';const MERGED_ROW_DATA = `CASE WHEN ${EXPIRED_STORED_ROW} THEN EXCLUDED.row_data ELSE kv.row_data || EXCLUDED.row_data END`;const KEPT_EXPIRES_AT = `CASE WHEN ${EXPIRED_STORED_ROW} THEN NULL ELSE kv.expires_at END`;@@ -723,7 +726,7 @@ export async function ensurePostgresKvSchema(client: IPostgresClient): Promise<vawait client.transaction(async (db) => {await db.query("SELECT set_config('statement_timeout', $1, true)", [POSTGRES_KV_SCHEMA_LOCK_TIMEOUT]);await db.query('SELECT pg_advisory_xact_lock($1, hashtext($2))', [POSTGRES_KV_SCHEMA_LOCK_NAMESPACE, kvTable]);- await db.query("SELECT set_config('statement_timeout', '0', true)");+ await db.query("SELECT set_config('statement_timeout', $1, true)", [POSTGRES_KV_SCHEMA_MIGRATION_TIMEOUT]);await db.query(`CREATE TABLE IF NOT EXISTS ${table} (table_name text NOT NULL,@@ -752,20 +755,32 @@ CREATE TABLE IF NOT EXISTS ${table} (`CREATE INDEX IF NOT EXISTS ${quoteIdentifier(`${kvTable}_message_reactions_message_idx`)} ON ${table} (partition_key, ((CASE WHEN row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN (row_data -> 'message_id' ->> 'value')::bigint END))) WHERE table_name = 'message_reactions'`,);await repairInvalidPostgresKvIndexes(db, kvTable);- const pending = await db.query(`+ const migrated = await db.query(`SELECT 1 FROM ${table} WHERE table_name = $1 AND row_key = $2 LIMIT 1`, [+ POSTGRES_KV_MIGRATION_TABLE,+ POSTGRES_KV_MESSAGES_PARTITION_MIGRATION,+ ]);+ if (migrated.rows.length === 0) {+ const pending = await db.query(`SELECT 1FROM ${table}WHERE table_name = 'messages'AND partition_key = row_keyAND split_part(row_key, chr(31), 3) <> ''LIMIT 1`);- if (pending.rows.length > 0) {- await db.query(`+ if (pending.rows.length > 0) {+ await db.query(`UPDATE ${table}SET partition_key = split_part(row_key, chr(31), 1) || chr(31) || split_part(row_key, chr(31), 2)WHERE table_name = 'messages'AND partition_key = row_keyAND split_part(row_key, chr(31), 3) <> ''`);+ }+ await db.query(+ `INSERT INTO ${table} (table_name, partition_key, row_key, row_data)+VALUES ($1, $2, $2, jsonb_build_object('applied_at', now()))+ON CONFLICT (table_name, row_key) DO NOTHING`,+ [POSTGRES_KV_MIGRATION_TABLE, POSTGRES_KV_MESSAGES_PARTITION_MIGRATION],+ );}await db.query(`DROP INDEX IF EXISTS ${quoteIdentifier(`${kvTable}_partition_idx`)}`);});diff --git a/fluxer_api/src/api/database/PostgresKvQueryPlan.test.ts b/fluxer_api/src/api/database/PostgresKvQueryPlan.test.tsindex 7bdb05516..bcbb0b638 100644--- a/fluxer_api/src/api/database/PostgresKvQueryPlan.test.ts+++ b/fluxer_api/src/api/database/PostgresKvQueryPlan.test.ts@@ -2,9 +2,17 @@import cassandra from 'cassandra-driver';import {describe, expect, it} from 'vitest';+import '../Tables';+import {getTableMetadata} from './CassandraMetaRegistry';import {defineTable} from './CassandraTableDsl';import type {CassandraParams, PreparedQuery} from './CassandraTypes';-import {buildCandidatePlan, type CandidatePlan, keyFromColumns, matchesWhere} from './PostgresKvQueryExecutor';+import {+ buildCandidatePlan,+ type CandidatePlan,+ keyFromColumns,+ matchesWhere,+ POSTGRES_KV_MIGRATION_TABLE,+} from './PostgresKvQueryExecutor';type Row = Record<string, unknown>;@@ -488,3 +496,10 @@ describe('PostgresKvQueryExecutor key equality', () => {expect(matchesWhere({a: '2020-01-01'}, eqWhere, {a: localDate} as CassandraParams)).toBe(false);});});++describe('PostgresKvQueryExecutor migration bookkeeping', () => {+ it('reserves a table name that no logical table can claim', () => {+ expect(POSTGRES_KV_MIGRATION_TABLE.startsWith('__')).toBe(true);+ expect(getTableMetadata(POSTGRES_KV_MIGRATION_TABLE)).toBeUndefined();+ });+});diff --git a/fluxer_api/src/api/database/PostgresKvUpgradeSafety.test.ts b/fluxer_api/src/api/database/PostgresKvUpgradeSafety.test.tsindex 9766dfadf..ef9610d71 100644--- a/fluxer_api/src/api/database/PostgresKvUpgradeSafety.test.ts+++ b/fluxer_api/src/api/database/PostgresKvUpgradeSafety.test.ts@@ -12,7 +12,7 @@ import {import {afterAll, beforeAll, describe, expect, it} from 'vitest';import {GuildMembers, ReadStates, Users} from '../Tables';import {LegacyPostgresKvQueryExecutor, legacyEnsurePostgresKvSchema} from './__testref__/LegacyPostgresKvQueryExecutor';-import {ensurePostgresKvSchema, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';+import {ensurePostgresKvSchema, POSTGRES_KV_MIGRATION_TABLE, PostgresKvQueryExecutor} from './PostgresKvQueryExecutor';type Row = Record<string, unknown>;@@ -307,11 +307,11 @@ suite('postgres kv upgrade safety', () => {}const diff = await raw.query<{n: string}>(`SELECT count(*) AS n FROM (- (SELECT table_name, partition_key, row_key FROM ${OLD}- EXCEPT SELECT table_name, partition_key, row_key FROM ${NEW})+ (SELECT table_name, partition_key, row_key FROM ${OLD} WHERE table_name <> '${POSTGRES_KV_MIGRATION_TABLE}'+ EXCEPT SELECT table_name, partition_key, row_key FROM ${NEW} WHERE table_name <> '${POSTGRES_KV_MIGRATION_TABLE}')UNION ALL- (SELECT table_name, partition_key, row_key FROM ${NEW}- EXCEPT SELECT table_name, partition_key, row_key FROM ${OLD})+ (SELECT table_name, partition_key, row_key FROM ${NEW} WHERE table_name <> '${POSTGRES_KV_MIGRATION_TABLE}'+ EXCEPT SELECT table_name, partition_key, row_key FROM ${OLD} WHERE table_name <> '${POSTGRES_KV_MIGRATION_TABLE}')) d`);const schemaDiff = await raw.query<{n: string}>(`SELECT count(*) AS n FROM (@@ -377,30 +377,48 @@ suite('postgres kv upgrade safety', () => {expect(failures.length).toBe(0);}, 120_000);- it('backfills the messages partition key once and then skips the update', async () => {+ it('backfills the messages partition key once and never scans for it again', async () => {const BACKFILL = 'kv_backfill';- const legacyKey = `"c"${String.fromCharCode(31)}"b"${String.fromCharCode(31)}"m"`;+ const SEP = String.fromCharCode(31);const backfillClient = new TableClient(raw, BACKFILL);+ const legacyKey = (id: string) => `"c"${SEP}"b"${SEP}"${id}"`;+ const insertLegacy = async (id: string) => {+ await raw.query(+ `INSERT INTO ${BACKFILL} (table_name, partition_key, row_key, row_data) VALUES ('messages', $1, $1, '{}'::jsonb)`,+ [legacyKey(id)],+ );+ };+ const partitionOf = async (id: string) => {+ const result = await raw.query<{partition_key: string}>(+ `SELECT partition_key FROM ${BACKFILL} WHERE table_name = 'messages' AND row_key = $1`,+ [legacyKey(id)],+ );+ return result.rows[0]?.partition_key;+ };await raw.query(`DROP TABLE IF EXISTS ${BACKFILL}`);- await ensurePostgresKvSchema(backfillClient);- await raw.query(- `INSERT INTO ${BACKFILL} (table_name, partition_key, row_key, row_data) VALUES ('messages', $1, $1, '{}'::jsonb)`,- [legacyKey],- );+ await legacyEnsurePostgresKvSchema(backfillClient);+ await insertLegacy('m1');const pendingBefore = await raw.query(`SELECT 1 FROM ${BACKFILL} WHERE table_name = 'messages' AND partition_key = row_key AND split_part(row_key, chr(31), 3) <> ''`,);expect(pendingBefore.rows.length).toBe(1);await ensurePostgresKvSchema(backfillClient);- const after = await raw.query<{partition_key: string}>(`SELECT partition_key FROM ${BACKFILL}`);- expect(after.rows[0]?.partition_key).toBe(`"c"${String.fromCharCode(31)}"b"`);+ const marker = await raw.query(+ `SELECT 1 FROM ${BACKFILL} WHERE table_name = $1 AND row_key = 'messages_partition_key_v1'`,+ [POSTGRES_KV_MIGRATION_TABLE],+ );const pendingAfter = await raw.query(`SELECT 1 FROM ${BACKFILL} WHERE table_name = 'messages' AND partition_key = row_key AND split_part(row_key, chr(31), 3) <> ''`,);+ expect(await partitionOf('m1')).toBe(`"c"${SEP}"b"`);expect(pendingAfter.rows.length).toBe(0);+ expect(marker.rows.length).toBe(1);+ await insertLegacy('m2');await ensurePostgresKvSchema(backfillClient);- const stable = await raw.query<{partition_key: string}>(`SELECT partition_key FROM ${BACKFILL}`);- expect(stable.rows[0]?.partition_key).toBe(`"c"${String.fromCharCode(31)}"b"`);+ const skipped = await partitionOf('m2');+ console.log(`second boot left the new legacy row alone: partition_key=${skipped === legacyKey('m2')}`);+ expect(skipped).toBe(legacyKey('m2'));+ expect(await partitionOf('m1')).toBe(`"c"${SEP}"b"`);await raw.query(`DROP TABLE IF EXISTS ${BACKFILL}`);}, 120_000);