A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(api): fill kv sort_id in postgres for rows from any writer
Hampuse1df98b85aa0
diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.tsindex 43170194f..e550af263 100644--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts@@ -911,6 +911,29 @@ async function postgresKvHasSortId(db: PostgresQueryable, kvTable: string): Promreturn result.rows.length > 0;}+async function postgresKvHasSortIdFill(db: PostgresQueryable, kvTable: string): Promise<boolean> {+ const result = await db.query('SELECT 1 FROM pg_trigger WHERE tgrelid = to_regclass($1) AND tgname = $2', [+ kvTable,+ `${kvTable}_sort_id_fill`,+ ]);+ return result.rows.length > 0;+}++async function createPostgresKvSortIdFill(db: PostgresQueryable, kvTable: string): Promise<void> {+ const fill = quoteIdentifier(`${kvTable}_sort_id_fill`);+ await db.query(`CREATE OR REPLACE FUNCTION ${fill}() RETURNS trigger LANGUAGE plpgsql AS $fill$+BEGIN+ IF NEW.row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN+ NEW.sort_id := (NEW.row_data -> 'message_id' ->> 'value')::bigint;+ END IF;+ RETURN NEW;+END+$fill$`);+ await db.query(+ `CREATE TRIGGER ${fill} BEFORE INSERT OR UPDATE ON ${quoteIdentifier(kvTable)} FOR EACH ROW WHEN (NEW.sort_id IS NULL AND NEW.table_name IN ('messages', 'message_reactions')) EXECUTE FUNCTION ${fill}()`,+ );+}+async function postgresKvNeedsLz4(db: PostgresQueryable, kvTable: string): Promise<boolean> {const result = await db.query<{needs_lz4: boolean}>(`SELECT att.attcompression <> 'l' AND EXISTS (SELECT 1 FROM pg_settings WHERE name = 'default_toast_compression' AND 'lz4' = ANY(enumvals)) AS needs_lz4@@ -949,11 +972,13 @@ CREATE TABLE IF NOT EXISTS ${table} (PRIMARY KEY (table_name, row_key))`);const addSortId = !(await postgresKvHasSortId(db, kvTable));+ const addSortIdFill = !(await postgresKvHasSortIdFill(db, kvTable));const missingOptions = await missingPostgresKvStorageOptions(db, kvTable);const setLz4 = await postgresKvNeedsLz4(db, kvTable);- if (addSortId || missingOptions.length > 0 || setLz4) {+ if (addSortId || addSortIdFill || missingOptions.length > 0 || setLz4) {await db.query("SELECT set_config('lock_timeout', $1, true)", [POSTGRES_KV_ALTER_LOCK_TIMEOUT]);if (addSortId) await db.query(`ALTER TABLE ${table} ADD COLUMN sort_id bigint`);+ if (addSortIdFill) await createPostgresKvSortIdFill(db, kvTable);if (missingOptions.length > 0) await db.query(`ALTER TABLE ${table} SET (${missingOptions.join(', ')})`);if (setLz4) await db.query(`ALTER TABLE ${table} ALTER COLUMN row_data SET COMPRESSION lz4`);await db.query("SELECT set_config('lock_timeout', '0', true)");diff --git a/fluxer_api/src/api/database/PostgresKvRustParity.test.ts b/fluxer_api/src/api/database/PostgresKvRustParity.test.tsindex 574ebae40..6b1fd78f5 100644--- a/fluxer_api/src/api/database/PostgresKvRustParity.test.ts+++ b/fluxer_api/src/api/database/PostgresKvRustParity.test.ts@@ -11,7 +11,7 @@ const REPO_ROOT = path.resolve(THIS_DIR, '../../../..');const TYPESCRIPT_SOURCE = fs.readFileSync(path.join(THIS_DIR, 'PostgresKvQueryExecutor.ts'), 'utf8');const RUST_SOURCE = fs.readFileSync(path.join(REPO_ROOT, 'fluxer_svc/src/postgres.rs'), 'utf8');-const RUST_INDEX_SUFFIXES = ['expires_idx', 'partition_idx', 'row_key_c_idx'];+const RUST_INDEX_SUFFIXES = ['expires_idx', 'partition_idx', 'row_key_c_idx', 'sort_id_fill'];const API_ONLY_INDEX_SUFFIXES = ['message_reactions_message_idx',@@ -33,6 +33,9 @@ const SHARED_LITERALS = ["'lz4' = ANY(enumvals)","att.attcompression <> 'l'",'ALTER COLUMN row_data SET COMPRESSION lz4',+ "IF NEW.row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN",+ "NEW.sort_id := (NEW.row_data -> 'message_id' ->> 'value')::bigint;",+ "FOR EACH ROW WHEN (NEW.sort_id IS NULL AND NEW.table_name IN ('messages', 'message_reactions')) EXECUTE FUNCTION",];const BACKFILL_FRAGMENTS = [diff --git a/fluxer_svc/src/postgres.rs b/fluxer_svc/src/postgres.rsindex c58571e64..f0bde1fe4 100644--- a/fluxer_svc/src/postgres.rs+++ b/fluxer_svc/src/postgres.rs@@ -190,6 +190,37 @@ async fn has_sort_id(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::ROk(row.is_some())}+async fn has_sort_id_fill(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::Result<bool> {+ let row = transaction+ .query_opt(+ "SELECT 1 FROM pg_trigger WHERE tgrelid = to_regclass($1) AND tgname = $2",+ &[&kv_table, &format!("{kv_table}_sort_id_fill")],+ )+ .await+ .context("failed to inspect the Postgres KV sort_id fill trigger")?;+ Ok(row.is_some())+}++async fn create_sort_id_fill(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::Result<()> {+ let table = quote_identifier(kv_table)?;+ let fill = quote_identifier(&format!("{kv_table}_sort_id_fill"))?;+ transaction+ .batch_execute(&format!(+ r#"CREATE OR REPLACE FUNCTION {fill}() RETURNS trigger LANGUAGE plpgsql AS $fill$+BEGIN+ IF NEW.row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN+ NEW.sort_id := (NEW.row_data -> 'message_id' ->> 'value')::bigint;+ END IF;+ RETURN NEW;+END+$fill$;+CREATE TRIGGER {fill} BEFORE INSERT OR UPDATE ON {table} FOR EACH ROW WHEN (NEW.sort_id IS NULL AND NEW.table_name IN ('messages', 'message_reactions')) EXECUTE FUNCTION {fill}();"#+ ))+ .await+ .context("failed to create the Postgres KV sort_id fill trigger")?;+ Ok(())+}+async fn needs_lz4(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::Result<bool> {let row = transaction.query_opt(@@ -308,9 +339,10 @@ CREATE TABLE IF NOT EXISTS {table} (.await.context("failed to ensure Postgres KV schema")?;let add_sort_id = !has_sort_id(&transaction, kv_table).await?;+ let add_sort_id_fill = !has_sort_id_fill(&transaction, kv_table).await?;let missing_options = missing_storage_options(&transaction, kv_table).await?;let set_lz4 = needs_lz4(&transaction, kv_table).await?;- if add_sort_id || !missing_options.is_empty() || set_lz4 {+ if add_sort_id || add_sort_id_fill || !missing_options.is_empty() || set_lz4 {transaction.query_one("SELECT set_config('lock_timeout', $1, true)",@@ -324,6 +356,9 @@ CREATE TABLE IF NOT EXISTS {table} (.await.context("failed to add the Postgres KV sort_id column")?;}+ if add_sort_id_fill {+ create_sort_id_fill(&transaction, kv_table).await?;+ }if !missing_options.is_empty() {transaction.batch_execute(&format!(