Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(api): fill kv sort_id in postgres for rows from any writer

Hampuse1df98b85aa0
diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
index 43170194f..e550af263 100644
--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
@@ -911,6 +911,29 @@ async function postgresKvHasSortId(db: PostgresQueryable, kvTable: string): Prom
return result.rows.length > 0;
}
+async function postgresKvHasSortIdFill(db: PostgresQueryable, kvTable: string): Promise<boolean> {
+ const result = await db.query('SELECT 1 FROM pg_trigger WHERE tgrelid = to_regclass($1) AND tgname = $2', [
+ kvTable,
+ `${kvTable}_sort_id_fill`,
+ ]);
+ return result.rows.length > 0;
+}
+
+async function createPostgresKvSortIdFill(db: PostgresQueryable, kvTable: string): Promise<void> {
+ const fill = quoteIdentifier(`${kvTable}_sort_id_fill`);
+ await db.query(`CREATE OR REPLACE FUNCTION ${fill}() RETURNS trigger LANGUAGE plpgsql AS $fill$
+BEGIN
+ IF NEW.row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN
+ NEW.sort_id := (NEW.row_data -> 'message_id' ->> 'value')::bigint;
+ END IF;
+ RETURN NEW;
+END
+$fill$`);
+ await db.query(
+ `CREATE TRIGGER ${fill} BEFORE INSERT OR UPDATE ON ${quoteIdentifier(kvTable)} FOR EACH ROW WHEN (NEW.sort_id IS NULL AND NEW.table_name IN ('messages', 'message_reactions')) EXECUTE FUNCTION ${fill}()`,
+ );
+}
+
async function postgresKvNeedsLz4(db: PostgresQueryable, kvTable: string): Promise<boolean> {
const result = await db.query<{needs_lz4: boolean}>(
`SELECT att.attcompression <> 'l' AND EXISTS (SELECT 1 FROM pg_settings WHERE name = 'default_toast_compression' AND 'lz4' = ANY(enumvals)) AS needs_lz4
@@ -949,11 +972,13 @@ CREATE TABLE IF NOT EXISTS ${table} (
PRIMARY KEY (table_name, row_key)
)`);
const addSortId = !(await postgresKvHasSortId(db, kvTable));
+ const addSortIdFill = !(await postgresKvHasSortIdFill(db, kvTable));
const missingOptions = await missingPostgresKvStorageOptions(db, kvTable);
const setLz4 = await postgresKvNeedsLz4(db, kvTable);
- if (addSortId || missingOptions.length > 0 || setLz4) {
+ if (addSortId || addSortIdFill || missingOptions.length > 0 || setLz4) {
await db.query("SELECT set_config('lock_timeout', $1, true)", [POSTGRES_KV_ALTER_LOCK_TIMEOUT]);
if (addSortId) await db.query(`ALTER TABLE ${table} ADD COLUMN sort_id bigint`);
+ if (addSortIdFill) await createPostgresKvSortIdFill(db, kvTable);
if (missingOptions.length > 0) await db.query(`ALTER TABLE ${table} SET (${missingOptions.join(', ')})`);
if (setLz4) await db.query(`ALTER TABLE ${table} ALTER COLUMN row_data SET COMPRESSION lz4`);
await db.query("SELECT set_config('lock_timeout', '0', true)");
diff --git a/fluxer_api/src/api/database/PostgresKvRustParity.test.ts b/fluxer_api/src/api/database/PostgresKvRustParity.test.ts
index 574ebae40..6b1fd78f5 100644
--- a/fluxer_api/src/api/database/PostgresKvRustParity.test.ts
+++ b/fluxer_api/src/api/database/PostgresKvRustParity.test.ts
@@ -11,7 +11,7 @@ const REPO_ROOT = path.resolve(THIS_DIR, '../../../..');
const TYPESCRIPT_SOURCE = fs.readFileSync(path.join(THIS_DIR, 'PostgresKvQueryExecutor.ts'), 'utf8');
const RUST_SOURCE = fs.readFileSync(path.join(REPO_ROOT, 'fluxer_svc/src/postgres.rs'), 'utf8');
-const RUST_INDEX_SUFFIXES = ['expires_idx', 'partition_idx', 'row_key_c_idx'];
+const RUST_INDEX_SUFFIXES = ['expires_idx', 'partition_idx', 'row_key_c_idx', 'sort_id_fill'];
const API_ONLY_INDEX_SUFFIXES = [
'message_reactions_message_idx',
@@ -33,6 +33,9 @@ const SHARED_LITERALS = [
"'lz4' = ANY(enumvals)",
"att.attcompression <> 'l'",
'ALTER COLUMN row_data SET COMPRESSION lz4',
+ "IF NEW.row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN",
+ "NEW.sort_id := (NEW.row_data -> 'message_id' ->> 'value')::bigint;",
+ "FOR EACH ROW WHEN (NEW.sort_id IS NULL AND NEW.table_name IN ('messages', 'message_reactions')) EXECUTE FUNCTION",
];
const BACKFILL_FRAGMENTS = [
diff --git a/fluxer_svc/src/postgres.rs b/fluxer_svc/src/postgres.rs
index c58571e64..f0bde1fe4 100644
--- a/fluxer_svc/src/postgres.rs
+++ b/fluxer_svc/src/postgres.rs
@@ -190,6 +190,37 @@ async fn has_sort_id(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::R
Ok(row.is_some())
}
+async fn has_sort_id_fill(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::Result<bool> {
+ let row = transaction
+ .query_opt(
+ "SELECT 1 FROM pg_trigger WHERE tgrelid = to_regclass($1) AND tgname = $2",
+ &[&kv_table, &format!("{kv_table}_sort_id_fill")],
+ )
+ .await
+ .context("failed to inspect the Postgres KV sort_id fill trigger")?;
+ Ok(row.is_some())
+}
+
+async fn create_sort_id_fill(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::Result<()> {
+ let table = quote_identifier(kv_table)?;
+ let fill = quote_identifier(&format!("{kv_table}_sort_id_fill"))?;
+ transaction
+ .batch_execute(&format!(
+ r#"CREATE OR REPLACE FUNCTION {fill}() RETURNS trigger LANGUAGE plpgsql AS $fill$
+BEGIN
+ IF NEW.row_data -> 'message_id' ->> 'value' ~ '^-?[0-9]+$' THEN
+ NEW.sort_id := (NEW.row_data -> 'message_id' ->> 'value')::bigint;
+ END IF;
+ RETURN NEW;
+END
+$fill$;
+CREATE TRIGGER {fill} BEFORE INSERT OR UPDATE ON {table} FOR EACH ROW WHEN (NEW.sort_id IS NULL AND NEW.table_name IN ('messages', 'message_reactions')) EXECUTE FUNCTION {fill}();"#
+ ))
+ .await
+ .context("failed to create the Postgres KV sort_id fill trigger")?;
+ Ok(())
+}
+
async fn needs_lz4(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::Result<bool> {
let row = transaction
.query_opt(
@@ -308,9 +339,10 @@ CREATE TABLE IF NOT EXISTS {table} (
.await
.context("failed to ensure Postgres KV schema")?;
let add_sort_id = !has_sort_id(&transaction, kv_table).await?;
+ let add_sort_id_fill = !has_sort_id_fill(&transaction, kv_table).await?;
let missing_options = missing_storage_options(&transaction, kv_table).await?;
let set_lz4 = needs_lz4(&transaction, kv_table).await?;
- if add_sort_id || !missing_options.is_empty() || set_lz4 {
+ if add_sort_id || add_sort_id_fill || !missing_options.is_empty() || set_lz4 {
transaction
.query_one(
"SELECT set_config('lock_timeout', $1, true)",
@@ -324,6 +356,9 @@ CREATE TABLE IF NOT EXISTS {table} (
.await
.context("failed to add the Postgres KV sort_id column")?;
}
+ if add_sort_id_fill {
+ create_sort_id_fill(&transaction, kv_table).await?;
+ }
if !missing_options.is_empty() {
transaction
.batch_execute(&format!(