A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
perf(api): compress kv row data with lz4 when available
Hampus992f63f24e26
diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.tsindex 881420012..dd3190e01 100644--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts@@ -911,6 +911,16 @@ async function postgresKvHasSortId(db: PostgresQueryable, kvTable: string): Promreturn result.rows.length > 0;}+async function postgresKvNeedsLz4(db: PostgresQueryable, kvTable: string): Promise<boolean> {+ const result = await db.query<{needs_lz4: boolean}>(+ `SELECT att.attcompression <> 'l' AND EXISTS (SELECT 1 FROM pg_settings WHERE name = 'default_toast_compression' AND 'lz4' = ANY(enumvals)) AS needs_lz4+FROM pg_attribute att+WHERE att.attrelid = to_regclass($1) AND att.attname = 'row_data' AND NOT att.attisdropped`,+ [kvTable],+ );+ return result.rows[0]?.needs_lz4 === true;+}+async function missingPostgresKvStorageOptions(db: PostgresQueryable, kvTable: string): Promise<Array<string>> {const result = await db.query<{options: Array<string> | null}>('SELECT reloptions AS options FROM pg_class WHERE oid = to_regclass($1)',@@ -940,10 +950,12 @@ CREATE TABLE IF NOT EXISTS ${table} ()`);const addSortId = !(await postgresKvHasSortId(db, kvTable));const missingOptions = await missingPostgresKvStorageOptions(db, kvTable);- if (addSortId || missingOptions.length > 0) {+ const setLz4 = await postgresKvNeedsLz4(db, kvTable);+ if (addSortId || missingOptions.length > 0 || setLz4) {await db.query("SELECT set_config('lock_timeout', $1, true)", [POSTGRES_KV_ALTER_LOCK_TIMEOUT]);if (addSortId) await db.query(`ALTER TABLE ${table} ADD COLUMN sort_id bigint`);if (missingOptions.length > 0) await db.query(`ALTER TABLE ${table} SET (${missingOptions.join(', ')})`);+ if (setLz4) await db.query(`ALTER TABLE ${table} ALTER COLUMN row_data SET COMPRESSION lz4`);await db.query("SELECT set_config('lock_timeout', '0', true)");}if (!(await postgresKvRowKeyIsCCollated(db, kvTable))) {diff --git a/fluxer_api/src/api/database/PostgresKvRustParity.test.ts b/fluxer_api/src/api/database/PostgresKvRustParity.test.tsindex 71a5bfd81..574ebae40 100644--- a/fluxer_api/src/api/database/PostgresKvRustParity.test.ts+++ b/fluxer_api/src/api/database/PostgresKvRustParity.test.ts@@ -30,6 +30,9 @@ const SHARED_LITERALS = ['sort_id bigint,','ADD COLUMN sort_id bigint',"attname = 'sort_id'",+ "'lz4' = ANY(enumvals)",+ "att.attcompression <> 'l'",+ 'ALTER COLUMN row_data SET COMPRESSION lz4',];const BACKFILL_FRAGMENTS = [diff --git a/fluxer_svc/src/postgres.rs b/fluxer_svc/src/postgres.rsindex 8fa9d8dc6..c58571e64 100644--- a/fluxer_svc/src/postgres.rs+++ b/fluxer_svc/src/postgres.rs@@ -190,6 +190,19 @@ async fn has_sort_id(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::ROk(row.is_some())}+async fn needs_lz4(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::Result<bool> {+ let row = transaction+ .query_opt(+ r#"SELECT att.attcompression <> 'l' AND EXISTS (SELECT 1 FROM pg_settings WHERE name = 'default_toast_compression' AND 'lz4' = ANY(enumvals)) AS needs_lz4+FROM pg_attribute att+WHERE att.attrelid = to_regclass($1) AND att.attname = 'row_data' AND NOT att.attisdropped"#,+ &[&kv_table],+ )+ .await+ .context("failed to inspect Postgres KV row_data compression")?;+ Ok(row.and_then(|row| row.get::<_, Option<bool>>("needs_lz4")) == Some(true))+}+async fn missing_storage_options(transaction: &Transaction<'_>,kv_table: &str,@@ -296,7 +309,8 @@ CREATE TABLE IF NOT EXISTS {table} (.context("failed to ensure Postgres KV schema")?;let add_sort_id = !has_sort_id(&transaction, kv_table).await?;let missing_options = missing_storage_options(&transaction, kv_table).await?;- if add_sort_id || !missing_options.is_empty() {+ let set_lz4 = needs_lz4(&transaction, kv_table).await?;+ if add_sort_id || !missing_options.is_empty() || set_lz4 {transaction.query_one("SELECT set_config('lock_timeout', $1, true)",@@ -319,6 +333,14 @@ CREATE TABLE IF NOT EXISTS {table} (.await.context("failed to set Postgres KV storage options")?;}+ if set_lz4 {+ transaction+ .batch_execute(&format!(+ "ALTER TABLE {table} ALTER COLUMN row_data SET COMPRESSION lz4"+ ))+ .await+ .context("failed to set Postgres KV row_data compression")?;+ }transaction.query_one("SELECT set_config('lock_timeout', '0', true)", &[]).await