Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

perf(api): compress kv row data with lz4 when available

Hampus992f63f24e26
diff --git a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
index 881420012..dd3190e01 100644
--- a/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
+++ b/fluxer_api/src/api/database/PostgresKvQueryExecutor.ts
@@ -911,6 +911,16 @@ async function postgresKvHasSortId(db: PostgresQueryable, kvTable: string): Prom
return result.rows.length > 0;
}
+async function postgresKvNeedsLz4(db: PostgresQueryable, kvTable: string): Promise<boolean> {
+ const result = await db.query<{needs_lz4: boolean}>(
+ `SELECT att.attcompression <> 'l' AND EXISTS (SELECT 1 FROM pg_settings WHERE name = 'default_toast_compression' AND 'lz4' = ANY(enumvals)) AS needs_lz4
+FROM pg_attribute att
+WHERE att.attrelid = to_regclass($1) AND att.attname = 'row_data' AND NOT att.attisdropped`,
+ [kvTable],
+ );
+ return result.rows[0]?.needs_lz4 === true;
+}
+
async function missingPostgresKvStorageOptions(db: PostgresQueryable, kvTable: string): Promise<Array<string>> {
const result = await db.query<{options: Array<string> | null}>(
'SELECT reloptions AS options FROM pg_class WHERE oid = to_regclass($1)',
@@ -940,10 +950,12 @@ CREATE TABLE IF NOT EXISTS ${table} (
)`);
const addSortId = !(await postgresKvHasSortId(db, kvTable));
const missingOptions = await missingPostgresKvStorageOptions(db, kvTable);
- if (addSortId || missingOptions.length > 0) {
+ const setLz4 = await postgresKvNeedsLz4(db, kvTable);
+ if (addSortId || missingOptions.length > 0 || setLz4) {
await db.query("SELECT set_config('lock_timeout', $1, true)", [POSTGRES_KV_ALTER_LOCK_TIMEOUT]);
if (addSortId) await db.query(`ALTER TABLE ${table} ADD COLUMN sort_id bigint`);
if (missingOptions.length > 0) await db.query(`ALTER TABLE ${table} SET (${missingOptions.join(', ')})`);
+ if (setLz4) await db.query(`ALTER TABLE ${table} ALTER COLUMN row_data SET COMPRESSION lz4`);
await db.query("SELECT set_config('lock_timeout', '0', true)");
}
if (!(await postgresKvRowKeyIsCCollated(db, kvTable))) {
diff --git a/fluxer_api/src/api/database/PostgresKvRustParity.test.ts b/fluxer_api/src/api/database/PostgresKvRustParity.test.ts
index 71a5bfd81..574ebae40 100644
--- a/fluxer_api/src/api/database/PostgresKvRustParity.test.ts
+++ b/fluxer_api/src/api/database/PostgresKvRustParity.test.ts
@@ -30,6 +30,9 @@ const SHARED_LITERALS = [
'sort_id bigint,',
'ADD COLUMN sort_id bigint',
"attname = 'sort_id'",
+ "'lz4' = ANY(enumvals)",
+ "att.attcompression <> 'l'",
+ 'ALTER COLUMN row_data SET COMPRESSION lz4',
];
const BACKFILL_FRAGMENTS = [
diff --git a/fluxer_svc/src/postgres.rs b/fluxer_svc/src/postgres.rs
index 8fa9d8dc6..c58571e64 100644
--- a/fluxer_svc/src/postgres.rs
+++ b/fluxer_svc/src/postgres.rs
@@ -190,6 +190,19 @@ async fn has_sort_id(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::R
Ok(row.is_some())
}
+async fn needs_lz4(transaction: &Transaction<'_>, kv_table: &str) -> anyhow::Result<bool> {
+ let row = transaction
+ .query_opt(
+ r#"SELECT att.attcompression <> 'l' AND EXISTS (SELECT 1 FROM pg_settings WHERE name = 'default_toast_compression' AND 'lz4' = ANY(enumvals)) AS needs_lz4
+FROM pg_attribute att
+WHERE att.attrelid = to_regclass($1) AND att.attname = 'row_data' AND NOT att.attisdropped"#,
+ &[&kv_table],
+ )
+ .await
+ .context("failed to inspect Postgres KV row_data compression")?;
+ Ok(row.and_then(|row| row.get::<_, Option<bool>>("needs_lz4")) == Some(true))
+}
+
async fn missing_storage_options(
transaction: &Transaction<'_>,
kv_table: &str,
@@ -296,7 +309,8 @@ CREATE TABLE IF NOT EXISTS {table} (
.context("failed to ensure Postgres KV schema")?;
let add_sort_id = !has_sort_id(&transaction, kv_table).await?;
let missing_options = missing_storage_options(&transaction, kv_table).await?;
- if add_sort_id || !missing_options.is_empty() {
+ let set_lz4 = needs_lz4(&transaction, kv_table).await?;
+ if add_sort_id || !missing_options.is_empty() || set_lz4 {
transaction
.query_one(
"SELECT set_config('lock_timeout', $1, true)",
@@ -319,6 +333,14 @@ CREATE TABLE IF NOT EXISTS {table} (
.await
.context("failed to set Postgres KV storage options")?;
}
+ if set_lz4 {
+ transaction
+ .batch_execute(&format!(
+ "ALTER TABLE {table} ALTER COLUMN row_data SET COMPRESSION lz4"
+ ))
+ .await
+ .context("failed to set Postgres KV row_data compression")?;
+ }
transaction
.query_one("SELECT set_config('lock_timeout', '0', true)", &[])
.await