PostgreSQL backup tool with Point-In-Time-Recovery and restore verification
TypeScript backupbackupsdatabasedatabase-backupdevopsdockergolangkubernetesmariadbmongodbmysqlpgpostgrespostgresqls3self-hostedsystem-administrationtoolsweb-ui
FEATURE (mysql/mariadb): Make extended inserts and Galera wsrep_on=OFF optional (issues #636, #630, #635)
Rostislav Dugin38f56941e147
diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.goindex 61c0dd92..5104f530 100644--- a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc.go@@ -113,11 +113,17 @@ func (uc *CreateMariadbBackupUsecase) buildMariadbDumpArgs("--single-transaction","--routines","--quick",- "--skip-extended-insert","--skip-add-locks","--verbose",}+ // One INSERT per row caps mariadb-dump memory on huge tables, but bloats the+ // dump and makes restores far slower. Opting into extended inserts batches+ // rows (mariadb-dump's default) for fast restores at higher backup memory.+ if !mdb.IsUseExtendedInsert {+ args = append(args, "--skip-extended-insert")+ }+if mdb.HasPrivilege("TRIGGER") {args = append(args, "--triggers")}diff --git a/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.gonew file mode 100644index 00000000..30feb5ea--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/logical/mariadb/create_backup_uc_test.go@@ -0,0 +1,37 @@+package usecases_logical_mariadb++import (+ "slices"+ "testing"++ mariadbtypes "databasus-backend/internal/features/databases/databases/mariadb"+ "databasus-backend/internal/util/tools"+)++func Test_BuildMariadbDumpArgs_WhenExtendedInsertDisabled_SkipsExtendedInsert(t *testing.T) {+ uc := &CreateMariadbBackupUsecase{}+ database := &mariadbtypes.MariadbDatabase{+ Version: tools.MariadbVersion1011,+ IsUseExtendedInsert: false,+ }++ args := uc.buildMariadbDumpArgs(database)++ if !slices.Contains(args, "--skip-extended-insert") {+ t.Fatalf("expected --skip-extended-insert when extended inserts are disabled, got %v", args)+ }+}++func Test_BuildMariadbDumpArgs_WhenExtendedInsertEnabled_OmitsSkipExtendedInsert(t *testing.T) {+ uc := &CreateMariadbBackupUsecase{}+ database := &mariadbtypes.MariadbDatabase{+ Version: tools.MariadbVersion1011,+ IsUseExtendedInsert: true,+ }++ args := uc.buildMariadbDumpArgs(database)++ if slices.Contains(args, "--skip-extended-insert") {+ t.Fatalf("expected no --skip-extended-insert when extended inserts are enabled, got %v", args)+ }+}diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.goindex b84ec5aa..4b151a37 100644--- a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc.go@@ -112,11 +112,17 @@ func (uc *CreateMysqlBackupUsecase) buildMysqldumpArgs(my *mysqltypes.MysqlDatab"--routines","--set-gtid-purged=OFF","--quick",- "--skip-extended-insert","--skip-add-locks","--verbose",}+ // One INSERT per row caps mysqldump memory on huge tables, but bloats the+ // dump and makes restores far slower. Opting into extended inserts batches+ // rows (mysqldump's default) for fast restores at higher backup memory.+ if !my.IsUseExtendedInsert {+ args = append(args, "--skip-extended-insert")+ }+if my.HasPrivilege("TRIGGER") {args = append(args, "--triggers")}diff --git a/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.gonew file mode 100644index 00000000..2f745060--- /dev/null+++ b/backend/internal/features/backups/backups/usecases/logical/mysql/create_backup_uc_test.go@@ -0,0 +1,37 @@+package usecases_logical_mysql++import (+ "slices"+ "testing"++ mysqltypes "databasus-backend/internal/features/databases/databases/mysql"+ "databasus-backend/internal/util/tools"+)++func Test_BuildMysqldumpArgs_WhenExtendedInsertDisabled_SkipsExtendedInsert(t *testing.T) {+ uc := &CreateMysqlBackupUsecase{}+ database := &mysqltypes.MysqlDatabase{+ Version: tools.MysqlVersion80,+ IsUseExtendedInsert: false,+ }++ args := uc.buildMysqldumpArgs(database)++ if !slices.Contains(args, "--skip-extended-insert") {+ t.Fatalf("expected --skip-extended-insert when extended inserts are disabled, got %v", args)+ }+}++func Test_BuildMysqldumpArgs_WhenExtendedInsertEnabled_OmitsSkipExtendedInsert(t *testing.T) {+ uc := &CreateMysqlBackupUsecase{}+ database := &mysqltypes.MysqlDatabase{+ Version: tools.MysqlVersion80,+ IsUseExtendedInsert: true,+ }++ args := uc.buildMysqldumpArgs(database)++ if slices.Contains(args, "--skip-extended-insert") {+ t.Fatalf("expected no --skip-extended-insert when extended inserts are enabled, got %v", args)+ }+}diff --git a/backend/internal/features/databases/databases/mariadb/model.go b/backend/internal/features/databases/databases/mariadb/model.goindex 3fd3a3d2..2a17fc55 100644--- a/backend/internal/features/databases/databases/mariadb/model.go+++ b/backend/internal/features/databases/databases/mariadb/model.go@@ -26,16 +26,18 @@ type MariadbDatabase struct {Version tools.MariadbVersion `json:"version" gorm:"type:text;not null"`- Host string `json:"host" gorm:"type:text;not null"`- Port int `json:"port" gorm:"type:int;not null"`- Username string `json:"username" gorm:"type:text;not null"`- Password string `json:"password" gorm:"type:text;not null"`- Database *string `json:"database" gorm:"type:text"`- IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`- IsExcludeEvents bool `json:"isExcludeEvents" gorm:"type:boolean;default:false"`- ExcludeTables []string `json:"excludeTables" gorm:"-"`- ExcludeTablesString string `json:"-" gorm:"column:exclude_tables;type:text;not null;default:''"`- Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`+ Host string `json:"host" gorm:"type:text;not null"`+ Port int `json:"port" gorm:"type:int;not null"`+ Username string `json:"username" gorm:"type:text;not null"`+ Password string `json:"password" gorm:"type:text;not null"`+ Database *string `json:"database" gorm:"type:text"`+ IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`+ IsExcludeEvents bool `json:"isExcludeEvents" gorm:"type:boolean;default:false"`+ IsUseExtendedInsert bool `json:"isUseExtendedInsert" gorm:"column:is_use_extended_insert;type:boolean;not null;default:false"`+ IsSkipGaleraDisable bool `json:"isSkipGaleraDisable" gorm:"column:is_skip_galera_disable;type:boolean;not null;default:false"`+ ExcludeTables []string `json:"excludeTables" gorm:"-"`+ ExcludeTablesString string `json:"-" gorm:"column:exclude_tables;type:text;not null;default:''"`+ Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`}func (m *MariadbDatabase) TableName() string {@@ -192,6 +194,8 @@ func (m *MariadbDatabase) Update(incoming *MariadbDatabase) {m.Database = incoming.Databasem.IsHttps = incoming.IsHttpsm.IsExcludeEvents = incoming.IsExcludeEvents+ m.IsUseExtendedInsert = incoming.IsUseExtendedInsert+ m.IsSkipGaleraDisable = incoming.IsSkipGaleraDisablem.ExcludeTables = incoming.ExcludeTablesm.Privileges = incoming.Privilegesdiff --git a/backend/internal/features/databases/databases/mysql/model.go b/backend/internal/features/databases/databases/mysql/model.goindex 438b09d8..414348a4 100644--- a/backend/internal/features/databases/databases/mysql/model.go+++ b/backend/internal/features/databases/databases/mysql/model.go@@ -26,16 +26,17 @@ type MysqlDatabase struct {Version tools.MysqlVersion `json:"version" gorm:"type:text;not null"`- Host string `json:"host" gorm:"type:text;not null"`- Port int `json:"port" gorm:"type:int;not null"`- Username string `json:"username" gorm:"type:text;not null"`- Password string `json:"password" gorm:"type:text;not null"`- Database *string `json:"database" gorm:"type:text"`- IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`- ExcludeTables []string `json:"excludeTables" gorm:"-"`- ExcludeTablesString string `json:"-" gorm:"column:exclude_tables;type:text;not null;default:''"`- Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`- IsZstdSupported bool `json:"isZstdSupported" gorm:"column:is_zstd_supported;type:boolean;not null;default:true"`+ Host string `json:"host" gorm:"type:text;not null"`+ Port int `json:"port" gorm:"type:int;not null"`+ Username string `json:"username" gorm:"type:text;not null"`+ Password string `json:"password" gorm:"type:text;not null"`+ Database *string `json:"database" gorm:"type:text"`+ IsHttps bool `json:"isHttps" gorm:"type:boolean;default:false"`+ ExcludeTables []string `json:"excludeTables" gorm:"-"`+ ExcludeTablesString string `json:"-" gorm:"column:exclude_tables;type:text;not null;default:''"`+ Privileges string `json:"privileges" gorm:"column:privileges;type:text;not null;default:''"`+ IsZstdSupported bool `json:"isZstdSupported" gorm:"column:is_zstd_supported;type:boolean;not null;default:true"`+ IsUseExtendedInsert bool `json:"isUseExtendedInsert" gorm:"column:is_use_extended_insert;type:boolean;not null;default:false"`}func (m *MysqlDatabase) TableName() string {@@ -195,6 +196,7 @@ func (m *MysqlDatabase) Update(incoming *MysqlDatabase) {m.ExcludeTables = incoming.ExcludeTablesm.Privileges = incoming.Privilegesm.IsZstdSupported = incoming.IsZstdSupported+ m.IsUseExtendedInsert = incoming.IsUseExtendedInsertif incoming.Password != "" {m.Password = incoming.Passworddiff --git a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.goindex 0e4d9c1d..fc720592 100644--- a/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go+++ b/backend/internal/features/restores/usecases/mariadb/restore_backup_uc.go@@ -75,7 +75,9 @@ func (uc *RestoreMariadbBackupUsecase) Execute(// "Maximum writeset size exceeded" errors on large restores.// wsrep_on is available in MariaDB 10.1+ (all builds with Galera support).// On non-Galera instances the variable still exists but is a no-op.- if mdb.Version != tools.MariadbVersion55 {+ // Setting it requires the SUPER privilege, so it is skippable for managed+ // clusters that deny SUPER (at the risk of writeset-size errors on big restores).+ if mdb.Version != tools.MariadbVersion55 && !mdb.IsSkipGaleraDisable {args = append(args, "--init-command=SET SESSION wsrep_on=OFF")}diff --git a/backend/migrations/20260620155346_add_extended_insert_and_galera_skip_to_mysql_mariadb.sql b/backend/migrations/20260620155346_add_extended_insert_and_galera_skip_to_mysql_mariadb.sqlnew file mode 100644index 00000000..d6ac580d--- /dev/null+++ b/backend/migrations/20260620155346_add_extended_insert_and_galera_skip_to_mysql_mariadb.sql@@ -0,0 +1,23 @@+-- +goose Up+-- +goose StatementBegin+ALTER TABLE mysql_databases+ ADD COLUMN IF NOT EXISTS is_use_extended_insert BOOLEAN NOT NULL DEFAULT FALSE;++ALTER TABLE mariadb_databases+ ADD COLUMN IF NOT EXISTS is_use_extended_insert BOOLEAN NOT NULL DEFAULT FALSE;++ALTER TABLE mariadb_databases+ ADD COLUMN IF NOT EXISTS is_skip_galera_disable BOOLEAN NOT NULL DEFAULT FALSE;+-- +goose StatementEnd++-- +goose Down+-- +goose StatementBegin+ALTER TABLE mysql_databases+ DROP COLUMN IF EXISTS is_use_extended_insert;++ALTER TABLE mariadb_databases+ DROP COLUMN IF EXISTS is_use_extended_insert;++ALTER TABLE mariadb_databases+ DROP COLUMN IF EXISTS is_skip_galera_disable;+-- +goose StatementEnddiff --git a/frontend/src/entity/databases/model/mariadb/MariadbDatabase.ts b/frontend/src/entity/databases/model/mariadb/MariadbDatabase.tsindex b39f4acd..2fd3a2a1 100644--- a/frontend/src/entity/databases/model/mariadb/MariadbDatabase.ts+++ b/frontend/src/entity/databases/model/mariadb/MariadbDatabase.ts@@ -10,5 +10,7 @@ export interface MariadbDatabase {database?: string;isHttps: boolean;isExcludeEvents?: boolean;+ isUseExtendedInsert?: boolean;+ isSkipGaleraDisable?: boolean;excludeTables?: string[];}diff --git a/frontend/src/entity/databases/model/mysql/MysqlDatabase.ts b/frontend/src/entity/databases/model/mysql/MysqlDatabase.tsindex 052217e7..854ff3e6 100644--- a/frontend/src/entity/databases/model/mysql/MysqlDatabase.ts+++ b/frontend/src/entity/databases/model/mysql/MysqlDatabase.ts@@ -10,5 +10,6 @@ export interface MysqlDatabase {password: string;database?: string;isHttps: boolean;+ isUseExtendedInsert?: boolean;excludeTables?: string[];}diff --git a/frontend/src/features/databases/ui/edit/EditMariaDbSpecificDataComponent.tsx b/frontend/src/features/databases/ui/edit/EditMariaDbSpecificDataComponent.tsxindex e610fb33..b764f04d 100644--- a/frontend/src/features/databases/ui/edit/EditMariaDbSpecificDataComponent.tsx+++ b/frontend/src/features/databases/ui/edit/EditMariaDbSpecificDataComponent.tsx@@ -48,7 +48,10 @@ export const EditMariaDbSpecificDataComponent = ({const [isConnectionFailed, setIsConnectionFailed] = useState(false);const hasAdvancedValues =- !!database.mariadb?.isExcludeEvents || !!database.mariadb?.excludeTables?.length;+ !!database.mariadb?.isExcludeEvents ||+ !!database.mariadb?.isUseExtendedInsert ||+ !!database.mariadb?.isSkipGaleraDisable ||+ !!database.mariadb?.excludeTables?.length;const [isShowAdvanced, setShowAdvanced] = useState(hasAdvancedValues);const [isShowPasteModal, setIsShowPasteModal] = useState(false);@@ -377,6 +380,64 @@ export const EditMariaDbSpecificDataComponent = ({</div></div>+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Use extended inserts</div>+ <div className="flex items-center">+ <Checkbox+ checked={editingDatabase.mariadb?.isUseExtendedInsert || false}+ onChange={(e) => {+ if (!editingDatabase.mariadb) return;++ setEditingDatabase({+ ...editingDatabase,+ mariadb: {+ ...editingDatabase.mariadb,+ isUseExtendedInsert: e.target.checked,+ },+ });+ }}+ >+ Enable extended inserts+ </Checkbox>++ <Tooltip+ className="cursor-pointer"+ title="Batch multiple rows per INSERT for much faster restores. Off by default because it uses more memory during backup - enable only if restores are slow and your server has enough memory."+ >+ <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />+ </Tooltip>+ </div>+ </div>++ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Galera replication</div>+ <div className="flex items-center">+ <Checkbox+ checked={editingDatabase.mariadb?.isSkipGaleraDisable || false}+ onChange={(e) => {+ if (!editingDatabase.mariadb) return;++ setEditingDatabase({+ ...editingDatabase,+ mariadb: {+ ...editingDatabase.mariadb,+ isSkipGaleraDisable: e.target.checked,+ },+ });+ }}+ >+ Skip disabling on restore+ </Checkbox>++ <Tooltip+ className="cursor-pointer"+ title="By default Databasus runs SET SESSION wsrep_on=OFF during restore to avoid Galera writeset-size errors. That requires the SUPER privilege. Enable this to skip it if your managed provider denies SUPER - large restores may then hit Galera writeset limits."+ >+ <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />+ </Tooltip>+ </div>+ </div>+<div className="mb-1 flex w-full items-center"><div className="min-w-[150px]">Exclude tables</div><Selectdiff --git a/frontend/src/features/databases/ui/edit/EditMySqlSpecificDataComponent.tsx b/frontend/src/features/databases/ui/edit/EditMySqlSpecificDataComponent.tsxindex 448d07d5..d1ed0552 100644--- a/frontend/src/features/databases/ui/edit/EditMySqlSpecificDataComponent.tsx+++ b/frontend/src/features/databases/ui/edit/EditMySqlSpecificDataComponent.tsx@@ -1,5 +1,5 @@import { CopyOutlined, DownOutlined, InfoCircleOutlined, UpOutlined } from '@ant-design/icons';-import { App, Button, Input, InputNumber, Select, Switch, Tooltip } from 'antd';+import { App, Button, Checkbox, Input, InputNumber, Select, Switch, Tooltip } from 'antd';import { useEffect, useState } from 'react';import { type Database, databaseApi } from '../../../../entity/databases';@@ -47,7 +47,8 @@ export const EditMySqlSpecificDataComponent = ({const [isTestingConnection, setIsTestingConnection] = useState(false);const [isConnectionFailed, setIsConnectionFailed] = useState(false);- const hasAdvancedValues = !!database.mysql?.excludeTables?.length;+ const hasAdvancedValues =+ !!database.mysql?.isUseExtendedInsert || !!database.mysql?.excludeTables?.length;const [isShowAdvanced, setShowAdvanced] = useState(hasAdvancedValues);const [isShowPasteModal, setIsShowPasteModal] = useState(false);@@ -346,29 +347,60 @@ export const EditMySqlSpecificDataComponent = ({</div>{isShowAdvanced && (- <div className="mb-1 flex w-full items-center">- <div className="min-w-[150px]">Exclude tables</div>- <Select- mode="tags"- value={editingDatabase.mysql?.excludeTables || []}- onChange={(values) => {- if (!editingDatabase.mysql) return;-- setEditingDatabase({- ...editingDatabase,- mysql: { ...editingDatabase.mysql, excludeTables: values },- });- }}- size="small"- className="max-w-[200px] grow"- placeholder="No tables excluded"- tokenSeparators={[',']}- />+ <>+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Use extended inserts</div>+ <div className="flex items-center">+ <Checkbox+ checked={editingDatabase.mysql?.isUseExtendedInsert || false}+ onChange={(e) => {+ if (!editingDatabase.mysql) return;++ setEditingDatabase({+ ...editingDatabase,+ mysql: {+ ...editingDatabase.mysql,+ isUseExtendedInsert: e.target.checked,+ },+ });+ }}+ >+ Enable extended inserts+ </Checkbox>++ <Tooltip+ className="cursor-pointer"+ title="Batch multiple rows per INSERT for much faster restores. Off by default because it uses more memory during backup - enable only if restores are slow and your server has enough memory."+ >+ <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />+ </Tooltip>+ </div>+ </div>- <Tooltip className="cursor-pointer" title="Table names to exclude from the backup.">- <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />- </Tooltip>- </div>+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Exclude tables</div>+ <Select+ mode="tags"+ value={editingDatabase.mysql?.excludeTables || []}+ onChange={(values) => {+ if (!editingDatabase.mysql) return;++ setEditingDatabase({+ ...editingDatabase,+ mysql: { ...editingDatabase.mysql, excludeTables: values },+ });+ }}+ size="small"+ className="max-w-[200px] grow"+ placeholder="No tables excluded"+ tokenSeparators={[',']}+ />++ <Tooltip className="cursor-pointer" title="Table names to exclude from the backup.">+ <InfoCircleOutlined className="ml-2" style={{ color: 'gray' }} />+ </Tooltip>+ </div>+ </>)}<div className="mt-5 flex">diff --git a/frontend/src/features/databases/ui/show/ShowMariaDbSpecificDataComponent.tsx b/frontend/src/features/databases/ui/show/ShowMariaDbSpecificDataComponent.tsxindex bf669660..caa78b03 100644--- a/frontend/src/features/databases/ui/show/ShowMariaDbSpecificDataComponent.tsx+++ b/frontend/src/features/databases/ui/show/ShowMariaDbSpecificDataComponent.tsx@@ -63,6 +63,20 @@ export const ShowMariaDbSpecificDataComponent = ({ database }: Props) => {</div>)}+ {database.mariadb?.isUseExtendedInsert && (+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Use extended inserts</div>+ <div>Yes</div>+ </div>+ )}++ {database.mariadb?.isSkipGaleraDisable && (+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Galera replication</div>+ <div>Skip disabling on restore</div>+ </div>+ )}+{!!database.mariadb?.excludeTables?.length && (<div className="mb-1 flex w-full items-center"><div className="min-w-[150px]">Exclude tables</div>diff --git a/frontend/src/features/databases/ui/show/ShowMySqlSpecificDataComponent.tsx b/frontend/src/features/databases/ui/show/ShowMySqlSpecificDataComponent.tsxindex 676d3216..05260978 100644--- a/frontend/src/features/databases/ui/show/ShowMySqlSpecificDataComponent.tsx+++ b/frontend/src/features/databases/ui/show/ShowMySqlSpecificDataComponent.tsx@@ -48,6 +48,13 @@ export const ShowMySqlSpecificDataComponent = ({ database }: Props) => {<div>{database.mysql?.isHttps ? 'Yes' : 'No'}</div></div>+ {database.mysql?.isUseExtendedInsert && (+ <div className="mb-1 flex w-full items-center">+ <div className="min-w-[150px]">Use extended inserts</div>+ <div>Yes</div>+ </div>+ )}+{!!database.mysql?.excludeTables?.length && (<div className="mb-1 flex w-full items-center"><div className="min-w-[150px]">Exclude tables</div>