SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer/postgres: create filemeta table on startup via createTable config (#11229)
Chris Lucb9fcd39d2ae
diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.tomlindex 397b938874..57b4d85db2 100644--- a/weed/command/scaffold/filer.toml+++ b/weed/command/scaffold/filer.toml@@ -106,13 +106,9 @@ enableUpsert = trueupsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""[postgres] # or cockroachdb, YugabyteDB-# CREATE TABLE IF NOT EXISTS filemeta (-# dirhash BIGINT,-# name VARCHAR(65535),-# directory VARCHAR(65535),-# meta bytea,-# PRIMARY KEY (dirhash, name)-# );+# createTable = true # auto-create filemeta with the default schema+# createTable = false # skip table creation (default; for restricted DB roles)+# createTable = """CREATE TABLE IF NOT EXISTS "%s" (...)""" # custom templateenabled = falsehostname = "localhost"port = 5432diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.goindex b35a24e8eb..74a6b162f8 100644--- a/weed/filer/abstract_sql/abstract_sql_store.go+++ b/weed/filer/abstract_sql/abstract_sql_store.go@@ -32,6 +32,7 @@ type AbstractSqlStore struct {DB *sql.DBKvDB *sql.DBSupportBucketTable bool+ SkipDDL booldbs map[string]booldbsLock sync.MutexRetryableErrorCallback func(err error) bool@@ -40,7 +41,7 @@ type AbstractSqlStore struct {var _ filer.BucketAware = (*AbstractSqlStore)(nil)func (store *AbstractSqlStore) CanDropWholeBucket() bool {- return store.SupportBucketTable+ return store.SupportBucketTable && !store.SkipDDL}func (store *AbstractSqlStore) OnBucketCreation(bucket string) {store.dbsLock.Lock()@@ -356,7 +357,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpatreturn fmt.Errorf("findDB %s : %w", fullpath, err)}- if isValidBucket(bucket) && shortPath == "/" {+ if isValidBucket(bucket) && shortPath == "/" && store.CanDropWholeBucket() {if err = store.deleteTable(ctx, bucket); err == nil {store.dbsLock.Lock()delete(store.dbs, bucket)@@ -458,15 +459,19 @@ func isValidBucket(bucket string) bool {}func (store *AbstractSqlStore) CreateTable(ctx context.Context, bucket string) error {- if !store.SupportBucketTable {+ if !store.SupportBucketTable || store.SkipDDL {+ return nil+ }+ sql := store.SqlGenerator.GetSqlCreateTable(bucket)+ if sql == "" {return nil}- _, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlCreateTable(bucket))+ _, err := store.DB.ExecContext(ctx, sql)return err}func (store *AbstractSqlStore) deleteTable(ctx context.Context, bucket string) error {- if !store.SupportBucketTable {+ if !store.SupportBucketTable || store.SkipDDL {return nil}_, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlDropTable(bucket))diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.goindex 610235f7c2..52c9e710cf 100644--- a/weed/filer/mysql/mysql_sql_gen.go+++ b/weed/filer/mysql/mysql_sql_gen.go@@ -73,6 +73,9 @@ func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {}func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {+ if gen.CreateTableSqlTemplate == "" {+ return ""+ }return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)}diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.goindex e308e26915..0f0064b3ea 100644--- a/weed/filer/postgres/postgres_sql_gen.go+++ b/weed/filer/postgres/postgres_sql_gen.go@@ -28,6 +28,21 @@ var (_ = abstract_sql.SqlGenerator(&SqlGenPostgres{}))+// ResolveCreateTableQuery normalizes the createTable config value. A boolean+// true (read by viper as the string "true") selects the default template.+// An empty or false value returns an empty string so the caller can skip+// table creation. Any other value is treated as a custom SQL template.+func ResolveCreateTableQuery(createTable string) string {+ switch createTable {+ case "true":+ return DefaultCreateTableQuery+ case "false", "":+ return ""+ default:+ return createTable+ }+}+func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {if gen.UpsertQueryTemplate != "" {return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)@@ -71,6 +86,9 @@ func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {}func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {+ if gen.CreateTableSqlTemplate == "" {+ return ""+ }return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)}diff --git a/weed/filer/postgres/postgres_sql_gen_test.go b/weed/filer/postgres/postgres_sql_gen_test.goindex 72b11e6577..1436d36a97 100644--- a/weed/filer/postgres/postgres_sql_gen_test.go+++ b/weed/filer/postgres/postgres_sql_gen_test.go@@ -76,3 +76,19 @@ func TestIsByteOrderedCollation(t *testing.T) {}}}++func TestResolveCreateTableQuery(t *testing.T) {+ cases := []struct {+ in, want string+ }{+ {"", ""},+ {"true", DefaultCreateTableQuery},+ {"false", ""},+ {"CREATE TABLE custom", "CREATE TABLE custom"},+ }+ for _, c := range cases {+ if got := ResolveCreateTableQuery(c.in); got != c.want {+ t.Fatalf("ResolveCreateTableQuery(%q) = %q, want %q", c.in, got, c.want)+ }+ }+}diff --git a/weed/filer/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.goindex 73926071df..fe1e574788 100644--- a/weed/filer/postgres/postgres_store.go+++ b/weed/filer/postgres/postgres_store.go@@ -8,7 +8,9 @@package postgresimport (+ "context""database/sql"+ "fmt""strconv""github.com/seaweedfs/seaweedfs/weed/filer"@@ -39,6 +41,7 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix// poisoning on Postgres; an explicit false still disables it.configuration.SetDefault(prefix+"enableUpsert", true)return store.initialize(+ configuration.GetString(prefix+"createTable"),configuration.GetString(prefix+"upsertQuery"),configuration.GetBool(prefix+"enableUpsert"),configuration.GetString(prefix+"username"),@@ -59,16 +62,17 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix)}-func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {+func (store *PostgresStore) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {store.SupportBucketTable = false+ createTable = ResolveCreateTableQuery(createTable)if !enableUpsert {upsertQuery = ""} else if upsertQuery == "" {upsertQuery = DefaultUpsertQuery}gen := &SqlGenPostgres{- CreateTableSqlTemplate: "",+ CreateTableSqlTemplate: createTable,DropTableSqlTemplate: `drop table if exists "%s"`,UpsertQueryTemplate: upsertQuery,}@@ -126,6 +130,12 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, usreturn err}+ if createTable != "" {+ if _, err = store.DB.ExecContext(context.Background(), gen.GetSqlCreateTable(abstract_sql.DEFAULT_TABLE)); err != nil {+ return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)+ }+ }+ConfigureListOrdering(store.DB, gen)return nildiff --git a/weed/filer/postgres2/postgres2_store.go b/weed/filer/postgres2/postgres2_store.goindex dfd7ce3874..d523a2abbc 100644--- a/weed/filer/postgres2/postgres2_store.go+++ b/weed/filer/postgres2/postgres2_store.go@@ -68,7 +68,9 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefixfunc (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {store.SupportBucketTable = true- if createTable == "" {+ store.SkipDDL = createTable == "false"+ createTable = postgres.ResolveCreateTableQuery(createTable)+ if createTable == "" && !store.SkipDDL {createTable = postgres.DefaultCreateTableQuery}if !enableUpsert {@@ -135,8 +137,10 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUreturn err}- if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {- return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)+ if !store.SkipDDL {+ if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {+ return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)+ }}postgres.ConfigureListOrdering(store.DB, gen)