Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

filer/postgres: create filemeta table on startup via createTable config (#11229)

Chris Lucb9fcd39d2ae
diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml
index 397b938874..57b4d85db2 100644
--- a/weed/command/scaffold/filer.toml
+++ b/weed/command/scaffold/filer.toml
@@ -106,13 +106,9 @@ enableUpsert = true
upsertQuery = """INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES (?,?,?,?) AS `new` ON DUPLICATE KEY UPDATE `meta` = `new`.`meta`"""
[postgres] # or cockroachdb, YugabyteDB
-# CREATE TABLE IF NOT EXISTS filemeta (
-# dirhash BIGINT,
-# name VARCHAR(65535),
-# directory VARCHAR(65535),
-# meta bytea,
-# PRIMARY KEY (dirhash, name)
-# );
+# createTable = true # auto-create filemeta with the default schema
+# createTable = false # skip table creation (default; for restricted DB roles)
+# createTable = """CREATE TABLE IF NOT EXISTS "%s" (...)""" # custom template
enabled = false
hostname = "localhost"
port = 5432
diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go
index b35a24e8eb..74a6b162f8 100644
--- a/weed/filer/abstract_sql/abstract_sql_store.go
+++ b/weed/filer/abstract_sql/abstract_sql_store.go
@@ -32,6 +32,7 @@ type AbstractSqlStore struct {
DB *sql.DB
KvDB *sql.DB
SupportBucketTable bool
+ SkipDDL bool
dbs map[string]bool
dbsLock sync.Mutex
RetryableErrorCallback func(err error) bool
@@ -40,7 +41,7 @@ type AbstractSqlStore struct {
var _ filer.BucketAware = (*AbstractSqlStore)(nil)
func (store *AbstractSqlStore) CanDropWholeBucket() bool {
- return store.SupportBucketTable
+ return store.SupportBucketTable && !store.SkipDDL
}
func (store *AbstractSqlStore) OnBucketCreation(bucket string) {
store.dbsLock.Lock()
@@ -356,7 +357,7 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
return fmt.Errorf("findDB %s : %w", fullpath, err)
}
- if isValidBucket(bucket) && shortPath == "/" {
+ if isValidBucket(bucket) && shortPath == "/" && store.CanDropWholeBucket() {
if err = store.deleteTable(ctx, bucket); err == nil {
store.dbsLock.Lock()
delete(store.dbs, bucket)
@@ -458,15 +459,19 @@ func isValidBucket(bucket string) bool {
}
func (store *AbstractSqlStore) CreateTable(ctx context.Context, bucket string) error {
- if !store.SupportBucketTable {
+ if !store.SupportBucketTable || store.SkipDDL {
+ return nil
+ }
+ sql := store.SqlGenerator.GetSqlCreateTable(bucket)
+ if sql == "" {
return nil
}
- _, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlCreateTable(bucket))
+ _, err := store.DB.ExecContext(ctx, sql)
return err
}
func (store *AbstractSqlStore) deleteTable(ctx context.Context, bucket string) error {
- if !store.SupportBucketTable {
+ if !store.SupportBucketTable || store.SkipDDL {
return nil
}
_, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlDropTable(bucket))
diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go
index 610235f7c2..52c9e710cf 100644
--- a/weed/filer/mysql/mysql_sql_gen.go
+++ b/weed/filer/mysql/mysql_sql_gen.go
@@ -73,6 +73,9 @@ func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
}
func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
+ if gen.CreateTableSqlTemplate == "" {
+ return ""
+ }
return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
}
diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.go
index e308e26915..0f0064b3ea 100644
--- a/weed/filer/postgres/postgres_sql_gen.go
+++ b/weed/filer/postgres/postgres_sql_gen.go
@@ -28,6 +28,21 @@ var (
_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
)
+// ResolveCreateTableQuery normalizes the createTable config value. A boolean
+// true (read by viper as the string "true") selects the default template.
+// An empty or false value returns an empty string so the caller can skip
+// table creation. Any other value is treated as a custom SQL template.
+func ResolveCreateTableQuery(createTable string) string {
+ switch createTable {
+ case "true":
+ return DefaultCreateTableQuery
+ case "false", "":
+ return ""
+ default:
+ return createTable
+ }
+}
+
func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
if gen.UpsertQueryTemplate != "" {
return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
@@ -71,6 +86,9 @@ func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
}
func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
+ if gen.CreateTableSqlTemplate == "" {
+ return ""
+ }
return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
}
diff --git a/weed/filer/postgres/postgres_sql_gen_test.go b/weed/filer/postgres/postgres_sql_gen_test.go
index 72b11e6577..1436d36a97 100644
--- a/weed/filer/postgres/postgres_sql_gen_test.go
+++ b/weed/filer/postgres/postgres_sql_gen_test.go
@@ -76,3 +76,19 @@ func TestIsByteOrderedCollation(t *testing.T) {
}
}
}
+
+func TestResolveCreateTableQuery(t *testing.T) {
+ cases := []struct {
+ in, want string
+ }{
+ {"", ""},
+ {"true", DefaultCreateTableQuery},
+ {"false", ""},
+ {"CREATE TABLE custom", "CREATE TABLE custom"},
+ }
+ for _, c := range cases {
+ if got := ResolveCreateTableQuery(c.in); got != c.want {
+ t.Fatalf("ResolveCreateTableQuery(%q) = %q, want %q", c.in, got, c.want)
+ }
+ }
+}
diff --git a/weed/filer/postgres/postgres_store.go b/weed/filer/postgres/postgres_store.go
index 73926071df..fe1e574788 100644
--- a/weed/filer/postgres/postgres_store.go
+++ b/weed/filer/postgres/postgres_store.go
@@ -8,7 +8,9 @@
package postgres
import (
+ "context"
"database/sql"
+ "fmt"
"strconv"
"github.com/seaweedfs/seaweedfs/weed/filer"
@@ -39,6 +41,7 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
// poisoning on Postgres; an explicit false still disables it.
configuration.SetDefault(prefix+"enableUpsert", true)
return store.initialize(
+ configuration.GetString(prefix+"createTable"),
configuration.GetString(prefix+"upsertQuery"),
configuration.GetBool(prefix+"enableUpsert"),
configuration.GetString(prefix+"username"),
@@ -59,16 +62,17 @@ func (store *PostgresStore) Initialize(configuration util.Configuration, prefix
)
}
-func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
+func (store *PostgresStore) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = false
+ createTable = ResolveCreateTableQuery(createTable)
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
upsertQuery = DefaultUpsertQuery
}
gen := &SqlGenPostgres{
- CreateTableSqlTemplate: "",
+ CreateTableSqlTemplate: createTable,
DropTableSqlTemplate: `drop table if exists "%s"`,
UpsertQueryTemplate: upsertQuery,
}
@@ -126,6 +130,12 @@ func (store *PostgresStore) initialize(upsertQuery string, enableUpsert bool, us
return err
}
+ if createTable != "" {
+ if _, err = store.DB.ExecContext(context.Background(), gen.GetSqlCreateTable(abstract_sql.DEFAULT_TABLE)); err != nil {
+ return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
+ }
+ }
+
ConfigureListOrdering(store.DB, gen)
return nil
diff --git a/weed/filer/postgres2/postgres2_store.go b/weed/filer/postgres2/postgres2_store.go
index dfd7ce3874..d523a2abbc 100644
--- a/weed/filer/postgres2/postgres2_store.go
+++ b/weed/filer/postgres2/postgres2_store.go
@@ -68,7 +68,9 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = true
- if createTable == "" {
+ store.SkipDDL = createTable == "false"
+ createTable = postgres.ResolveCreateTableQuery(createTable)
+ if createTable == "" && !store.SkipDDL {
createTable = postgres.DefaultCreateTableQuery
}
if !enableUpsert {
@@ -135,8 +137,10 @@ func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableU
return err
}
- if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
- return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
+ if !store.SkipDDL {
+ if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
+ return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
+ }
}
postgres.ConfigureListOrdering(store.DB, gen)