Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

filer: default the filemeta CREATE TABLE for postgres2/mysql2 when createTable is unset (#10232)

Chris Lu873705baf9fa
diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go
index 30f470d0d3..610235f7c2 100644
--- a/weed/filer/mysql/mysql_sql_gen.go
+++ b/weed/filer/mysql/mysql_sql_gen.go
@@ -22,6 +22,10 @@ type SqlGenMysql struct {
// is left for users to opt into via an explicit upsertQuery.
const DefaultUpsertQuery = "INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE `meta` = VALUES(`meta`)"
+// DefaultCreateTableQuery is used when createTable is left unset; an empty
+// template would otherwise render as %!(EXTRA ...) garbage SQL.
+const DefaultCreateTableQuery = "CREATE TABLE IF NOT EXISTS `%s` (`dirhash` BIGINT NOT NULL, `name` VARCHAR(766) NOT NULL, `directory` TEXT NOT NULL, `meta` LONGBLOB, PRIMARY KEY (`dirhash`, `name`)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
+
var (
_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
)
diff --git a/weed/filer/mysql/mysql_sql_gen_test.go b/weed/filer/mysql/mysql_sql_gen_test.go
index f72607a32a..3f01965f39 100644
--- a/weed/filer/mysql/mysql_sql_gen_test.go
+++ b/weed/filer/mysql/mysql_sql_gen_test.go
@@ -19,6 +19,17 @@ func TestDefaultUpsertQueryUsesOnDuplicateKey(t *testing.T) {
}
}
+func TestDefaultCreateTableQueryRendersValidSql(t *testing.T) {
+ gen := &SqlGenMysql{CreateTableSqlTemplate: DefaultCreateTableQuery}
+ got := gen.GetSqlCreateTable("filemeta")
+ if strings.Contains(got, "%!") {
+ t.Fatalf("template rendered to malformed SQL, got: %s", got)
+ }
+ if !strings.Contains(got, "`filemeta`") {
+ t.Fatalf("expected backticked table name, got: %s", got)
+ }
+}
+
func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
gen := &SqlGenMysql{}
got := gen.GetSqlInsert("filemeta")
diff --git a/weed/filer/mysql2/mysql2_store.go b/weed/filer/mysql2/mysql2_store.go
index 0ff58f2aca..0783398a3c 100644
--- a/weed/filer/mysql2/mysql2_store.go
+++ b/weed/filer/mysql2/mysql2_store.go
@@ -58,6 +58,9 @@ func (store *MysqlStore2) initialize(createTable, upsertQuery string, enableUpse
maxLifetimeSeconds int, interpolateParams bool) (err error) {
store.SupportBucketTable = true
+ if createTable == "" {
+ createTable = mysql.DefaultCreateTableQuery
+ }
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {
diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.go
index c16b0d9d08..e308e26915 100644
--- a/weed/filer/postgres/postgres_sql_gen.go
+++ b/weed/filer/postgres/postgres_sql_gen.go
@@ -20,6 +20,10 @@ type SqlGenPostgres struct {
// user enables upsert but does not provide their own template.
const DefaultUpsertQuery = `INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash, name) DO UPDATE SET directory=EXCLUDED.directory, meta=EXCLUDED.meta`
+// DefaultCreateTableQuery is used when createTable is left unset; an empty
+// template would otherwise render as %!(EXTRA ...) garbage SQL.
+const DefaultCreateTableQuery = `CREATE TABLE IF NOT EXISTS "%s" (dirhash BIGINT, name VARCHAR(65535), directory VARCHAR(65535), meta bytea, PRIMARY KEY (dirhash, name))`
+
var (
_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
)
diff --git a/weed/filer/postgres/postgres_sql_gen_test.go b/weed/filer/postgres/postgres_sql_gen_test.go
index f539bfb89b..72b11e6577 100644
--- a/weed/filer/postgres/postgres_sql_gen_test.go
+++ b/weed/filer/postgres/postgres_sql_gen_test.go
@@ -16,6 +16,17 @@ func TestDefaultUpsertQueryIsConflictSafe(t *testing.T) {
}
}
+func TestDefaultCreateTableQueryRendersValidSql(t *testing.T) {
+ gen := &SqlGenPostgres{CreateTableSqlTemplate: DefaultCreateTableQuery}
+ got := gen.GetSqlCreateTable("filemeta")
+ if strings.Contains(got, "%!") {
+ t.Fatalf("template rendered to malformed SQL, got: %s", got)
+ }
+ if !strings.Contains(got, `"filemeta"`) {
+ t.Fatalf("expected quoted table name, got: %s", got)
+ }
+}
+
func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
gen := &SqlGenPostgres{}
got := gen.GetSqlInsert("filemeta")
diff --git a/weed/filer/postgres2/postgres2_store.go b/weed/filer/postgres2/postgres2_store.go
index 3a08118af0..e846bdf9fe 100644
--- a/weed/filer/postgres2/postgres2_store.go
+++ b/weed/filer/postgres2/postgres2_store.go
@@ -63,6 +63,9 @@ func (store *PostgresStore2) Initialize(configuration util.Configuration, prefix
func (store *PostgresStore2) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
store.SupportBucketTable = true
+ if createTable == "" {
+ createTable = postgres.DefaultCreateTableQuery
+ }
if !enableUpsert {
upsertQuery = ""
} else if upsertQuery == "" {