Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

filer: escape LIKE wildcards in SQL list prefixes (#11757)

Chris Lua5ccefb661c6
diff --git a/weed/filer/abstract_sql/abstract_sql_store.go b/weed/filer/abstract_sql/abstract_sql_store.go
index 74a6b162f8..e358727743 100644
--- a/weed/filer/abstract_sql/abstract_sql_store.go
+++ b/weed/filer/abstract_sql/abstract_sql_store.go
@@ -390,6 +390,14 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
return doDeleteFolderChildren()
}
+var likeEscaper = strings.NewReplacer("!", "!!", "%", "!%", "_", "!_")
+
+// LikePattern escapes LIKE wildcards so the prefix matches literally;
+// '!' pairs with the ESCAPE '!' clause emitted by the SqlGenerators.
+func LikePattern(prefix string) string {
+ return likeEscaper.Replace(prefix) + "%"
+}
+
func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
db, bucket, shortPath, err := store.getTxOrDB(ctx, dirPath, true)
@@ -402,7 +410,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context,
sqlText = store.GetSqlListInclusive(bucket)
}
- rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), prefix+"%", limit+1)
+ rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), LikePattern(prefix), limit+1)
if err != nil {
return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
}
diff --git a/weed/filer/abstract_sql/abstract_sql_store_test.go b/weed/filer/abstract_sql/abstract_sql_store_test.go
index 540183e9eb..818360327c 100644
--- a/weed/filer/abstract_sql/abstract_sql_store_test.go
+++ b/weed/filer/abstract_sql/abstract_sql_store_test.go
@@ -52,6 +52,24 @@ func TestGetTxOrDBRealBucket(t *testing.T) {
}
}
+func TestLikePattern(t *testing.T) {
+ cases := []struct {
+ prefix, want string
+ }{
+ {"", "%"},
+ {"plain", "plain%"},
+ {"call_03_KNl", "call!_03!_KNl%"},
+ {"100%", "100!%%"},
+ {"a!b", "a!!b%"},
+ {`a\b`, `a\b%`},
+ }
+ for _, c := range cases {
+ if got := LikePattern(c.prefix); got != c.want {
+ t.Errorf("LikePattern(%q) = %q, want %q", c.prefix, got, c.want)
+ }
+ }
+}
+
func TestSplitPoolForKv(t *testing.T) {
cases := []struct {
maxOpen, mainOpen, kvOpen int
diff --git a/weed/filer/mysql/mysql_sql_gen.go b/weed/filer/mysql/mysql_sql_gen.go
index 52c9e710cf..049ea685a0 100644
--- a/weed/filer/mysql/mysql_sql_gen.go
+++ b/weed/filer/mysql/mysql_sql_gen.go
@@ -64,12 +64,12 @@ func (gen *SqlGenMysql) nameExpr() string {
func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
name := gen.nameExpr()
- return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
+ return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
}
func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
name := gen.nameExpr()
- return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s >= ? AND `directory` = ? AND %s LIKE ? ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
+ return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s >= ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
}
func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
diff --git a/weed/filer/mysql/mysql_sql_gen_test.go b/weed/filer/mysql/mysql_sql_gen_test.go
index 3f01965f39..d032fc7b74 100644
--- a/weed/filer/mysql/mysql_sql_gen_test.go
+++ b/weed/filer/mysql/mysql_sql_gen_test.go
@@ -65,6 +65,15 @@ func TestListSqlBinaryOrderingOnNonBinaryColumn(t *testing.T) {
}
}
+func TestListSqlEscapesLikeWildcards(t *testing.T) {
+ gen := &SqlGenMysql{}
+ for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} {
+ if !strings.Contains(got, "LIKE ? ESCAPE '!'") {
+ t.Fatalf("list query must declare the escape char matching LikePattern, got: %s", got)
+ }
+ }
+}
+
func TestIsBinaryCollation(t *testing.T) {
binary := []string{"", "binary", "utf8mb4_bin", "utf8mb3_bin", "latin1_bin", "UTF8MB4_BIN"}
for _, c := range binary {
diff --git a/weed/filer/postgres/postgres_sql_gen.go b/weed/filer/postgres/postgres_sql_gen.go
index 0f0064b3ea..9ff50d96a1 100644
--- a/weed/filer/postgres/postgres_sql_gen.go
+++ b/weed/filer/postgres/postgres_sql_gen.go
@@ -77,12 +77,12 @@ func (gen *SqlGenPostgres) nameExpr() string {
func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
name := gen.nameExpr()
- return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
+ return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
}
func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
name := gen.nameExpr()
- return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>=$2 AND directory=$3 AND %s like $4 ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
+ return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>=$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
}
func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
diff --git a/weed/filer/postgres/postgres_sql_gen_test.go b/weed/filer/postgres/postgres_sql_gen_test.go
index 1436d36a97..1cb947bd77 100644
--- a/weed/filer/postgres/postgres_sql_gen_test.go
+++ b/weed/filer/postgres/postgres_sql_gen_test.go
@@ -62,6 +62,15 @@ func TestListSqlBinaryOrderingOnNonBinaryColumn(t *testing.T) {
}
}
+func TestListSqlEscapesLikeWildcards(t *testing.T) {
+ gen := &SqlGenPostgres{}
+ for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} {
+ if !strings.Contains(got, "like $4 ESCAPE '!'") {
+ t.Fatalf("list query must declare the escape char matching LikePattern, got: %s", got)
+ }
+ }
+}
+
func TestIsByteOrderedCollation(t *testing.T) {
byteOrdered := []string{"C", "POSIX", "c", "C.UTF-8", "C.utf8"}
for _, c := range byteOrdered {
diff --git a/weed/filer/sqlite/sqlite_like_escape_test.go b/weed/filer/sqlite/sqlite_like_escape_test.go
new file mode 100644
index 0000000000..d2d39bf3ca
--- /dev/null
+++ b/weed/filer/sqlite/sqlite_like_escape_test.go
@@ -0,0 +1,75 @@
+//go:build sqlite
+
+package sqlite
+
+import (
+ "context"
+ "database/sql"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/filer"
+ "github.com/seaweedfs/seaweedfs/weed/filer/mysql"
+ "github.com/seaweedfs/seaweedfs/weed/util"
+ _ "modernc.org/sqlite"
+)
+
+func TestListDirectoryPrefixedEntriesEscapesLikeWildcards(t *testing.T) {
+ db, err := sql.Open("sqlite", "file::memory:?cache=shared")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+
+ _, err = db.Exec(`CREATE TABLE filemeta (dirhash BIGINT, name VARCHAR(1000), directory TEXT, meta BLOB, PRIMARY KEY (dirhash, name))`)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ store := &SqliteStore{}
+ store.SqlGenerator = &mysql.SqlGenMysql{}
+ store.DB = db
+
+ ctx := context.Background()
+ names := []string{
+ "call_03_KNlHPt6grkZMe4WZVwQi9710", // target: literal _ prefix
+ "callX03XKNlZZZ", // would match if _ were a wildcard
+ "call_99_other",
+ "100%_done", // % and _ in the name
+ "100aXdone", // would match prefix "100%_" if unescaped
+ "bang!file",
+ }
+ for _, n := range names {
+ entry := &filer.Entry{FullPath: util.FullPath("/dir/" + n)}
+ if err := store.InsertEntry(ctx, entry); err != nil {
+ t.Fatalf("insert %s: %v", n, err)
+ }
+ }
+
+ list := func(prefix string) []string {
+ var got []string
+ _, err := store.ListDirectoryPrefixedEntries(ctx, "/dir", "", false, 100, prefix, func(e *filer.Entry) (bool, error) {
+ got = append(got, e.Name())
+ return true, nil
+ })
+ if err != nil {
+ t.Fatalf("list %q: %v", prefix, err)
+ }
+ return got
+ }
+
+ if got := list("call_03_"); len(got) != 1 || got[0] != "call_03_KNlHPt6grkZMe4WZVwQi9710" {
+ t.Fatalf("prefix with _: got %v", got)
+ }
+ if got := list("call_"); len(got) != 2 {
+ t.Fatalf("prefix call_: got %v", got)
+ }
+ if got := list("100%_"); len(got) != 1 || got[0] != "100%_done" {
+ t.Fatalf("prefix with %%: got %v", got)
+ }
+ if got := list("bang!"); len(got) != 1 || got[0] != "bang!file" {
+ t.Fatalf("prefix with !: got %v", got)
+ }
+ if got := list(""); len(got) != len(names) {
+ t.Fatalf("empty prefix: got %v", got)
+ }
+}
diff --git a/weed/filer/ydb/ydb_queries.go b/weed/filer/ydb/ydb_queries.go
index 1118cceb64..2b57a069cc 100644
--- a/weed/filer/ydb/ydb_queries.go
+++ b/weed/filer/ydb/ydb_queries.go
@@ -55,7 +55,7 @@ const (
SELECT name, meta
FROM ` + asql.DEFAULT_TABLE + `
- WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix
+ WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix ESCAPE '!'
ORDER BY name ASC LIMIT $limit;`
listInclusiveDirectoryQuery = `
@@ -68,6 +68,6 @@ const (
SELECT name, meta
FROM ` + asql.DEFAULT_TABLE + `
- WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix
+ WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix ESCAPE '!'
ORDER BY name ASC LIMIT $limit;`
)
diff --git a/weed/filer/ydb/ydb_store.go b/weed/filer/ydb/ydb_store.go
index e5d14b5ff4..459c6d2b72 100644
--- a/weed/filer/ydb/ydb_store.go
+++ b/weed/filer/ydb/ydb_store.go
@@ -288,7 +288,7 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath
table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
table.ValueParam("$directory", types.UTF8Value(*shortDir)),
table.ValueParam("$start_name", types.UTF8Value(startFileName)),
- table.ValueParam("$prefix", types.UTF8Value(prefix+"%")),
+ table.ValueParam("$prefix", types.UTF8Value(abstract_sql.LikePattern(prefix))),
table.ValueParam("$limit", types.Uint64Value(uint64(chunkLimit))),
)