A Q&A platform software for teams at any scales. Whether it's a community forum, help center, or knowledge management platform, you can always count on Apache Answer.
feat(tests): add comprehensive tests for KV storage plugin
Sonui6df436c7183f
diff --git a/plugin/plugin_test/kv_storage_test.go b/plugin/plugin_test/kv_storage_test.gonew file mode 100644index 00000000..0dcd5b46--- /dev/null+++ b/plugin/plugin_test/kv_storage_test.go@@ -0,0 +1,412 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one+ * or more contributor license agreements. See the NOTICE file+ * distributed with this work for additional information+ * regarding copyright ownership. The ASF licenses this file+ * to you under the Apache License, Version 2.0 (the+ * "License"); you may not use this file except in compliance+ * with the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing,+ * software distributed under the License is distributed on an+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ * KIND, either express or implied. See the License for the+ * specific language governing permissions and limitations+ * under the License.+ */++package plugin_test++import (+ "context"+ "fmt"+ "math/rand"+ "sync"+ "testing"+ "time"++ "github.com/apache/answer/plugin"+ "github.com/segmentfault/pacman/log"+ _ "modernc.org/sqlite"+)++var (+ testPlugin *TestKVStoragePlugin+)++// Helper functions for testing+func mustSet(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, key, value string) {+ if err := kv.Set(ctx, plugin.KVParams{Group: group, Key: key, Value: value}); err != nil {+ t.Fatalf("Failed to set %s/%s: %v", group, key, err)+ }+}++func mustGet(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, key, expected string) {+ val, err := kv.Get(ctx, plugin.KVParams{Group: group, Key: key})+ if err != nil {+ t.Fatalf("Failed to get %s/%s: %v", group, key, err)+ }+ if val != expected {+ t.Errorf("Expected '%s' for %s/%s, got '%s'", expected, group, key, val)+ }+}++func mustDel(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, key string) {+ if err := kv.Del(ctx, plugin.KVParams{Group: group, Key: key}); err != nil {+ t.Fatalf("Failed to delete %s/%s: %v", group, key, err)+ }+}++func assertNotFound(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, key string) {+ val, err := kv.Get(ctx, plugin.KVParams{Group: group, Key: key})+ if err != plugin.ErrKVKeyNotFound {+ t.Errorf("Expected ErrKVKeyNotFound for %s/%s, got: %v", group, key, err)+ }+ if val != "" {+ t.Errorf("Expected empty value for %s/%s, got: '%s'", group, key, val)+ }+}++func assertError(t *testing.T, err error, expected error, msg string) {+ if err != expected {+ t.Errorf("%s: expected %v, got %v", msg, expected, err)+ }+}++// TestKVStoragePlugin implements KVStorage interface for testing+type TestKVStoragePlugin struct {+ operator *plugin.KVOperator+}++// Info returns plugin information+func (p *TestKVStoragePlugin) Info() plugin.Info {+ return plugin.Info{+ Name: plugin.MakeTranslator("test_kv_storage_name"),+ SlugName: "test_kv_storage",+ Description: plugin.MakeTranslator("test_kv_storage_desc"),+ Author: "Answer Team",+ Version: "1.0.0",+ Link: "https://github.com/apache/answer",+ }+}++// SetOperator sets KV operator+func (p *TestKVStoragePlugin) SetOperator(operator *plugin.KVOperator) {+ p.operator = operator+}++// setupTestEnvironment sets up test environment+func setupTestEnvironment() {+ // Initialize only once+ if testPlugin != nil {+ return+ }++ // Create and register test plugin+ testPlugin = &TestKVStoragePlugin{}+ plugin.Register(testPlugin)++ // Enable plugin+ plugin.StatusManager.Enable("test_kv_storage", true)++ // Initialize plugin data, refer to plugin_common_service.go implementation+ _ = plugin.CallKVStorage(func(k plugin.KVStorage) error {+ k.SetOperator(plugin.NewKVOperator(+ testDataSource.DB,+ testDataSource.Cache,+ k.Info().SlugName,+ ))+ return nil+ })+}++// Test basic operations including CRUD and edge cases+func TestBasicOperations(t *testing.T) {+ setupTestEnvironment()+ kv := testPlugin.operator+ ctx := context.Background()++ t.Run("BasicCRUD", func(t *testing.T) {+ // Set/Get+ mustSet(t, kv, ctx, "group1", "key1", "value1")+ mustGet(t, kv, ctx, "group1", "key1", "value1")++ // Update+ mustSet(t, kv, ctx, "group1", "key1", "new_value")+ mustGet(t, kv, ctx, "group1", "key1", "new_value")++ // Delete+ mustDel(t, kv, ctx, "group1", "key1")+ assertNotFound(t, kv, ctx, "group1", "key1")++ // Group operation+ mustSet(t, kv, ctx, "group1", "key2", "value2")+ mustSet(t, kv, ctx, "group1", "key3", "value3")+ groupData, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "group1", Page: 1, PageSize: 10})+ if err != nil {+ t.Fatalf("Failed to get group data: %v", err)+ }++ // the groupData should only have key2 and key3 because key1 is deleted+ if len(groupData) != 2 {+ t.Errorf("Expected 2 items, got %d", len(groupData))+ }+ if groupData["key2"] != "value2" || groupData["key3"] != "value3" {+ t.Errorf("Unexpected group data: %v", groupData)+ }+ })++ t.Run("EdgeCases", func(t *testing.T) {+ // Empty key+ err := kv.Set(ctx, plugin.KVParams{Group: "group", Key: "", Value: "value"})+ assertError(t, err, plugin.ErrKVKeyEmpty, "Empty key test")++ // Empty group query+ _, err = kv.GetByGroup(ctx, plugin.KVParams{Group: "", Page: 1, PageSize: 10})+ assertError(t, err, plugin.ErrKVGroupEmpty, "Empty group test")++ // Non-existent key+ assertNotFound(t, kv, ctx, "non_exist_group", "non_exist_key")++ // Cache penetration protection+ key := fmt.Sprintf("non_exist_key_%d", time.Now().UnixNano())+ assertNotFound(t, kv, ctx, "cache_penetration", key)+ })++ t.Run("CacheConsistency", func(t *testing.T) {+ mustSet(t, kv, ctx, "cache_group", "cache_key", "cache_value")+ mustGet(t, kv, ctx, "cache_group", "cache_key", "cache_value")++ // Update and verify immediate consistency+ mustSet(t, kv, ctx, "cache_group", "cache_key", "updated_value")+ mustGet(t, kv, ctx, "cache_group", "cache_key", "updated_value")+ })+}++// Test transactions including rollback and nested transactions+func TestTransactions(t *testing.T) {+ setupTestEnvironment()+ kv := testPlugin.operator+ ctx := context.Background()++ t.Run("SuccessfulTransaction", func(t *testing.T) {+ err := kv.Tx(ctx, func(ctx context.Context, txKv *plugin.KVOperator) error {+ if err := txKv.Set(ctx, plugin.KVParams{Group: "tx_group", Key: "tx_key1", Value: "tx_value1"}); err != nil {+ return err+ }+ if err := txKv.Set(ctx, plugin.KVParams{Group: "tx_group", Key: "tx_key2", Value: "tx_value2"}); err != nil {+ return err+ }+ return nil+ })+ if err != nil {+ t.Fatalf("Successful transaction failed: %v", err)+ }++ mustGet(t, kv, ctx, "tx_group", "tx_key1", "tx_value1")+ mustGet(t, kv, ctx, "tx_group", "tx_key2", "tx_value2")+ })++ t.Run("TransactionRollback", func(t *testing.T) {+ err := kv.Tx(ctx, func(ctx context.Context, txKv *plugin.KVOperator) error {+ if err := txKv.Set(ctx, plugin.KVParams{Group: "tx_group", Key: "tx_key3", Value: "tx_value3"}); err != nil {+ return err+ }+ return fmt.Errorf("mock error")+ })+ if err == nil {+ t.Error("Expected transaction to fail but it succeeded")+ }++ assertNotFound(t, kv, ctx, "tx_group", "tx_key3")+ })++ t.Run("NestedTransactions", func(t *testing.T) {+ err := kv.Tx(ctx, func(ctx context.Context, txKv *plugin.KVOperator) error {+ if err := txKv.Set(ctx, plugin.KVParams{Group: "nested", Key: "key1", Value: "value1"}); err != nil {+ return err+ }++ return txKv.Tx(ctx, func(ctx context.Context, nestedKv *plugin.KVOperator) error {+ if err := nestedKv.Set(ctx, plugin.KVParams{Group: "nested", Key: "key2", Value: "value2"}); err != nil {+ return err+ }+ return fmt.Errorf("mock nested error")+ })+ })+ if err == nil {+ t.Error("Expected nested transaction to fail but it succeeded")+ }++ // Verify outer transaction also rolled back+ assertNotFound(t, kv, ctx, "nested", "key1")+ assertNotFound(t, kv, ctx, "nested", "key2")+ })+}++// Test pagination in GetByGroup+func TestPagination(t *testing.T) {+ setupTestEnvironment()+ kv := testPlugin.operator+ ctx := context.Background()+ totalItems := 25++ for i := range totalItems {+ mustSet(t, kv, ctx, "pagination", fmt.Sprintf("key%d", i), fmt.Sprintf("value%d", i))+ }++ // Test pagination+ page1, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", Page: 1, PageSize: 10})+ if err != nil {+ t.Fatalf("Failed to get page 1: %v", err)+ }+ if len(page1) != 10 {+ t.Errorf("Page 1: expected 10 items, got %d", len(page1))+ }++ page2, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", Page: 2, PageSize: 10})+ if err != nil {+ t.Fatalf("Failed to get page 2: %v", err)+ }+ if len(page2) != 10 {+ t.Errorf("Page 2: expected 10 items, got %d", len(page2))+ }++ page3, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", Page: 3, PageSize: 10})+ if err != nil {+ t.Fatalf("Failed to get page 3: %v", err)+ }+ if len(page3) != 5 {+ t.Errorf("Page 3: expected 5 items, got %d", len(page3))+ }++ // Verify different keys on different pages+ for i := range 10 {+ key := fmt.Sprintf("key%d", i)+ if _, ok := page1[key]; !ok {+ t.Errorf("Pagination test failed, key %s should be on page 1", key)+ }+ }+ for i := range 10 {+ key := fmt.Sprintf("key%d", i+10)+ if _, ok := page2[key]; !ok {+ t.Errorf("Pagination test failed, key %s should be on page 2", key)+ }+ }+}++// Test concurrent operations and performance+func TestConcurrency(t *testing.T) {+ setupTestEnvironment()+ kv := testPlugin.operator+ ctx := context.Background()++ t.Run("BasicConcurrency", func(t *testing.T) {+ parallel := 10+ var wg sync.WaitGroup+ wg.Add(parallel)++ for i := range parallel {+ go func(index int) {+ defer wg.Done()+ time.Sleep(time.Duration(rand.Intn(200)) * time.Millisecond)+ mustSet(t, kv, ctx, "concurrent", fmt.Sprintf("key%d", index), "value")+ }(i)+ }+ wg.Wait()++ // Verify results+ wg.Add(parallel)+ for i := range parallel {+ go func(index int) {+ defer wg.Done()+ time.Sleep(time.Duration(rand.Intn(200)) * time.Millisecond)+ mustGet(t, kv, ctx, "concurrent", fmt.Sprintf("key%d", index), "value")+ }(i)+ }+ wg.Wait()+ })++ t.Run("StressTest", func(t *testing.T) {+ if testing.Short() {+ t.Skip("Skipping stress test in short mode")+ }++ totalOps := 1000+ workerCount := 20+ prefix := "stress_test"+ opsPerWorker := totalOps / workerCount++ log.Info("Starting KV storage stress test...")+ startTime := time.Now()++ // Concurrent write test+ var wg sync.WaitGroup+ errorCount := int64(0)++ for w := range workerCount {+ wg.Add(1)+ go func(workerID int) {+ defer wg.Done()+ startIdx := workerID * opsPerWorker++ for i := range opsPerWorker {+ i := startIdx + i+ err := kv.Set(ctx, plugin.KVParams{+ Group: prefix,+ Key: fmt.Sprintf("key%d", i),+ Value: fmt.Sprintf("value%d", i),+ })+ if err != nil {+ log.Warnf("Write error: %v", err)+ errorCount+++ }+ }+ }(w)+ }+ wg.Wait()++ writeTime := time.Since(startTime)++ // Verify data integrity+ groupData, err := kv.GetByGroup(ctx, plugin.KVParams{Group: prefix, Page: 1, PageSize: totalOps})+ if err != nil {+ t.Fatalf("Failed to verify data: %v", err)+ }+ if len(groupData) != totalOps {+ t.Errorf("Data loss: expected %d items, got %d", totalOps, len(groupData))+ }++ // Concurrent read test+ startTime = time.Now()+ readErrors := int64(0)++ wg.Add(workerCount)+ for range workerCount {+ go func() {+ defer wg.Done()+ for range opsPerWorker {+ keyIdx := rand.Intn(totalOps)+ key := fmt.Sprintf("key%d", keyIdx)+ expected := fmt.Sprintf("value%d", keyIdx)++ val, err := kv.Get(ctx, plugin.KVParams{Group: prefix, Key: key})+ if err != nil {+ readErrors+++ } else if val != expected {+ t.Errorf("Data inconsistency: key=%s, expected=%s, got=%s", key, expected, val)+ }+ }+ }()+ }+ wg.Wait()++ readTime := time.Since(startTime)++ log.Infof("Stress test completed:")+ log.Infof(" Write: %d ops in %v (%.1f ops/sec), %d errors", totalOps, writeTime, float64(totalOps)/writeTime.Seconds(), errorCount)+ log.Infof(" Read: %d ops in %v (%.1f ops/sec), %d errors", totalOps, readTime, float64(totalOps)/readTime.Seconds(), readErrors)+ })+}diff --git a/plugin/plugin_test/plugin_main_test.go b/plugin/plugin_test/plugin_main_test.gonew file mode 100644index 00000000..4f92bebe--- /dev/null+++ b/plugin/plugin_test/plugin_main_test.go@@ -0,0 +1,207 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one+ * or more contributor license agreements. See the NOTICE file+ * distributed with this work for additional information+ * regarding copyright ownership. The ASF licenses this file+ * to you under the Apache License, Version 2.0 (the+ * "License"); you may not use this file except in compliance+ * with the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing,+ * software distributed under the License is distributed on an+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ * KIND, either express or implied. See the License for the+ * specific language governing permissions and limitations+ * under the License.+ */++package plugin_test++import (+ "context"+ "database/sql"+ "fmt"+ "os"+ "path/filepath"+ "testing"+ "time"++ "github.com/apache/answer/internal/base/data"+ "github.com/apache/answer/internal/migrations"+ "github.com/ory/dockertest/v3"+ "github.com/ory/dockertest/v3/docker"+ "github.com/segmentfault/pacman/cache"+ "github.com/segmentfault/pacman/log"+ "xorm.io/xorm"+ "xorm.io/xorm/schemas"+)++var (+ mysqlDBSetting = TestDBSetting{+ Driver: string(schemas.MYSQL),+ ImageName: "mariadb",+ ImageVersion: "10.4.7",+ ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=answer", "MYSQL_ROOT_HOST=%"},+ PortID: "3306/tcp",+ Connection: "root:root@(localhost:%s)/answer?parseTime=true", // port is not fixed, it will be got by port id+ }+ postgresDBSetting = TestDBSetting{+ Driver: string(schemas.POSTGRES),+ ImageName: "postgres",+ ImageVersion: "14",+ ENV: []string{"POSTGRES_USER=root", "POSTGRES_PASSWORD=root", "POSTGRES_DB=answer", "LISTEN_ADDRESSES='*'"},+ PortID: "5432/tcp",+ Connection: "host=localhost port=%s user=root password=root dbname=answer sslmode=disable",+ }+ sqlite3DBSetting = TestDBSetting{+ Driver: string(schemas.SQLITE),+ Connection: filepath.Join(os.TempDir(), "answer-test-data.db"),+ }+ dbSettingMapping = map[string]TestDBSetting{+ mysqlDBSetting.Driver: mysqlDBSetting,+ sqlite3DBSetting.Driver: sqlite3DBSetting,+ postgresDBSetting.Driver: postgresDBSetting,+ }+ // after all test down will execute tearDown function to clean-up+ tearDown func()+ // testDataSource used for repo testing+ testDataSource *data.Data+ testCache cache.Cache+)++func TestMain(t *testing.M) {+ dbSetting, ok := dbSettingMapping[os.Getenv("TEST_DB_DRIVER")]+ if !ok {+ // Use sqlite3 to test.+ dbSetting = dbSettingMapping[string(schemas.SQLITE)]+ }+ if dbSetting.Driver == string(schemas.SQLITE) {+ os.RemoveAll(dbSetting.Connection)+ }++ defer func() {+ if tearDown != nil {+ tearDown()+ }+ }()+ if err := initTestDataSource(dbSetting); err != nil {+ panic(err)+ }+ log.Info("init test database successfully")++ if ret := t.Run(); ret != 0 {+ panic(ret)+ }+}++type TestDBSetting struct {+ Driver string+ ImageName string+ ImageVersion string+ ENV []string+ PortID string+ Connection string+}++func initTestDataSource(dbSetting TestDBSetting) error {+ connection, imageCleanUp, err := initDatabaseImage(dbSetting)+ if err != nil {+ return err+ }+ dbSetting.Connection = connection++ dbEngine, err := initDatabase(dbSetting)+ if err != nil {+ return err+ }++ newCache, err := initCache()+ if err != nil {+ return err+ }++ newData, dbCleanUp, err := data.NewData(dbEngine, newCache)+ if err != nil {+ return err+ }+ testDataSource = newData+ testCache = newCache++ tearDown = func() {+ dbCleanUp()+ log.Info("cleanup test database successfully")+ imageCleanUp()+ log.Info("cleanup test database image successfully")+ }+ return nil+}++func initDatabaseImage(dbSetting TestDBSetting) (connection string, cleanup func(), err error) {+ // sqlite3 don't need to set up image+ if dbSetting.Driver == string(schemas.SQLITE) {+ return dbSetting.Connection, func() {+ log.Info("remove database", dbSetting.Connection)+ err = os.Remove(dbSetting.Connection)+ if err != nil {+ log.Error(err)+ }+ }, nil+ }+ pool, err := dockertest.NewPool("")+ pool.MaxWait = time.Minute * 5+ if err != nil {+ return "", nil, fmt.Errorf("could not connect to docker: %s", err)+ }++ //resource, err := pool.Run(dbSetting.ImageName, dbSetting.ImageVersion, dbSetting.ENV)+ resource, err := pool.RunWithOptions(&dockertest.RunOptions{+ Repository: dbSetting.ImageName,+ Tag: dbSetting.ImageVersion,+ Env: dbSetting.ENV,+ }, func(config *docker.HostConfig) {+ config.AutoRemove = true+ config.RestartPolicy = docker.RestartPolicy{Name: "no"}+ })+ if err != nil {+ return "", nil, fmt.Errorf("could not pull resource: %s", err)+ }++ connection = fmt.Sprintf(dbSetting.Connection, resource.GetPort(dbSetting.PortID))+ if err := pool.Retry(func() error {+ db, err := sql.Open(dbSetting.Driver, connection)+ if err != nil {+ return err+ }+ return db.Ping()+ }); err != nil {+ return "", nil, fmt.Errorf("could not connect to database: %s", err)+ }+ return connection, func() { _ = pool.Purge(resource) }, nil+}++func initDatabase(dbSetting TestDBSetting) (dbEngine *xorm.Engine, err error) {+ dataConf := &data.Database{Driver: dbSetting.Driver, Connection: dbSetting.Connection}+ dbEngine, err = data.NewDB(true, dataConf)+ if err != nil {+ return nil, fmt.Errorf("connection to database failed: %s", err)+ }+ if err := migrations.NewMentor(context.TODO(), dbEngine, &migrations.InitNeedUserInputData{+ Language: "en_US",+ SiteName: "ANSWER",+ SiteURL: "http://127.0.0.1:8080/",+ ContactEmail: "[email protected]",+ AdminName: "admin",+ AdminPassword: "admin",+ AdminEmail: "[email protected]",+ }).InitDB(); err != nil {+ return nil, fmt.Errorf("migrations init database failed: %s", err)+ }+ return dbEngine, nil+}++func initCache() (newCache cache.Cache, err error) {+ newCache, _, err = data.NewCache(&data.CacheConf{})+ return newCache, err+}