Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor!: remove Custom API Endpoint secret manager integration (#17832)
Danny Xu201f9ed8a99b
diff --git a/backend/api/v1/instance_service.go b/backend/api/v1/instance_service.goindex 5d1092371b..4d72e409d2 100644--- a/backend/api/v1/instance_service.go+++ b/backend/api/v1/instance_service.go@@ -20,7 +20,6 @@ import ("github.com/bytebase/bytebase/backend/component/dbfactory""github.com/bytebase/bytebase/backend/component/iam""github.com/bytebase/bytebase/backend/component/sampleinstance"- "github.com/bytebase/bytebase/backend/component/secret""github.com/bytebase/bytebase/backend/component/state""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -465,10 +464,7 @@ func (s *InstanceService) checkDataSource(instance *store.InstanceMessage, dataSif dataSource.GetExternalSecret() != nil {return missingFeatureError}- if ok, _ := secret.GetExternalSecretURL(dataSource.GetPassword()); !ok {- return nil- }- return missingFeatureError+ return nil}// Validate extra connection parameters for MySQL-based enginesdiff --git a/backend/component/secret/secret.go b/backend/component/secret/secret.goindex 0d30ed9291..a56b059a65 100644--- a/backend/component/secret/secret.go+++ b/backend/component/secret/secret.go@@ -3,19 +3,12 @@ package secretimport ("context"- "encoding/base64"- "encoding/json"- "net/http"- "net/url"- "strings""github.com/pkg/errors"storepb "github.com/bytebase/bytebase/backend/generated-go/store")-var secretCache = make(map[string]string)-// ReplaceExternalSecret replaces the secret with external secret.func ReplaceExternalSecret(ctx context.Context, secret string, externalSecret *storepb.DataSourceExternalSecret) (string, error) {if externalSecret != nil {@@ -32,67 +25,5 @@ func ReplaceExternalSecret(ctx context.Context, secret string, externalSecret *s}}- return getSecretFromText(secret)-}--func getSecretFromText(secret string) (string, error) {- ok, secretURL := GetExternalSecretURL(secret)- if !ok {- return secret, nil- }- if v, ok := secretCache[secretURL]; ok {- return v, nil- }- secret, err := getSecretFromURL(secretURL)- if err != nil {- return "", err- }- secretCache[secretURL] = secret- return secret, err-}--// GetExternalSecretURL gets external secret URL from secret.-func GetExternalSecretURL(secret string) (bool, string) {- if !strings.HasPrefix(secret, "{{") {- return false, ""- }- if !strings.HasSuffix(secret, "}}") {- return false, ""- }- s := secret[2 : len(secret)-2]- if _, err := url.ParseRequestURI(s); err != nil {- return false, ""- }- return true, s-}--type payload struct {- Data string `json:"data"`-}--type accessResponse struct {- Payload payload `json:"payload"`-}--func getSecretFromURL(secretURL string) (string, error) {- response, err := http.Get(secretURL)- if err != nil {- return "", errors.Wrapf(err, "failed to get secret from %q", secretURL)- }- defer response.Body.Close()-- if response.StatusCode != http.StatusOK {- return "", errors.Wrapf(err, "failed to get secret from %q status %v", secretURL, response.StatusCode)- }-- var r accessResponse- decoder := json.NewDecoder(response.Body)- if err := decoder.Decode(&r); err != nil {- return "", errors.Wrapf(err, "failed to decode JSON response")- }- secret, err := base64.StdEncoding.DecodeString(r.Payload.Data)- if err != nil {- return "", errors.Wrapf(err, "failed to base64 decode secret")- }- return string(secret), nil+ return secret, nil}diff --git a/backend/component/secret/secret_manager.py b/backend/component/secret/secret_manager.pydeleted file mode 100644index 4ff50fcfe6..0000000000--- a/backend/component/secret/secret_manager.py+++ /dev/null@@ -1,31 +0,0 @@-import base64-import http.server-import json-from http.server import BaseHTTPRequestHandler--# We can use {{http://localhost:1137/data}} for testing external secret manager.--class JSONHandler(BaseHTTPRequestHandler):- def do_GET(self):- if self.path == '/data':- password = 'ktPYr0bQixOHzCux'- response_data = {- 'payload': {- 'data': base64.b64encode(password.encode('utf-8')).decode('utf-8'),- },- }- response_json = json.dumps(response_data)-- self.send_response(200)- self.send_header('Content-Type', 'application/json')- self.end_headers()- self.wfile.write(response_json.encode('utf-8'))- else:- self.send_response(404)- self.wfile.write(b'Not Found')--server_address = ('', 1137)-httpd = http.server.HTTPServer(server_address, JSONHandler)--print("Server is running on port 1137...")-httpd.serve_forever()\ No newline at end of filediff --git a/backend/tests/data_source_test.go b/backend/tests/data_source_test.goindex 179687bc13..e216f493a1 100644--- a/backend/tests/data_source_test.go+++ b/backend/tests/data_source_test.go@@ -2,8 +2,6 @@ package testsimport ("context"- "fmt"- "net/http""testing""connectrpc.com/connect"@@ -11,7 +9,6 @@ import ("google.golang.org/protobuf/types/known/fieldmaskpb"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"- "github.com/bytebase/bytebase/backend/tests/fake")func TestDataSource(t *testing.T) {@@ -127,54 +124,3 @@ func TestDataSource(t *testing.T) {}))a.NoError(err)}--func TestExternalSecretManager(t *testing.T) {- t.Parallel()- a := require.New(t)- ctx := context.Background()- ctl := &controller{}- ctx, err := ctl.StartServerWithExternalPg(ctx)- a.NoError(err)- defer ctl.Close(ctx)-- pgContainer, err := getPgContainer(ctx)- defer func() {- pgContainer.Close(ctx)- }()- a.NoError(err)-- smPort := getTestPort()- sm := fake.NewSecretManager(smPort)- go func() {- if err := sm.Run(); err != http.ErrServerClosed {- a.NoError(err)- }- }()- defer sm.Close()-- pgDB := pgContainer.db- err = pgDB.Ping()- a.NoError(err)-- _, err = pgDB.Exec("CREATE USER bytebase WITH ENCRYPTED PASSWORD 'bytebase'")- a.NoError(err)- _, err = pgDB.Exec("ALTER USER bytebase WITH SUPERUSER")- a.NoError(err)-- secretURL := fmt.Sprintf("{{http://localhost:%d/secrets/hello-secret-id:access}}", smPort)- instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{- InstanceId: generateRandomString("instance"),- Instance: &v1pb.Instance{- Title: "pgInstance",- Engine: v1pb.Engine_POSTGRES,- Environment: stringPtr("environments/prod"),- Activation: true,- DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: pgContainer.host, Port: pgContainer.port, Username: "bytebase", Password: secretURL, Id: "admin"}},- },- }))- a.NoError(err)- instance := instanceResp.Msg-- err = ctl.createDatabaseV2(ctx, ctl.project, instance, nil /* environment */, databaseName, "bytebase")- a.NoError(err)-}diff --git a/backend/tests/fake/fake.go b/backend/tests/fake/fake.godeleted file mode 100644index 4d666f1c2d..0000000000--- a/backend/tests/fake/fake.go+++ /dev/null@@ -1,20 +0,0 @@-// Package fake provides the fake implementation for several dependency services.-package fake--import (- "github.com/labstack/echo/v4"- "github.com/labstack/echo/v4/middleware"-)--func newEchoServer() *echo.Echo {- e := echo.New()- e.Debug = false- e.HideBanner = true- e.HidePort = true-- // Middleware- e.Use(middleware.Logger())- e.Use(middleware.Recover())-- return e-}diff --git a/backend/tests/fake/license b/backend/tests/fake/licensedeleted file mode 100644index 97f9794097..0000000000--- a/backend/tests/fake/license+++ /dev/null@@ -1 +0,0 @@-eyJhbGciOiJSUzI1NiIsImtpZCI6InYxIiwidHlwIjoiSldUIn0.eyJpbnN0YW5jZUNvdW50Ijo5OTksInRyaWFsaW5nIjpmYWxzZSwicGxhbiI6IkVOVEVSUFJJU0UiLCJvcmdOYW1lIjoiYmIiLCJhdWQiOiJiYi5saWNlbnNlIiwiZXhwIjo3OTc0OTc5MjAwLCJpYXQiOjE2NjM2Njc1NjEsImlzcyI6ImJ5dGViYXNlIiwic3ViIjoiMDAwMDEwMDAuIn0.JjYCMeAAMB9FlVeDFLdN3jvFcqtPsbEzaIm1YEDhUrfekthCbIOeX_DB2Bg2OUji3HSX5uDvG9AkK4Gtrc4gLMPI3D5mk3L-6wUKZ0L4REztS47LT4oxVhpqPQayYa9lKJB1YoHaqeMV4Z5FXeOXwuACoELznlwpT6pXo9xXm_I6QwQiO7-zD83XOTO4PRjByc-q3GKQu_64zJMIKiCW0I8a3GvrdSnO7jUuYU1KPmCuk0ZRq3I91m29LTo478BMST59HqCLj1GGuCKtR3SL_376XsZfUUM0iSAur5scg99zNGWRj-sUo05wbAadYx6V6TKaWrBUi_8_0RnJyP5gbA\ No newline at end of filediff --git a/backend/tests/fake/secret_manager.go b/backend/tests/fake/secret_manager.godeleted file mode 100644index 32e8b59104..0000000000--- a/backend/tests/fake/secret_manager.go+++ /dev/null@@ -1,51 +0,0 @@-package fake--import (- "fmt"- "net"- "net/http"-- "github.com/labstack/echo/v4"-)--// SecretManager is a fake implementation of secret manager.-type SecretManager struct {- port int- echo *echo.Echo- client *http.Client-}--// NewSecretManager creates a new fake implementation of secret manager.-func NewSecretManager(port int) *SecretManager {- e := newEchoServer()- sm := &SecretManager{- port: port,- echo: e,- client: &http.Client{},- }- e.GET("/secrets/hello-secret-id:access", func(c echo.Context) error {- // Base64 format of "bytebase".- data := `{- "payload": {- "data": "Ynl0ZWJhc2U="- }- }`- return c.String(http.StatusOK, data)- })- return sm-}--// Run starts the secret manager server.-func (sm *SecretManager) Run() error {- return sm.echo.Start(fmt.Sprintf(":%d", sm.port))-}--// Close shuts down the secret manager server.-func (sm *SecretManager) Close() error {- return sm.echo.Close()-}--// ListenerAddr returns the secret manager server listener address.-func (sm *SecretManager) ListenerAddr() net.Addr {- return sm.echo.ListenerAddr()-}diff --git a/frontend/src/components/InstanceForm/DataSourceSection/DataSourceForm.vue b/frontend/src/components/InstanceForm/DataSourceSection/DataSourceForm.vueindex 21e2c77ece..35ec69d41d 100644--- a/frontend/src/components/InstanceForm/DataSourceSection/DataSourceForm.vue+++ b/frontend/src/components/InstanceForm/DataSourceSection/DataSourceForm.vue@@ -310,18 +310,6 @@<label class="textlabel block">{{ $t("common.password") }}</label>- <div v-if="!hideAdvancedFeatures" class="flex space-x-2 text-sm">- <div class="textinfolabel">- {{ $t("instance.password-type.password-tip") }}- </div>- <LearnMoreLink- url="https://docs.bytebase.com/get-started/connect/overview/#use-secret-manager?source=console"- class="ml-1 text-sm"- />- <FeatureBadge- :feature="PlanFeature.FEATURE_EXTERNAL_SECRET_MANAGER"- />- </div><div class="mt-2"><NCheckboxv-if="!isCreating && allowUsingEmptyPassword"diff --git a/frontend/src/locales/en-US.json b/frontend/src/locales/en-US.jsonindex 4286e2adfe..3c08784aa6 100644--- a/frontend/src/locales/en-US.json+++ b/frontend/src/locales/en-US.json@@ -1362,7 +1362,6 @@"password-write-only": "YOUR_DB_PWD - write only","password-type": {"password": "Password",- "password-tip": "You can also specify external secret manage endpoint","google-iam": "Google Cloud SQL IAM","aws-iam": "AWS RDS IAM","azure-iam": "Azure IAM",diff --git a/frontend/src/locales/es-ES.json b/frontend/src/locales/es-ES.jsonindex 4cd9a054bf..f170751344 100644--- a/frontend/src/locales/es-ES.json+++ b/frontend/src/locales/es-ES.json@@ -1362,7 +1362,6 @@"password-write-only": "SU_DB_PWD - solo escritura","password-type": {"password": "Contraseña",- "password-tip": "También puede especificar un punto final de gestión de secreto externo","google-iam": "Google Cloud SQL IAM","aws-iam": "AWS RDS IAM","azure-iam": "Azure IAM",diff --git a/frontend/src/locales/ja-JP.json b/frontend/src/locales/ja-JP.jsonindex 06de27044f..0bc9f03ac6 100644--- a/frontend/src/locales/ja-JP.json+++ b/frontend/src/locales/ja-JP.json@@ -1362,7 +1362,6 @@"password-write-only": "YOUR_DB_PWD - 書き込み専用","password-type": {"password": "パスワード",- "password-tip": "外部のパスワード管理サービスのエンドポイントを指定することもできます","google-iam": "Google Cloud SQL IAM","aws-iam": "AWS RDS IAM","azure-iam": "Azure IAM",diff --git a/frontend/src/locales/vi-VN.json b/frontend/src/locales/vi-VN.jsonindex fac74332c7..e70fcf141d 100644--- a/frontend/src/locales/vi-VN.json+++ b/frontend/src/locales/vi-VN.json@@ -1362,7 +1362,6 @@"password-write-only": "YOUR_DB_PWD - chỉ ghi","password-type": {"password": "Mật khẩu",- "password-tip": "Bạn cũng có thể chỉ định điểm cuối quản lý bí mật bên ngoài","google-iam": "Google Cloud SQL IAM","aws-iam": "AWS RDS IAM","azure-iam": "Azure IAM",diff --git a/frontend/src/locales/zh-CN.json b/frontend/src/locales/zh-CN.jsonindex 3e6163acc3..18af259286 100644--- a/frontend/src/locales/zh-CN.json+++ b/frontend/src/locales/zh-CN.json@@ -1362,7 +1362,6 @@"password-write-only": "YOUR_DB_PWD - 仅写入","password-type": {"password": "密码",- "password-tip": "您也可以指定外部的密码管理服务 endpoint","google-iam": "Google Cloud SQL IAM","aws-iam": "AWS RDS IAM","azure-iam": "Azure IAM",