Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(api): redact audit payloads from proto annotations (BYT-10090) (#21226)

Danny Xu37339d73c8a7
diff --git a/backend/api/v1/audit.go b/backend/api/v1/audit.go
index f3bb93b7a4..7deeaddbb9 100644
--- a/backend/api/v1/audit.go
+++ b/backend/api/v1/audit.go
@@ -11,7 +11,6 @@ import (
"connectrpc.com/connect"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/durationpb"
@@ -29,15 +28,14 @@ import (
"github.com/bytebase/bytebase/backend/store"
)
-// used for replacing sensitive fields.
-var (
- maskedString string
-)
-
const (
// maxAuditPayloadChars is the maximum characters for request/response payloads in stdout logs.
// Set to 100KB (102400 chars) to match AWS CloudTrail industry standard for audit logs.
maxAuditPayloadChars = 102400
+
+ // unencodablePayload stands in for a payload protojson refused, so the row
+ // records that something was there and could not be written down.
+ unencodablePayload = `{"_bytebase":"payload could not be encoded"}`
)
// AuditInterceptor is the v1 audit interceptor for gRPC server.
@@ -88,10 +86,12 @@ func (in *AuditInterceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc
// and a refused agent is interesting either way. Only the internal MCP
// chain runs the gate, so the public chain is unaffected.
//
- // Recording a request that was never recorded is why getRequestString
- // grew redactors below: a denial must not transcribe the secret it
- // refused. The rule for a new one is the population above, not the four
- // named methods.
+ // Recording a request that was never recorded is why redaction has to
+ // cover more than the audited RPCs: a denial must not transcribe the
+ // secret it refused. Since redaction is driven by the field annotation
+ // rather than by a per-RPC redactor, a gate-refused method is covered
+ // the moment its fields are annotated — the population is the one above,
+ // not the four named methods.
mcpPolicyDenied := false
ctx = common.WithSetMCPPolicyDenied(ctx, func() { mcpPolicyDenied = true })
@@ -237,14 +237,8 @@ func (in *AuditInterceptor) createAuditLog(ctx context.Context, e *auditEntry) e
return nil
}
- requestString, err := getRequestString(e.request)
- if err != nil {
- return errors.Wrapf(err, "failed to get request string")
- }
- responseString, err := getResponseString(e.response)
- if err != nil {
- return errors.Wrapf(err, "failed to get response string")
- }
+ requestString := marshalAuditPayload(e.request)
+ responseString := marshalAuditPayload(e.response)
var user string
if u, ok := GetUserFromContext(ctx); ok {
@@ -349,6 +343,23 @@ func (in *AuditInterceptor) createAuditLog(ctx context.Context, e *auditEntry) e
}
}
+ // service_data and status.details are the only fields on the row that need
+ // redacting, and neither passes marshalAuditPayload. Both carry
+ // google.protobuf.Any values whose packed type the descriptor cannot see:
+ // service_data holds a before-image Setting or an IAM policy delta, and
+ // status.details holds whatever a failing handler attached.
+ //
+ // The rest of the row is left alone deliberately. request_metadata and
+ // mcp_delegation are ordinary store messages with nothing annotated under
+ // them, and request and response are already redacted strings.
+ // TestAuditRowNeedsNoRedactionBeyondTheAnyPayloads fails if that stops
+ // being true.
+ //
+ // Computed once: none of the three varies by parent.
+ serviceData := redactAuditServiceData(e.serviceData)
+ auditStatus := redactAuditStatus(convertErrToStatus(e.rerr))
+ mcpDelegation := mcpDelegationFromAuthContext(authContext)
+
createAuditLogCtx := context.WithoutCancel(ctx)
for _, ap := range parents {
resource := getRequestResource(e.request, e.method)
@@ -373,11 +384,11 @@ func (in *AuditInterceptor) createAuditLog(ctx context.Context, e *auditEntry) e
User: user,
Request: requestString,
Response: responseString,
- Status: convertErrToStatus(e.rerr),
+ Status: auditStatus,
Latency: durationpb.New(e.latency),
- ServiceData: e.serviceData,
+ ServiceData: serviceData,
RequestMetadata: requestMetadata,
- McpDelegation: mcpDelegationFromAuthContext(authContext),
+ McpDelegation: mcpDelegation,
}
// Resolve workspace for audit log.
workspaceIDForAudit := ap.auditWorkspaceID
@@ -469,7 +480,7 @@ func logAuditToStdout(ctx context.Context, p *storepb.AuditLog) {
attrs = append(attrs, mcpDelegationAttrs(p.McpDelegation)...)
// Include request payload (truncated to 100KB for log manageability)
- // Request is already redacted for sensitive data by getRequestString()
+ // Request is already redacted for sensitive data by marshalAuditPayload()
if p.Request != "" {
request := p.Request
if truncated, wasTruncated := common.TruncateString(p.Request, maxAuditPayloadChars); wasTruncated {
@@ -479,7 +490,7 @@ func logAuditToStdout(ctx context.Context, p *storepb.AuditLog) {
}
// Include response payload (truncated to 100KB for log manageability)
- // Response is already redacted for sensitive data by getResponseString()
+ // Response is already redacted for sensitive data by marshalAuditPayload()
if p.Response != "" {
response := p.Response
if truncated, wasTruncated := common.TruncateString(p.Response, maxAuditPayloadChars); wasTruncated {
@@ -563,843 +574,44 @@ func getRequestResource(request any, method string) string {
return getResourceFromSingleRequest(message.ProtoReflect(), shortMethod)
}
-func getRequestString(request any) (string, error) {
- m := func() protoreflect.ProtoMessage {
- if request == nil || reflect.ValueOf(request).IsNil() {
- return nil
- }
- if redacted, ok := redactMCPDenialRequest(request); ok {
- return redacted
- }
- switch r := request.(type) {
- case *v1pb.ExportRequest:
- return redactExportRequest(r)
- case *v1pb.CreateUserRequest:
- return redactCreateUserRequest(r)
- case *v1pb.UpdateUserRequest:
- return redactUpdateUserRequest(r)
- case *v1pb.LoginRequest:
- return redactLoginRequest(r)
- case *v1pb.SignupRequest:
- return redactSignupRequest(r)
- case *v1pb.ExchangeTokenRequest:
- return redactExchangeTokenRequest(r)
- case *v1pb.ResetPasswordRequest:
- return redactResetPasswordRequest(r)
- case *v1pb.UploadLicenseRequest:
- return redactUploadLicenseRequest(r)
- case *v1pb.CreateProjectRequest:
- r = proto.CloneOf(r)
- r.Project = redactProject(r.Project)
- return r
- case *v1pb.UpdateProjectRequest:
- r = proto.CloneOf(r)
- r.Project = redactProject(r.Project)
- return r
- case *v1pb.AddWebhookRequest:
- r = proto.CloneOf(r)
- r.Webhook = redactWebhook(r.Webhook)
- return r
- case *v1pb.UpdateWebhookRequest:
- r = proto.CloneOf(r)
- r.Webhook = redactWebhook(r.Webhook)
- return r
- case *v1pb.RemoveWebhookRequest:
- r = proto.CloneOf(r)
- r.Webhook = redactWebhook(r.Webhook)
- return r
- case *v1pb.CreateReleaseRequest:
- r = proto.CloneOf(r)
- r.Release = redactRelease(r.Release)
- return r
- case *v1pb.UpdateReleaseRequest:
- r = proto.CloneOf(r)
- r.Release = redactRelease(r.Release)
- return r
- case *v1pb.CreateSavedQueryRequest:
- r = proto.CloneOf(r)
- r.SavedQuery = redactSavedQuery(r.SavedQuery)
- return r
- case *v1pb.UpdateSavedQueryRequest:
- r = proto.CloneOf(r)
- r.SavedQuery = redactSavedQuery(r.SavedQuery)
- return r
-
- case *v1pb.CreateInstanceRequest:
- r = proto.CloneOf(r)
- r.Instance = redactInstance(r.Instance)
- return r
- case *v1pb.UpdateInstanceRequest:
- r = proto.CloneOf(r)
- r.Instance = redactInstance(r.Instance)
- return r
- case *v1pb.BatchUpdateInstancesRequest:
- r = proto.CloneOf(r)
- for _, ur := range r.Requests {
- ur.Instance = redactInstance(ur.Instance)
- }
- return r
- case *v1pb.AddDataSourceRequest:
- r = proto.CloneOf(r)
- r.DataSource = redactDataSource(r.DataSource)
- return r
- case *v1pb.UpdateDataSourceRequest:
- r = proto.CloneOf(r)
- r.DataSource = redactDataSource(r.DataSource)
- return r
- case *v1pb.RemoveDataSourceRequest:
- r = proto.CloneOf(r)
- r.DataSource = redactDataSource(r.DataSource)
- return r
- case *v1pb.UpdateSettingRequest:
- r = proto.CloneOf(r)
- r.Setting = redactSetting(r.Setting)
- return r
- case *v1pb.CreateIdentityProviderRequest:
- r = proto.CloneOf(r)
- r.IdentityProvider = redactIdentityProvider(r.IdentityProvider)
- return r
- case *v1pb.CreateSheetRequest:
- // The clone is already private; drop the content in place instead
- // of cloning the potentially large sheet a second time.
- r = proto.CloneOf(r)
- if r.Sheet != nil {
- r.Sheet.Content = nil
- }
- return r
- case *v1pb.BatchCreateSheetsRequest:
- r = proto.CloneOf(r)
- for _, cr := range r.Requests {
- if cr.Sheet != nil {
- cr.Sheet.Content = nil
- }
- }
- return r
- case *v1pb.UpdateIdentityProviderRequest:
- r = proto.CloneOf(r)
- r.IdentityProvider = redactIdentityProvider(r.IdentityProvider)
- return r
- default:
- if p, ok := r.(protoreflect.ProtoMessage); ok {
- return p
- }
- return nil
- }
- }()
- if m == nil {
- return "", nil
- }
-
- b, err := protojson.Marshal(m)
- if err != nil {
- return "", err
- }
- return string(b), nil
-}
-
-// redactMCPDenialRequest handles the requests that reach the audit log only
-// because MCP denials are recorded. The methods themselves are unaudited; what
-// arrives here is a refusal, and a refusal that transcribed the secret it
-// refused would be worse than the silence it replaced. Every secret masked here
-// is already masked on a sibling method that has always been audited.
-//
-// It is a function rather than six more arms of getRequestString's switch
-// because it is one population with one reason for existing, and that reason is
-// worth stating once where a reader can find it.
-//
-// Which six is not a judgment to repeat by eye — the first pass of it missed
-// ListInstanceDatabase, whose request carries a whole Instance and therefore
-// every data-source credential the product has. The population is derived from
-// the descriptors instead, and TestLintDenialRequestsAreReviewedForRedaction
-// fails when a method joins it.
-//
-// The bool reports whether this was one of them, so an unrelated request falls
-// through to the main switch untouched.
-func redactMCPDenialRequest(request any) (protoreflect.ProtoMessage, bool) {
- switch r := request.(type) {
- case *v1pb.TestWebhookRequest:
- r = proto.CloneOf(r)
- r.Webhook = redactWebhook(r.Webhook)
- return r, true
- case *v1pb.TestEmailSettingRequest:
- r = proto.CloneOf(r)
- r.EmailSetting = redactEmailSetting(r.EmailSetting)
- return r, true
- case *v1pb.TestIdentityProviderRequest:
- return redactTestIdentityProviderRequest(r), true
- case *v1pb.SwitchWorkspaceRequest:
- return redactSwitchWorkspaceRequest(r), true
- case *v1pb.AIChatRequest:
- return redactAIChatRequest(r), true
- case *v1pb.ListInstanceDatabaseRequest:
- // The request carries a whole Instance — "we need to set this
- // field if the target instance is not created yet" — so it holds
- // every data-source credential: password, ssl_key,
- // ssh_private_key, the Kerberos keytab, the AWS/Azure/GCP
- // credentials, the external-secret token and master_password.
- // redactInstance is the same helper CreateInstance and
- // UpdateInstance already use.
- r = proto.CloneOf(r)
- r.Instance = redactInstance(r.Instance)
- return r, true
- default:
- return nil, false
- }
-}
-
-func getResponseString(response any) (string, error) {
- m := func() protoreflect.ProtoMessage {
- if response == nil || reflect.ValueOf(response).IsNil() {
- return nil
- }
- switch r := response.(type) {
- case *v1pb.QueryResponse:
- return redactQueryResponse(r)
- case *v1pb.AdminExecuteResponse:
- return redactAdminExecuteResponse(r)
- case *v1pb.ExportResponse:
- return redactExportResponse(r)
- case *v1pb.LoginResponse:
- return redactLoginResponse(r)
- case *v1pb.ExchangeTokenResponse:
- return redactExchangeTokenResponse(r)
- case *v1pb.RotateDirectorySyncTokenResponse:
- return redactRotateDirectorySyncTokenResponse(r)
- case *v1pb.ServiceAccount:
- return redactServiceAccount(r)
- case *v1pb.Setting:
- return redactSetting(r)
- case *v1pb.IdentityProvider:
- return redactIdentityProvider(r)
- case *v1pb.User:
- return redactUser(r)
- case *v1pb.Instance:
- return redactInstance(r)
- case *v1pb.Project:
- return redactProject(r)
- case *v1pb.Release:
- return redactRelease(r)
- case *v1pb.SavedQuery:
- return redactSavedQuery(r)
- case *v1pb.PurchaseResponse:
- return redactPurchaseResponse(r)
- case *v1pb.ExportVCSProviderUsersResponse:
- return redactExportVCSProviderUsersResponse(r)
- case *v1pb.ExportAuditLogsResponse:
- return redactExportAuditLogsResponse(r)
- case *v1pb.Sheet:
- return redactSheet(r)
- case *v1pb.BatchCreateSheetsResponse:
- n := &v1pb.BatchCreateSheetsResponse{}
- for _, sheet := range r.Sheets {
- n.Sheets = append(n.Sheets, redactSheet(sheet))
- }
- return n
- case *v1pb.BatchUpdateInstancesResponse:
- n := &v1pb.BatchUpdateInstancesResponse{}
- for _, instance := range r.Instances {
- n.Instances = append(n.Instances, redactInstance(instance))
- }
- return n
- default:
- if p, ok := r.(protoreflect.ProtoMessage); ok {
- return p
- }
- return nil
- }
- }()
- if m == nil {
- return "", nil
- }
- b, err := protojson.Marshal(m)
- if err != nil {
- return "", err
- }
- return string(b), nil
-}
-
-func redactResetPasswordRequest(r *v1pb.ResetPasswordRequest) *v1pb.ResetPasswordRequest {
- if r == nil {
- return nil
- }
- cloned := proto.CloneOf(r)
- cloned.Code = maskedString
- cloned.NewPassword = maskedString
- return cloned
-}
-
-func redactUploadLicenseRequest(r *v1pb.UploadLicenseRequest) *v1pb.UploadLicenseRequest {
- if r == nil {
- return nil
- }
- return &v1pb.UploadLicenseRequest{License: maskedString}
-}
-
-func redactPurchaseResponse(r *v1pb.PurchaseResponse) *v1pb.PurchaseResponse {
- if r == nil {
- return nil
- }
- return &v1pb.PurchaseResponse{}
-}
-
-func redactExportVCSProviderUsersResponse(r *v1pb.ExportVCSProviderUsersResponse) *v1pb.ExportVCSProviderUsersResponse {
- if r == nil {
- return nil
- }
- return &v1pb.ExportVCSProviderUsersResponse{}
-}
-
-func redactExportAuditLogsResponse(r *v1pb.ExportAuditLogsResponse) *v1pb.ExportAuditLogsResponse {
- if r == nil {
- return nil
- }
- return &v1pb.ExportAuditLogsResponse{NextPageToken: r.NextPageToken}
-}
-
-func redactExportRequest(r *v1pb.ExportRequest) *v1pb.ExportRequest {
- if r == nil {
- return nil
- }
- r = proto.CloneOf(r)
- if r.Password != "" {
- r.Password = maskedString
- }
- return r
-}
-
-// redactExportResponse drops the exported file content but keeps the applied
-// access grant so audit logs record which grant authorized the export.
-func redactExportResponse(r *v1pb.ExportResponse) *v1pb.ExportResponse {
- if r == nil || r.AppliedAccessGrant == "" {
- return nil
- }
- return &v1pb.ExportResponse{
- AppliedAccessGrant: r.AppliedAccessGrant,
- }
-}
-
-func redactLoginRequest(r *v1pb.LoginRequest) *v1pb.LoginRequest {
- if r == nil {
- return nil
- }
-
- // Clone to avoid mutating original
- r = proto.CloneOf(r)
-
- // Mask sensitive fields.
- if r.Password != "" {
- r.Password = maskedString
- }
- maskOptionalString(&r.OtpCode)
- maskOptionalString(&r.RecoveryCode)
- maskOptionalString(&r.MfaTempToken)
- maskOptionalString(&r.EmailCode)
- if r.IdpContext != nil {
- r.IdpContext = nil
- }
- return r
-}
-
-func redactSignupRequest(r *v1pb.SignupRequest) *v1pb.SignupRequest {
- if r == nil {
- return nil
- }
- r = proto.CloneOf(r)
- if r.Password != "" {
- r.Password = maskedString
- }
- return r
-}
-
-// redactExchangeTokenRequest masks the external OIDC JWT. The token is a
-// credential — it could be replayed against the original IdP or, if logged,
-// reveal workload identity claims. The caller's email is kept for audit
-// correlation.
-func redactExchangeTokenRequest(r *v1pb.ExchangeTokenRequest) *v1pb.ExchangeTokenRequest {
- if r == nil {
- return nil
- }
- r = proto.CloneOf(r)
- if r.Token != "" {
- r.Token = maskedString
- }
- return r
-}
-
-// redactExchangeTokenResponse drops the issued Bytebase access token. Logging
-// it would give anyone with audit-log read access a valid API token for the
-// workload identity. Returns an empty response so the audit entry still
-// records that the call happened.
-func redactExchangeTokenResponse(r *v1pb.ExchangeTokenResponse) *v1pb.ExchangeTokenResponse {
- if r == nil {
- return nil
- }
- return &v1pb.ExchangeTokenResponse{}
-}
-
-// redactRotateDirectorySyncTokenResponse drops the newly minted SCIM token. The
-// whole point of hashing it at rest is that the plaintext exists only in the
-// single response to the admin who rotated it; writing it to the audit log would
-// hand a working credential to anyone who can read that log. Returns an empty
-// response so the audit entry still records that the rotation happened.
-func redactRotateDirectorySyncTokenResponse(r *v1pb.RotateDirectorySyncTokenResponse) *v1pb.RotateDirectorySyncTokenResponse {
- if r == nil {
- return nil
- }
- return &v1pb.RotateDirectorySyncTokenResponse{}
-}
-
-// redactServiceAccount drops the API key. Create and key rotation are the only
-// responses that carry it — the read path never populates it — and it is a live
-// credential, so logging it would hand a working key to anyone who can read the
-// audit log or its stdout stream.
-func redactServiceAccount(r *v1pb.ServiceAccount) *v1pb.ServiceAccount {
- if r == nil {
- return nil
- }
- r = proto.CloneOf(r)
- if r.ServiceKey != "" {
- r.ServiceKey = maskedString
- }
- return r
-}
-
-// redactIdentityProvider masks the IdP credentials. The read path already blanks
-// these (convertToIdentityProvider), so they only reach the audit log through
-// the create/update request payload.
-func redactIdentityProvider(r *v1pb.IdentityProvider) *v1pb.IdentityProvider {
- if r == nil {
- return nil
- }
- r = proto.CloneOf(r)
- switch config := r.GetConfig().GetConfig().(type) {
- case *v1pb.IdentityProviderConfig_Oauth2Config:
- if config.Oauth2Config.GetClientSecret() != "" {
- config.Oauth2Config.ClientSecret = maskedString
- }
- case *v1pb.IdentityProviderConfig_OidcConfig:
- if config.OidcConfig.GetClientSecret() != "" {
- config.OidcConfig.ClientSecret = maskedString
- }
- case *v1pb.IdentityProviderConfig_LdapConfig:
- if config.LdapConfig.GetBindPassword() != "" {
- config.LdapConfig.BindPassword = maskedString
- }
- default:
- }
- return r
-}
-
-// redactEmailSetting masks the SMTP password, the same field redactSetting
-// masks when the same value arrives through UpdateSetting.
-func redactEmailSetting(e *v1pb.EmailSetting) *v1pb.EmailSetting {
- if e == nil {
- return nil
- }
- e = proto.CloneOf(e)
- if smtp := e.GetSmtp(); smtp.GetPassword() != "" {
- smtp.Password = maskedString
- }
- return e
-}
-
-// redactTestIdentityProviderRequest masks both halves of a connection test: the
-// provider's own client secret or LDAP bind password, which redactIdentityProvider
-// already masks on Create and Update, and the credential supplied to run the
-// test — an authorization code, or a directory user's password. Each arm checks
-// the wrapped message: a oneof wrapper can be set with a nil payload, and this
-// runs on the audit path, where a panic would cost the row.
-func redactTestIdentityProviderRequest(r *v1pb.TestIdentityProviderRequest) *v1pb.TestIdentityProviderRequest {
- if r == nil {
- return nil
- }
- r = proto.CloneOf(r)
- r.IdentityProvider = redactIdentityProvider(r.IdentityProvider)
- switch context := r.GetContext().(type) {
- case *v1pb.TestIdentityProviderRequest_Oauth2Context:
- if context.Oauth2Context != nil {
- context.Oauth2Context.Code = maskedString
- }
- case *v1pb.TestIdentityProviderRequest_OidcContext:
- if context.OidcContext != nil {
- context.OidcContext.Code = maskedString
- }
- case *v1pb.TestIdentityProviderRequest_LdapContext:
- if context.LdapContext != nil {
- context.LdapContext.Password = maskedString
- }
- default:
- }
- return r
-}
-
-// redactSwitchWorkspaceRequest masks the MFA proofs, the same three fields
-// redactLoginRequest masks when they arrive on Login.
-func redactSwitchWorkspaceRequest(r *v1pb.SwitchWorkspaceRequest) *v1pb.SwitchWorkspaceRequest {
- if r == nil {
- return nil
- }
- r = proto.CloneOf(r)
- maskOptionalString(&r.OtpCode)
- maskOptionalString(&r.RecoveryCode)
- maskOptionalString(&r.MfaTempToken)
- return r
-}
-
-// redactAIChatRequest drops the conversation instead of masking a field. The
-// request has no secret in it; it has an unbounded body — every message the
-// caller sent, plus the tool definitions — and the audit row stores it whole
-// (the size cap applies to the stdout logger only). AIService/Chat is EXCLUDED
-// and unaudited, so the only rows it ever produces are the gate's denials, and
-// a denial needs the fact of the call, not its transcript.
-func redactAIChatRequest(r *v1pb.AIChatRequest) *v1pb.AIChatRequest {
- if r == nil {
- return nil
- }
- return &v1pb.AIChatRequest{}
-}
-
-// maskOptionalString masks an optional string field in place when it is set,
-// so the several credential fields spelled `optional string` mask the same way
-// wherever they appear.
-func maskOptionalString(field **string) {
- if *field != nil {
- *field = &maskedString
- }
-}
-
-func redactWebhook(w *v1pb.Webhook) *v1pb.Webhook {
- if w == nil {
- return nil
- }
- cloned := proto.CloneOf(w)
- if cloned.Url != "" {
- cloned.Url = maskedString
- }
- return cloned
-}
-
-func redactProject(p *v1pb.Project) *v1pb.Project {
- if p == nil {
- return nil
- }
- cloned := proto.CloneOf(p)
- for i, webhook := range cloned.Webhooks {
- cloned.Webhooks[i] = redactWebhook(webhook)
- }
- return cloned
-}
-
-func redactRelease(r *v1pb.Release) *v1pb.Release {
- if r == nil {
- return nil
- }
- cloned := proto.CloneOf(r)
- for _, file := range cloned.Files {
- if file != nil {
- file.Statement = nil
- }
- }
- return cloned
-}
-
-func redactSavedQuery(r *v1pb.SavedQuery) *v1pb.SavedQuery {
- if r == nil {
- return nil
- }
- cloned := proto.CloneOf(r)
- cloned.Content = nil
- return cloned
-}
-
-// redactSetting masks every credential a settings payload can carry. The read
-// path blanks these, so they reach the audit log only through UpdateSetting's
-// request. Each secret is masked rather than dropped so the log still records
-// that the field was being written.
-func redactSetting(r *v1pb.Setting) *v1pb.Setting {
- if r == nil {
- return nil
- }
- r = proto.CloneOf(r)
- switch value := r.GetValue().GetValue().(type) {
- case *v1pb.SettingValue_Email:
- if smtp := value.Email.GetSmtp(); smtp.GetPassword() != "" {
- smtp.Password = maskedString
- }
- case *v1pb.SettingValue_Ai:
- if value.Ai.GetApiKey() != "" {
- value.Ai.ApiKey = maskedString
- }
- case *v1pb.SettingValue_AppIm:
- maskAppIMSecrets(value.AppIm)
- default:
- }
- return r
-}
-
-func maskAppIMSecrets(s *v1pb.AppIMSetting) {
- for _, setting := range s.GetSettings() {
- if v := setting.GetSlack(); v.GetToken() != "" {
- v.Token = maskedString
- }
- if v := setting.GetFeishu(); v.GetAppSecret() != "" {
- v.AppSecret = maskedString
- }
- if v := setting.GetWecom(); v.GetSecret() != "" {
- v.Secret = maskedString
- }
- if v := setting.GetLark(); v.GetAppSecret() != "" {
- v.AppSecret = maskedString
- }
- if v := setting.GetDingtalk(); v.GetClientSecret() != "" {
- v.ClientSecret = maskedString
- }
- if v := setting.GetTeams(); v.GetClientSecret() != "" {
- v.ClientSecret = maskedString
- }
- }
-}
-
-func redactCreateUserRequest(r *v1pb.CreateUserRequest) *v1pb.CreateUserRequest {
- if r == nil {
- return nil
- }
- return &v1pb.CreateUserRequest{
- User: redactUser(r.User),
- }
-}
-
-func redactUpdateUserRequest(r *v1pb.UpdateUserRequest) *v1pb.UpdateUserRequest {
- if r == nil {
- return nil
- }
- return &v1pb.UpdateUserRequest{
- User: redactUser(r.User),
- UpdateMask: r.UpdateMask,
- OtpCode: r.OtpCode,
- RegenerateTempMfaSecret: r.RegenerateTempMfaSecret,
- RegenerateRecoveryCodes: r.RegenerateRecoveryCodes,
- }
-}
-
-func redactUser(r *v1pb.User) *v1pb.User {
- if r == nil {
- return nil
- }
- return &v1pb.User{
- Name: r.Name,
- Email: r.Email,
- Title: r.Title,
- }
-}
-
-// redactSheet strips the sheet content from audit payloads. Sheets carry full
-// SQL statements (potentially megabytes); the audit record keeps the resource
-// name and content size.
-func redactSheet(s *v1pb.Sheet) *v1pb.Sheet {
- if s == nil {
- return nil
- }
- cloned := proto.CloneOf(s)
- cloned.Content = nil
- return cloned
-}
-
-func redactInstance(i *v1pb.Instance) *v1pb.Instance {
- if i == nil {
- return nil
- }
- // Clone the instance to avoid modifying the original response
- cloned := proto.CloneOf(i)
- var dataSources []*v1pb.DataSource
- for _, d := range cloned.DataSources {
- dataSources = append(dataSources, redactDataSource(d))
- }
- cloned.DataSources = dataSources
- return cloned
-}
-
-func redactDataSource(d *v1pb.DataSource) *v1pb.DataSource {
- if d == nil {
- return nil
- }
- // Clone the datasource to avoid modifying the original
- cloned := proto.CloneOf(d)
- if cloned.Password != "" {
- cloned.Password = maskedString
- }
- if cloned.SslCa != "" {
- cloned.SslCa = maskedString
- }
- if cloned.SslCaPath != "" {
- cloned.SslCaPath = maskedString
- }
- if cloned.SslCert != "" {
- cloned.SslCert = maskedString
- }
- if cloned.SslCertPath != "" {
- cloned.SslCertPath = maskedString
- }
- if cloned.SslKey != "" {
- cloned.SslKey = maskedString
- }
- if cloned.SslKeyPath != "" {
- cloned.SslKeyPath = maskedString
- }
- if cloned.SshPassword != "" {
- cloned.SshPassword = maskedString
- }
- if cloned.SshPrivateKey != "" {
- cloned.SshPrivateKey = maskedString
- }
- if cloned.AuthenticationPrivateKey != "" {
- cloned.AuthenticationPrivateKey = maskedString
- }
- if cloned.AuthenticationPrivateKeyPassphrase != "" {
- cloned.AuthenticationPrivateKeyPassphrase = maskedString
- }
- redactIAMExtension(cloned)
- if cloned.ExternalSecret != nil {
- cloned.ExternalSecret = new(v1pb.DataSourceExternalSecret)
- }
- if cloned.SaslConfig != nil {
- if krbConf := cloned.SaslConfig.GetKrbConfig(); krbConf != nil {
- krbConf.Keytab = []byte(maskedString)
- cloned.SaslConfig.Mechanism = &v1pb.SASLConfig_KrbConfig{KrbConfig: krbConf}
- }
- }
- if cloned.MasterPassword != "" {
- cloned.MasterPassword = maskedString
- }
- return cloned
-}
-
-// redactIAMExtension strips the IAM credential while keeping the variant that
-// was set, so the row still records which credential type the caller supplied.
-// It keys off the oneof rather than authentication_type, which a request is
-// free to leave unset or contradict.
+// marshalAuditPayload renders one side of an audited call. Redaction is driven
+// by the (bytebase.v1.audit_behavior) annotation on the message's own
+// descriptor — see audit_redact.go — so nothing here switches on a type, and a
+// field is protected on every RPC that carries it rather than on the ones
+// someone wrote a redactor for.
//
-// What survives mirrors the read path (instance_service_converter.go): the
-// Azure tenant and client IDs name the principal and are not INPUT_ONLY, so
-// they stay; every AWSCredential field is, including the role ARN, so none
-// does; and the GCP content is a whole service-account key.
-//
-// Keeping the ARN was considered — it names a principal rather than
-// authenticating as one, and the row could then answer which role a data
-// source was pointed at. It is dropped anyway, because "no INPUT_ONLY field
-// survives redaction" is an invariant a descriptor walk can check forever
-// (TestAuditRedactsEveryInputOnlyDataSourceField), and a hand-kept exception
-// list is the drift this function exists to stop.
-func redactIAMExtension(d *v1pb.DataSource) {
- switch extension := d.GetIamExtension().(type) {
- case *v1pb.DataSource_AzureCredential_:
- d.IamExtension = &v1pb.DataSource_AzureCredential_{
- AzureCredential: &v1pb.DataSource_AzureCredential{
- TenantId: extension.AzureCredential.GetTenantId(),
- ClientId: extension.AzureCredential.GetClientId(),
- },
- }
- case *v1pb.DataSource_AwsCredential:
- d.IamExtension = &v1pb.DataSource_AwsCredential{
- AwsCredential: &v1pb.DataSource_AWSCredential{},
- }
- case *v1pb.DataSource_GcpCredential:
- d.IamExtension = &v1pb.DataSource_GcpCredential{
- GcpCredential: &v1pb.DataSource_GCPCredential{},
- }
- default:
- // A variant added to the oneof after this function was written is
- // dropped whole rather than logged. Unset lands here too and stays nil.
- d.IamExtension = nil
- }
-}
-
-func redactAdminExecuteResponse(r *v1pb.AdminExecuteResponse) *v1pb.AdminExecuteResponse {
- if r == nil {
- return nil
- }
- n := &v1pb.AdminExecuteResponse{
- Results: nil,
- }
- for _, result := range r.Results {
- if result == nil {
- n.Results = append(n.Results, &v1pb.QueryResult{})
- continue
- }
- n.Results = append(n.Results, &v1pb.QueryResult{
- ColumnNames: result.ColumnNames,
- ColumnTypeNames: result.ColumnTypeNames,
- Rows: nil, // Redacted
- Error: result.Error,
- Latency: result.Latency,
- Statement: result.Statement,
- DetailedError: result.DetailedError,
- Masked: redactMaskingReasons(result.Masked), // Redact icon data
- })
- }
-
- return n
-}
-
-func redactQueryResponse(r *v1pb.QueryResponse) *v1pb.QueryResponse {
- if r == nil {
- return nil
- }
- n := &v1pb.QueryResponse{
- Results: nil,
- AppliedAccessGrant: r.AppliedAccessGrant,
- }
- for _, result := range r.Results {
- n.Results = append(n.Results, &v1pb.QueryResult{
- ColumnNames: result.ColumnNames,
- ColumnTypeNames: result.ColumnTypeNames,
- Rows: nil, // Redacted
- RowsCount: result.RowsCount,
- Error: result.Error,
- Latency: result.Latency,
- Statement: result.Statement,
- DetailedError: result.DetailedError,
- Masked: redactMaskingReasons(result.Masked), // Redact icon data
- })
- }
- return n
-}
-
-func redactMaskingReasons(reasons []*v1pb.MaskingReason) []*v1pb.MaskingReason {
- if reasons == nil {
- return nil
- }
- var redacted []*v1pb.MaskingReason
- for _, reason := range reasons {
- if reason == nil {
- redacted = append(redacted, nil)
- continue
- }
- redacted = append(redacted, &v1pb.MaskingReason{
- SemanticTypeId: reason.SemanticTypeId,
- SemanticTypeTitle: reason.SemanticTypeTitle,
- MaskingRuleId: reason.MaskingRuleId,
- Algorithm: reason.Algorithm,
- Context: reason.Context,
- ClassificationLevel: reason.ClassificationLevel,
- // Omit SemanticTypeIcon to avoid polluting audit logs with base64 data
- })
- }
- return redacted
-}
-
-func redactLoginResponse(r *v1pb.LoginResponse) *v1pb.LoginResponse {
- if r == nil {
- return nil
+// One function for both directions: the annotation says what to do with a
+// field, and which side of the call it arrived on does not change that. It is
+// called from createAuditLog rather than from WrapUnary because the streaming
+// path builds its own auditEntry and calls createAuditLog directly, so a walk
+// that lived in the unary interceptor would skip AdminExecute entirely.
+func marshalAuditPayload(payload any) string {
+ if payload == nil {
+ return ""
}
-
- n := &v1pb.LoginResponse{
- RequireResetPassword: r.RequireResetPassword,
+ // IsNil panics on a kind that cannot be nil, and Receive/Send stash
+ // whatever the stream hands them, so the kind is checked first.
+ if value := reflect.ValueOf(payload); value.Kind() == reflect.Pointer && value.IsNil() {
+ return ""
}
- if r.User != nil {
- n.User = redactUser(r.User)
+ message, ok := payload.(proto.Message)
+ if !ok {
+ return ""
}
- return n
+ b, err := protojson.Marshal(redactForAudit(message))
+ if err != nil {
+ // A payload that will not encode must not cost the record. protojson
+ // rejects invalid UTF-8, and QueryResult.error and .statement are
+ // filled straight from the driver — so one bad byte from an Oracle or
+ // MySQL connection in a non-UTF-8 charset used to mean either no audit
+ // row at all (unary, where WrapUnary swallows the error) or a killed
+ // AdminExecute stream. Degrade the payload instead; everything else on
+ // the row, including the method and the outcome, still gets written.
+ slog.Warn("audit: payload could not be encoded, recording a placeholder",
+ log.BBError(err), slog.String("type", string(message.ProtoReflect().Descriptor().FullName())))
+ return unencodablePayload
+ }
+ return string(b)
}
func needAudit(ctx context.Context) bool {
diff --git a/backend/api/v1/audit_redact.go b/backend/api/v1/audit_redact.go
new file mode 100644
index 0000000000..d92a92b753
--- /dev/null
+++ b/backend/api/v1/audit_redact.go
@@ -0,0 +1,560 @@
+package v1
+
+import (
+ "log/slog"
+ "sync"
+
+ "google.golang.org/protobuf/encoding/protowire"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/reflect/protoreflect"
+ "google.golang.org/protobuf/reflect/protoregistry"
+ "google.golang.org/protobuf/types/descriptorpb"
+ "google.golang.org/protobuf/types/known/anypb"
+
+ spb "google.golang.org/genproto/googleapis/rpc/status"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// Redaction is driven by the (bytebase.v1.audit_behavior) field option rather
+// than by a per-RPC redactor. A field marked SENSITIVE or OMIT in the .proto is
+// dropped wherever it appears, at any depth, on every request and response the
+// audit interceptor records.
+//
+// It runs from createAuditLog rather than from WrapUnary because the streaming
+// path builds its own auditEntry and calls createAuditLog directly, so a walk
+// in the unary interceptor would skip AdminExecute — the one streaming RPC, and
+// the one carrying every row of an admin-mode query.
+//
+// It is a denylist, so an unannotated field is logged. The inventory lint in
+// audit_redact_inventory_test.go moves that failure to CI.
+
+const anyFullName protoreflect.FullName = "google.protobuf.Any"
+
+// The two Any-carrying fields on an audit row, redacted where they are
+// assigned rather than by walking the row.
+const (
+ auditServiceDataField protoreflect.FullName = "bytebase.store.AuditLog.service_data"
+ auditStatusDetailsField protoreflect.FullName = "google.rpc.Status.details"
+)
+
+// auditAnyRegistry names the types each Any field on an audit row may carry.
+// An Any whose packed type is not listed for its field is DROPPED rather than
+// logged: protojson.Marshal fails the entire row on an unresolvable Any, so
+// keeping it would lose the record.
+//
+// The descriptor walk sees only type_url and value, so a field annotated inside
+// a packed message is invisible to the redactor, the coverage sweep and the
+// inventory alike. Registering a type is what pulls its fields into the
+// inventory for classification.
+//
+// TestLintAuditAnyFieldsAreRegistered enforces the descriptor half: a new Any
+// FIELD reaching the row must be registered. The other half — a new TYPE packed
+// into an existing field — changes no descriptor, so nothing fails the build;
+// redactPackedAny logs the dropped type at runtime instead.
+var auditAnyRegistry = map[protoreflect.FullName][]protoreflect.FullName{
+ // UpdateSetting's before-image and the locked re-capture in the
+ // workspace-profile merge (setting_service.go:174, :658), plus SetIamPolicy
+ // on projects and on the workspace (project_service.go:621,
+ // workspace_service.go:509).
+ //
+ // Setting is the one that carries credentials; convertToSettingMessage
+ // blanks its own, but that is a property of the call site rather than an
+ // enforced one, which is why it is classified through the inventory like
+ // any other field. v1.AuditLog.service_data is deliberately absent: it is
+ // annotated OMIT, so SearchAuditLogs' own row does not re-transcribe the
+ // before-images of every row the caller read.
+ auditServiceDataField: {
+ "bytebase.v1.Setting",
+ "bytebase.v1.AuditData",
+ },
+ // Attached to a FAILED RPC by convertErrToStatus — exactly when a handler
+ // adds context about what went wrong. PermissionDeniedDetail carries a
+ // method, permissions and resource names; PlanCheckRun.Result carries
+ // advisory text.
+ auditStatusDetailsField: {
+ "bytebase.v1.PermissionDeniedDetail",
+ "bytebase.v1.PlanCheckRun.Result",
+ },
+}
+
+// auditAnyPermits reports whether a packed type may be recorded in one Any
+// field. An unregistered field permits nothing, so a new Any reaching the row
+// before anyone classifies it drops its payload rather than writing it.
+func auditAnyPermits(field, packed protoreflect.FullName) bool {
+ for _, permitted := range auditAnyRegistry[field] {
+ if permitted == packed {
+ return true
+ }
+ }
+ return false
+}
+
+// auditBehaviorFieldNumber is (bytebase.v1.audit_behavior)'s extension number,
+// used to recognize the annotation when it survives only as an unknown field.
+const auditBehaviorFieldNumber protowire.Number = 100010
+
+type actionKind uint8
+
+const (
+ // actionDrop leaves the field out of the copy entirely.
+ actionDrop actionKind = iota + 1
+ // actionBlank copies the field with its zero value, keeping presence. Used
+ // for SENSITIVE scalars that have explicit presence — a oneof arm or a
+ // proto3 optional — so the row still records that a credential was
+ // supplied. Clearing DataSourceExternalSecret.token would make token auth
+ // indistinguishable from unconfigured while its AppRole sibling stayed
+ // legible, and dropping LoginRequest.otp_code would make a failed login
+ // with an OTP indistinguishable from one without.
+ actionBlank
+ // actionPacked resolves and redacts a google.protobuf.Any. The descriptor
+ // walk sees only type_url and value, never the packed message.
+ actionPacked
+ // actionDescend rebuilds a subtree that contains something to redact.
+ actionDescend
+)
+
+type fieldAction struct {
+ kind actionKind
+ // sub is the descriptor to descend into, set only for actionDescend. It is
+ // a DESCRIPTOR rather than a plan so a recursive type resolves its own plan
+ // at every depth: caching a plan value here would freeze a finite chain,
+ // and an annotation below its end would be applied at the top and nowhere
+ // else.
+ sub protoreflect.MessageDescriptor
+}
+
+// redactPlan says what to do with one message type. A nil plan means the type
+// holds nothing to redact anywhere beneath it, so a value of that type is
+// shared by pointer and only read.
+type redactPlan struct {
+ fields map[protoreflect.FieldNumber]fieldAction
+}
+
+// planCache maps a message descriptor to its *redactPlan. Keyed on the
+// descriptor itself, not on its full name: plans are applied by field NUMBER,
+// so two distinct descriptors sharing a name would otherwise get each other's
+// plan. A nil-valued entry is meaningful — it records that the type needs
+// nothing.
+var planCache sync.Map
+
+// needsCache memoizes typeNeedsRedaction.
+var needsCache sync.Map
+
+// planFor returns the plan for a message type, building it on first use.
+func planFor(descriptor protoreflect.MessageDescriptor) *redactPlan {
+ if cached, ok := planCache.Load(descriptor); ok {
+ if plan, isPlan := cached.(*redactPlan); isPlan {
+ return plan
+ }
+ }
+
+ var fields map[protoreflect.FieldNumber]fieldAction
+ record := func(field protoreflect.FieldDescriptor, action fieldAction) {
+ if fields == nil {
+ fields = map[protoreflect.FieldNumber]fieldAction{}
+ }
+ fields[field.Number()] = action
+ }
+
+ declared := descriptor.Fields()
+ for i := range declared.Len() {
+ field := declared.Get(i)
+ switch auditBehaviorOf(field) {
+ case v1pb.AuditBehavior_SENSITIVE:
+ // A blanked field needs a scalar zero value to stand in for the
+ // removed one, and explicit presence for that value to survive
+ // marshaling. Everything else is dropped.
+ if isBlankable(field) && field.HasPresence() {
+ record(field, fieldAction{kind: actionBlank})
+ } else {
+ record(field, fieldAction{kind: actionDrop})
+ }
+ continue
+ case v1pb.AuditBehavior_OMIT:
+ record(field, fieldAction{kind: actionDrop})
+ continue
+ default:
+ }
+ sub := submessageOf(field)
+ if sub == nil {
+ continue
+ }
+ if sub.FullName() == anyFullName {
+ record(field, fieldAction{kind: actionPacked})
+ continue
+ }
+ if typeNeedsRedaction(sub) {
+ record(field, fieldAction{kind: actionDescend, sub: sub})
+ }
+ }
+
+ var plan *redactPlan
+ if fields != nil {
+ plan = &redactPlan{fields: fields}
+ }
+ cached, _ := planCache.LoadOrStore(descriptor, plan)
+ if stored, isPlan := cached.(*redactPlan); isPlan {
+ return stored
+ }
+ return plan
+}
+
+// typeNeedsRedaction reports whether anything reachable from a message type has
+// to be redacted. It is plain reachability with a visited set rather than a
+// fixed point, which is what makes the four cyclic components in the descriptor
+// graph terminate: ObjectSchema/StructKind/ArrayKind,
+// google.protobuf.Value/Struct/ListValue, google.api.expr's Expr, and
+// TablePartitionMetadata.subpartitions.
+func typeNeedsRedaction(descriptor protoreflect.MessageDescriptor) bool {
+ if cached, ok := needsCache.Load(descriptor); ok {
+ if needs, isBool := cached.(bool); isBool {
+ return needs
+ }
+ }
+ needs := reachesRedaction(descriptor, map[protoreflect.MessageDescriptor]bool{})
+ needsCache.Store(descriptor, needs)
+ return needs
+}
+
+func reachesRedaction(descriptor protoreflect.MessageDescriptor, seen map[protoreflect.MessageDescriptor]bool) bool {
+ if seen[descriptor] {
+ return false
+ }
+ seen[descriptor] = true
+
+ fields := descriptor.Fields()
+ for i := range fields.Len() {
+ field := fields.Get(i)
+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {
+ return true
+ }
+ sub := submessageOf(field)
+ if sub == nil {
+ continue
+ }
+ if sub.FullName() == anyFullName {
+ return true
+ }
+ if reachesRedaction(sub, seen) {
+ return true
+ }
+ }
+ return false
+}
+
+// submessageOf returns the message descriptor a field leads to: the element
+// type of a repeated field, the value type of a map, or the field's own type.
+func submessageOf(field protoreflect.FieldDescriptor) protoreflect.MessageDescriptor {
+ if field.IsMap() {
+ if isMessageKind(field.MapValue().Kind()) {
+ return field.MapValue().Message()
+ }
+ return nil
+ }
+ if isMessageKind(field.Kind()) {
+ return field.Message()
+ }
+ return nil
+}
+
+func isMessageKind(kind protoreflect.Kind) bool {
+ return kind == protoreflect.MessageKind || kind == protoreflect.GroupKind
+}
+
+func isBlankable(field protoreflect.FieldDescriptor) bool {
+ return !field.IsList() && !field.IsMap() && !isMessageKind(field.Kind())
+}
+
+// auditBehaviorOf reads the annotation, failing CLOSED. A denylist that cannot
+// read its own marker has to assume the field is a credential: the alternative
+// is writing it out verbatim with no error and no log line.
+func auditBehaviorOf(field protoreflect.FieldDescriptor) v1pb.AuditBehavior {
+ options, ok := field.Options().(*descriptorpb.FieldOptions)
+ if !ok {
+ return v1pb.AuditBehavior_SENSITIVE
+ }
+ if options == nil {
+ return v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED
+ }
+ if behavior, ok := proto.GetExtension(options, v1pb.E_AuditBehavior).(v1pb.AuditBehavior); ok &&
+ behavior != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {
+ return behavior
+ }
+ // The annotation survives as an UNKNOWN field when the descriptor was
+ // unmarshaled by a resolver that does not know this extension — which is
+ // reachable, since redactPackedAny resolves types out of a registry that
+ // can hold runtime-registered files. Reading UNSPECIFIED there would log
+ // the very field the annotation exists to protect.
+ if unknownCarriesAuditBehavior(options) {
+ return v1pb.AuditBehavior_SENSITIVE
+ }
+ return v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED
+}
+
+func unknownCarriesAuditBehavior(options *descriptorpb.FieldOptions) bool {
+ unknown := options.ProtoReflect().GetUnknown()
+ for len(unknown) > 0 {
+ number, kind, tagLen := protowire.ConsumeTag(unknown)
+ if tagLen < 0 {
+ // Unparseable options: fail closed rather than guess.
+ return true
+ }
+ if number == auditBehaviorFieldNumber {
+ return true
+ }
+ valueLen := protowire.ConsumeFieldValue(number, kind, unknown[tagLen:])
+ if valueLen < 0 {
+ return true
+ }
+ unknown = unknown[tagLen+valueLen:]
+ }
+ return false
+}
+
+// redactForAudit returns the message to marshal into an audit payload. When
+// nothing under the message needs redacting it returns the message itself —
+// the common case, and what makes a 5 MB sheet free.
+//
+// Otherwise it returns a partial copy. Every message on the path to a redacted
+// field is copied by Range — never by iterating Descriptor().Fields() and
+// calling Set, which panics on an unset message, list or map field. Range
+// yields only populated fields, which is also how a dropped field is never
+// copied: it is skipped during the copy rather than cleared afterwards on a
+// message that never held it. Every subtree with nothing to redact is shared by
+// pointer and only read.
+//
+// Because subtrees are shared, the result is write-once: marshal it and discard
+// it. Mutating it would write through to the caller's live message, which the
+// handler is still about to return to the client.
+func redactForAudit[M proto.Message](message M) M {
+ reflected := message.ProtoReflect()
+ if !reflected.IsValid() {
+ return message
+ }
+ plan := planFor(reflected.Descriptor())
+ if plan == nil {
+ return message
+ }
+ redacted, ok := redactMessage(reflected, plan).Interface().(M)
+ if !ok {
+ // src.New() returns the same concrete type, so this cannot happen. If
+ // a future runtime made it possible, an empty message of the right type
+ // is the only safe answer: returning the input would log it unredacted,
+ // and returning the zero value would hand a typed nil to callers that
+ // dereference it.
+ empty, isM := reflected.New().Interface().(M)
+ if !isM {
+ return message
+ }
+ return empty
+ }
+ return redacted
+}
+
+// redactMessage copies src, applying plan.
+func redactMessage(src protoreflect.Message, plan *redactPlan) protoreflect.Message {
+ dst := src.New()
+ src.Range(func(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {
+ action, ok := plan.fields[field.Number()]
+ if !ok {
+ dst.Set(field, value)
+ return true
+ }
+ switch action.kind {
+ case actionDrop:
+ case actionBlank:
+ dst.Set(field, field.Default())
+ case actionPacked:
+ copyAnyField(dst, field, value)
+ case actionDescend:
+ copyDescended(dst, field, value, action.sub)
+ default:
+ }
+ return true
+ })
+ return dst
+}
+
+// copyDescended rebuilds the subtree under one field.
+func copyDescended(dst protoreflect.Message, field protoreflect.FieldDescriptor, value protoreflect.Value, sub protoreflect.MessageDescriptor) {
+ plan := planFor(sub)
+ if plan == nil {
+ dst.Set(field, value)
+ return
+ }
+ switch {
+ case field.IsMap():
+ // dst.Set(field, value) on a map shares the Go map itself and its
+ // message values, so redacting an entry would write through to the
+ // caller's live message. Rebuild entry by entry.
+ out := dst.Mutable(field).Map()
+ value.Map().Range(func(key protoreflect.MapKey, entry protoreflect.Value) bool {
+ out.Set(key, protoreflect.ValueOfMessage(redactMessage(entry.Message(), plan)))
+ return true
+ })
+ case field.IsList():
+ // Sharing the list cannot produce redacted elements without writing
+ // through to the caller's, so it is rebuilt per element.
+ // Instance.roles[].password and instance_resource.data_sources[] both
+ // cross one of these — the first is the leak this design opens with.
+ out := dst.Mutable(field).List()
+ list := value.List()
+ for i := range list.Len() {
+ out.Append(protoreflect.ValueOfMessage(redactMessage(list.Get(i).Message(), plan)))
+ }
+ default:
+ dst.Set(field, protoreflect.ValueOfMessage(redactMessage(value.Message(), plan)))
+ }
+}
+
+// copyAnyField copies an Any-typed field, redacting each packed message and
+// dropping any the registry does not permit for this field.
+func copyAnyField(dst protoreflect.Message, field protoreflect.FieldDescriptor, value protoreflect.Value) {
+ switch {
+ case field.IsMap():
+ out := dst.Mutable(field).Map()
+ value.Map().Range(func(key protoreflect.MapKey, entry protoreflect.Value) bool {
+ if redacted, ok := redactPackedAny(field.FullName(), entry.Message()); ok {
+ out.Set(key, protoreflect.ValueOfMessage(redacted))
+ }
+ return true
+ })
+ case field.IsList():
+ out := dst.Mutable(field).List()
+ list := value.List()
+ for i := range list.Len() {
+ if redacted, ok := redactPackedAny(field.FullName(), list.Get(i).Message()); ok {
+ out.Append(protoreflect.ValueOfMessage(redacted))
+ }
+ }
+ default:
+ if redacted, ok := redactPackedAny(field.FullName(), value.Message()); ok {
+ dst.Set(field, protoreflect.ValueOfMessage(redacted))
+ }
+ }
+}
+
+// redactAuditServiceData redacts the Any a handler attached to the row through
+// common.SetServiceData — an UpdateSetting before-image, whose Setting carries
+// SMTP and AI credentials, or a SetIamPolicy delta. Returns nil when the packed
+// type is not one the registry names for this field.
+func redactAuditServiceData(serviceData *anypb.Any) *anypb.Any {
+ if serviceData == nil {
+ return nil
+ }
+ redacted, ok := redactPackedAny(auditServiceDataField, serviceData.ProtoReflect())
+ if !ok {
+ return nil
+ }
+ out, isAny := redacted.Interface().(*anypb.Any)
+ if !isAny {
+ return nil
+ }
+ return out
+}
+
+// redactAuditStatus rebuilds a failed RPC's status with only the details the
+// registry permits.
+//
+// Not about secrecy: PermissionDeniedDetail and PlanCheckRun.Result carry
+// nothing annotated. It is about keeping the row at all — protojson.Marshal
+// fails the WHOLE AuditLog on an Any it cannot resolve, so one unrecognized
+// detail from a wrapped upstream error would cost the entire record rather
+// than just its own payload.
+func redactAuditStatus(status *spb.Status) *spb.Status {
+ if len(status.GetDetails()) == 0 {
+ return status
+ }
+ details := make([]*anypb.Any, 0, len(status.GetDetails()))
+ for _, detail := range status.GetDetails() {
+ redacted, ok := redactPackedAny(auditStatusDetailsField, detail.ProtoReflect())
+ if !ok {
+ continue
+ }
+ if out, ok := redacted.Interface().(*anypb.Any); ok {
+ details = append(details, out)
+ }
+ }
+ if len(details) == len(status.GetDetails()) {
+ return status
+ }
+ // Copied rather than edited: the caller's error is still in flight.
+ return &spb.Status{Code: status.GetCode(), Message: status.GetMessage(), Details: details}
+}
+
+// redactPackedAny redacts one google.protobuf.Any. The bool reports whether it
+// may be recorded at all: an Any this function cannot vouch for is DROPPED, not
+// logged, because protojson.Marshal fails the ENTIRE audit row on an
+// unresolvable one — keeping it would lose the record rather than the payload.
+//
+// Vouching means resolving the type through the SAME registry protojson will
+// use and parsing the bytes, whether or not there is anything to redact.
+// Checking only the types that happen to have a plan would leave the drop
+// guarantee off exactly the types that do not — both Status.details types among
+// them — so a corrupt payload there would take the whole row down.
+//
+// Losing a payload silently is its own defect, so a drop is logged.
+func redactPackedAny(field protoreflect.FullName, packed protoreflect.Message) (protoreflect.Message, bool) {
+ fields := packed.Descriptor().Fields()
+ typeURLField := fields.ByName("type_url")
+ valueField := fields.ByName("value")
+ if typeURLField == nil || valueField == nil {
+ return nil, false
+ }
+ typeURL := packed.Get(typeURLField).String()
+ name := packedTypeName(typeURL)
+ drop := func(reason string) (protoreflect.Message, bool) {
+ slog.Warn("audit: dropped an Any from the audit row",
+ slog.String("field", string(field)),
+ slog.String("type", string(name)),
+ slog.String("reason", reason))
+ return nil, false
+ }
+ if !auditAnyPermits(field, name) {
+ return drop("type is not in auditAnyRegistry for this field")
+ }
+
+ messageType, err := protoregistry.GlobalTypes.FindMessageByName(name)
+ if err != nil {
+ return drop("type is not in the global type registry")
+ }
+ inner := messageType.New().Interface()
+ if err := proto.Unmarshal(packed.Get(valueField).Bytes(), inner); err != nil {
+ return drop("packed bytes do not parse")
+ }
+
+ plan := planFor(messageType.Descriptor())
+ if plan == nil {
+ // Resolvable and parseable, with nothing to redact. Returned as found
+ // so the type_url keeps whatever form the producer wrote: connect's
+ // ErrorDetail.Type() returns a bare full name, and a round-trip through
+ // anypb.New would rewrite it to type.googleapis.com/... and silently
+ // change what SearchAuditLogs emits as @type.
+ return packed, true
+ }
+
+ body, err := proto.Marshal(redactMessage(inner.ProtoReflect(), plan).Interface())
+ if err != nil {
+ slog.Warn("audit: dropped an Any from the audit row", log.BBError(err),
+ slog.String("field", string(field)), slog.String("type", string(name)),
+ slog.String("reason", "redacted message does not marshal"))
+ return nil, false
+ }
+ out := packed.New()
+ out.Set(typeURLField, protoreflect.ValueOfString(typeURL))
+ out.Set(valueField, protoreflect.ValueOfBytes(body))
+ return out, true
+}
+
+// packedTypeName strips the optional "type.googleapis.com/" prefix an Any's
+// type_url may carry.
+func packedTypeName(typeURL string) protoreflect.FullName {
+ for i := len(typeURL) - 1; i >= 0; i-- {
+ if typeURL[i] == '/' {
+ return protoreflect.FullName(typeURL[i+1:])
+ }
+ }
+ return protoreflect.FullName(typeURL)
+}
diff --git a/backend/api/v1/audit_redact_inventory_test.go b/backend/api/v1/audit_redact_inventory_test.go
new file mode 100644
index 0000000000..c137870f4e
--- /dev/null
+++ b/backend/api/v1/audit_redact_inventory_test.go
@@ -0,0 +1,1183 @@
+package v1
+
+import (
+ "fmt"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/reflect/protoreflect"
+ "google.golang.org/protobuf/reflect/protoregistry"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// The inventory is the oracle the coverage sweep cannot be. That sweep proves
+// annotated fields get blanked; it cannot tell an unannotated credential from
+// an ordinary field the row intentionally keeps, because both survive
+// redaction identically.
+//
+// So every string and bytes field reachable from an audit payload is listed
+// here, minus the annotated ones. A field missing from the list fails the
+// build; clearing that failure means annotating it or adding a line saying the
+// row may keep it. Annotated fields are absent by design, so the list reads as
+// "what the audit row writes down" and a new credential shows up as one line of
+// diff.
+//
+// The population is wider than the audited RPCs: WrapUnary writes a row on
+// `needAudit(ctx) || mcpPolicyDenied`, so gate-refused methods carrying no
+// audit annotation are in scope. Every registered Any type is in scope too,
+// since those reach the row without passing marshalAuditPayload — which is why
+// the registry is enforced at its call sites for this list to mean anything.
+//
+// The bytebase.store entries are not a layering slip: storepb.AuditLog IS the
+// audit row, protojson-marshaled into audit_log.payload by store.CreateAuditLog
+// and printed by logAuditToStdout. RequestMetadata and MCPDelegation are there
+// because createAuditLog assigns them directly, so nothing else would look at
+// them — and RequestMetadata.caller_supplied_user_agent is caller-controlled.
+
+// TestLintAuditPayloadInventory fails when a string or bytes field joins or
+// leaves the audited surface without anyone deciding what the audit row may
+// record for it.
+func TestLintAuditPayloadInventory(t *testing.T) {
+ found := auditRecordedScalarFields(t)
+
+ recorded := map[string]bool{}
+ for _, field := range auditRecordedFields {
+ require.False(t, recorded[field], "%s is listed twice", field)
+ recorded[field] = true
+ }
+
+ var undecided, unresolvable, unreachable []string
+ for _, field := range found {
+ if !recorded[field] {
+ undecided = append(undecided, field)
+ }
+ }
+ // A listed name stops meaning anything in two different ways, and the fix
+ // differs, so they are reported apart rather than as one "stale" bucket.
+ for field := range recorded {
+ switch {
+ case slices.Contains(found, field):
+ case inventoryFieldExists(field):
+ unreachable = append(unreachable, field)
+ default:
+ unresolvable = append(unresolvable, field)
+ }
+ }
+ slices.Sort(undecided)
+ slices.Sort(unresolvable)
+ slices.Sort(unreachable)
+
+ require.Empty(t, undecided,
+ "these reach an audit payload and nobody has decided about them: annotate the field "+
+ "(bytebase.v1.audit_behavior) = SENSITIVE or OMIT, or add it to auditRecordedFields to record that "+
+ "the audit row may keep it")
+ require.Empty(t, unresolvable,
+ "these name a field that does not exist, which reads as coverage while asserting nothing; drop them")
+ require.Empty(t, unreachable,
+ "these no longer reach an audit payload, or are now annotated; drop them so the list keeps meaning something")
+
+ require.True(t, slices.IsSorted(auditRecordedFields),
+ "auditRecordedFields is kept sorted so a new entry reads as one line of diff")
+}
+
+// auditRecordedScalarFields is every (message, string-or-bytes field) pair the
+// audit row can record, derived from the descriptors rather than from reading.
+// Reading is how the first sweep of this surface missed ListInstanceDatabase,
+// whose request carries an entire Instance.
+func auditRecordedScalarFields(t *testing.T) []string {
+ t.Helper()
+ types := map[protoreflect.FullName]bool{}
+ var walk func(protoreflect.MessageDescriptor)
+ walk = func(descriptor protoreflect.MessageDescriptor) {
+ if types[descriptor.FullName()] {
+ return
+ }
+ types[descriptor.FullName()] = true
+ fields := descriptor.Fields()
+ for i := range fields.Len() {
+ field := fields.Get(i)
+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {
+ continue
+ }
+ if sub := submessageOf(field); sub != nil {
+ walk(sub)
+ }
+ }
+ }
+
+ // Every v1 method, both directions.
+ protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
+ if fd.Package() != "bytebase.v1" {
+ return true
+ }
+ services := fd.Services()
+ for i := range services.Len() {
+ methods := services.Get(i).Methods()
+ for j := range methods.Len() {
+ walk(methods.Get(j).Input())
+ walk(methods.Get(j).Output())
+ }
+ }
+ return true
+ })
+
+ // The audit row itself: service_data, status.details, request_metadata and
+ // mcp_delegation never pass an entry point.
+ walk((&storepb.AuditLog{}).ProtoReflect().Descriptor())
+
+ // Every registered Any type. These reach the row packed, so no descriptor
+ // walk finds them: registering a type is what pulls its fields in here.
+ for _, packed := range auditAnyRegistry {
+ for _, name := range packed {
+ descriptor, err := protoregistry.GlobalFiles.FindDescriptorByName(name)
+ require.NoError(t, err, "registered Any type %s is not in the descriptor registry", name)
+ md, ok := descriptor.(protoreflect.MessageDescriptor)
+ require.True(t, ok, "registered Any type %s is not a message", name)
+ walk(md)
+ }
+ }
+
+ var found []string
+ for name := range types {
+ descriptor, err := protoregistry.GlobalFiles.FindDescriptorByName(name)
+ if err != nil {
+ continue
+ }
+ md, ok := descriptor.(protoreflect.MessageDescriptor)
+ if !ok {
+ continue
+ }
+ fields := md.Fields()
+ for i := range fields.Len() {
+ field := fields.Get(i)
+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {
+ continue
+ }
+ kind := field.Kind()
+ if field.IsMap() {
+ kind = field.MapValue().Kind()
+ }
+ if kind == protoreflect.StringKind || kind == protoreflect.BytesKind {
+ found = append(found, fmt.Sprintf("%s.%s", name, field.Name()))
+ }
+ }
+ }
+ slices.Sort(found)
+ return slices.Compact(found)
+}
+
+// inventoryFieldExists reports whether a "message.field" entry still resolves.
+func inventoryFieldExists(entry string) bool {
+ cut := strings.LastIndex(entry, ".")
+ if cut <= 0 {
+ return false
+ }
+ descriptor, err := protoregistry.GlobalFiles.FindDescriptorByName(protoreflect.FullName(entry[:cut]))
+ if err != nil {
+ return false
+ }
+ md, ok := descriptor.(protoreflect.MessageDescriptor)
+ return ok && md.Fields().ByName(protoreflect.Name(entry[cut+1:])) != nil
+}
+
+// auditRecordedFields is every string and bytes field an audit payload may
+// record, sorted, one per line.
+//
+// The rule for adding one is mechanical: if the value would let someone
+// authenticate, authorize, or act as somebody — or is an unbounded body, a
+// blob, or personal data the row has no business keeping — it belongs on the
+// annotation instead of here.
+//
+// Names that look like credentials and are not, so the next reader does not
+// re-litigate them: DataSourceExternalSecret's url, engine_name, secret_name,
+// password_key_name and mount_path say WHERE the secret lives, never what it
+// is; DataSource.AzureCredential's tenant_id and client_id name the principal
+// rather than authenticate as it; google.protobuf.Any.type_url and value are
+// the envelope of a packed message whose own fields come in through
+// auditAnyRegistry. AuditLog.request and .response are the redacted payloads
+// themselves, kept because the rule is every scalar on the row with no
+// exceptions — a new column lands here and forces a decision.
+var auditRecordedFields = []string{
+ "bytebase.store.AuditLog.method",
+ "bytebase.store.AuditLog.parent",
+ "bytebase.store.AuditLog.request",
+ "bytebase.store.AuditLog.resource",
+ "bytebase.store.AuditLog.response",
+ "bytebase.store.AuditLog.user",
+ "bytebase.store.MCPDelegation.client_id",
+ "bytebase.store.MCPDelegation.correlation_id",
+ "bytebase.store.MCPDelegation.resource",
+ "bytebase.store.MCPDelegation.scope",
+ "bytebase.store.RequestMetadata.caller_ip",
+ "bytebase.store.RequestMetadata.caller_supplied_user_agent",
+ "bytebase.v1.AIChatResponse.content",
+ "bytebase.v1.AIChatToolCall.arguments",
+ "bytebase.v1.AIChatToolCall.id",
+ "bytebase.v1.AIChatToolCall.metadata",
+ "bytebase.v1.AIChatToolCall.name",
+ "bytebase.v1.AISetting.endpoint",
+ "bytebase.v1.AISetting.model",
+ "bytebase.v1.AISetting.version",
+ "bytebase.v1.AccessGrant.container",
+ "bytebase.v1.AccessGrant.creator",
+ "bytebase.v1.AccessGrant.issue",
+ "bytebase.v1.AccessGrant.name",
+ "bytebase.v1.AccessGrant.query",
+ "bytebase.v1.AccessGrant.reason",
+ "bytebase.v1.AccessGrant.schema",
+ "bytebase.v1.AccessGrant.targets",
+ "bytebase.v1.ActivateAccessGrantRequest.name",
+ "bytebase.v1.ActuatorInfo.default_project",
+ "bytebase.v1.ActuatorInfo.external_url",
+ "bytebase.v1.ActuatorInfo.git_commit",
+ "bytebase.v1.ActuatorInfo.unlicensed_features",
+ "bytebase.v1.ActuatorInfo.version",
+ "bytebase.v1.ActuatorInfo.workspace",
+ "bytebase.v1.AddDataSourceRequest.name",
+ "bytebase.v1.AddWebhookRequest.project",
+ "bytebase.v1.AdminExecuteRequest.container",
+ "bytebase.v1.AdminExecuteRequest.name",
+ "bytebase.v1.AdminExecuteRequest.schema",
+ "bytebase.v1.AdminExecuteRequest.statement",
+ "bytebase.v1.Advice.content",
+ "bytebase.v1.Advice.title",
+ "bytebase.v1.Algorithm.FullMask.substitution",
+ "bytebase.v1.Algorithm.InnerOuterMask.substitution",
+ "bytebase.v1.Algorithm.MD5Mask.salt",
+ "bytebase.v1.Algorithm.RangeMask.Slice.substitution",
+ "bytebase.v1.Announcement.link",
+ "bytebase.v1.Announcement.text",
+ "bytebase.v1.AppIMSetting.DingTalk.client_id",
+ "bytebase.v1.AppIMSetting.DingTalk.robot_code",
+ "bytebase.v1.AppIMSetting.Feishu.app_id",
+ "bytebase.v1.AppIMSetting.Lark.app_id",
+ "bytebase.v1.AppIMSetting.Teams.client_id",
+ "bytebase.v1.AppIMSetting.Teams.tenant_id",
+ "bytebase.v1.AppIMSetting.Wecom.agent_id",
+ "bytebase.v1.AppIMSetting.Wecom.corp_id",
+ "bytebase.v1.ApprovalFlow.roles",
+ "bytebase.v1.ApprovalTemplate.description",
+ "bytebase.v1.ApprovalTemplate.title",
+ "bytebase.v1.ApproveIssueRequest.comment",
+ "bytebase.v1.ApproveIssueRequest.name",
+ "bytebase.v1.AuditLog.method",
+ "bytebase.v1.AuditLog.name",
+ "bytebase.v1.AuditLog.request",
+ "bytebase.v1.AuditLog.resource",
+ "bytebase.v1.AuditLog.response",
+ "bytebase.v1.AuditLog.user",
+ "bytebase.v1.AuthenticationInfo.workspace",
+ "bytebase.v1.BatchCancelTaskRunsRequest.parent",
+ "bytebase.v1.BatchCancelTaskRunsRequest.task_runs",
+ "bytebase.v1.BatchCreateRevisionsRequest.parent",
+ "bytebase.v1.BatchCreateSheetsRequest.parent",
+ "bytebase.v1.BatchDeleteProjectsRequest.names",
+ "bytebase.v1.BatchDeparseResponse.expressions",
+ "bytebase.v1.BatchGetDatabasesRequest.names",
+ "bytebase.v1.BatchGetDatabasesRequest.parent",
+ "bytebase.v1.BatchGetGroupsRequest.names",
+ "bytebase.v1.BatchGetProjectsRequest.names",
+ "bytebase.v1.BatchGetUsersRequest.names",
+ "bytebase.v1.BatchParseRequest.expressions",
+ "bytebase.v1.BatchRunTasksRequest.parent",
+ "bytebase.v1.BatchRunTasksRequest.tasks",
+ "bytebase.v1.BatchSkipTasksRequest.parent",
+ "bytebase.v1.BatchSkipTasksRequest.reason",
+ "bytebase.v1.BatchSkipTasksRequest.tasks",
+ "bytebase.v1.BatchSyncDatabasesRequest.names",
+ "bytebase.v1.BatchSyncDatabasesRequest.parent",
+ "bytebase.v1.BatchSyncInstancesRequest.parent",
+ "bytebase.v1.BatchUpdateDatabasesRequest.parent",
+ "bytebase.v1.BatchUpdateInstancesRequest.parent",
+ "bytebase.v1.BatchUpdateIssuesStatusRequest.issues",
+ "bytebase.v1.BatchUpdateIssuesStatusRequest.parent",
+ "bytebase.v1.BatchUpdateIssuesStatusRequest.reason",
+ "bytebase.v1.Binding.members",
+ "bytebase.v1.Binding.role",
+ "bytebase.v1.BindingDelta.member",
+ "bytebase.v1.BindingDelta.role",
+ "bytebase.v1.CancelPlanCheckRunRequest.name",
+ "bytebase.v1.CancelPurchaseRequest.comment",
+ "bytebase.v1.CancelPurchaseRequest.feedback",
+ "bytebase.v1.Changelog.name",
+ "bytebase.v1.Changelog.plan_title",
+ "bytebase.v1.Changelog.schema",
+ "bytebase.v1.Changelog.task_run",
+ "bytebase.v1.CheckConstraintMetadata.expression",
+ "bytebase.v1.CheckConstraintMetadata.name",
+ "bytebase.v1.CheckReleaseRequest.custom_rules",
+ "bytebase.v1.CheckReleaseRequest.parent",
+ "bytebase.v1.CheckReleaseRequest.targets",
+ "bytebase.v1.CheckReleaseResponse.CheckResult.file",
+ "bytebase.v1.CheckReleaseResponse.CheckResult.target",
+ "bytebase.v1.ColumnCatalog.classification",
+ "bytebase.v1.ColumnCatalog.labels",
+ "bytebase.v1.ColumnCatalog.name",
+ "bytebase.v1.ColumnCatalog.semantic_type",
+ "bytebase.v1.ColumnMetadata.character_set",
+ "bytebase.v1.ColumnMetadata.collation",
+ "bytebase.v1.ColumnMetadata.comment",
+ "bytebase.v1.ColumnMetadata.default",
+ "bytebase.v1.ColumnMetadata.default_constraint_name",
+ "bytebase.v1.ColumnMetadata.name",
+ "bytebase.v1.ColumnMetadata.on_update",
+ "bytebase.v1.ColumnMetadata.type",
+ "bytebase.v1.CompositeTypeAttribute.collation",
+ "bytebase.v1.CompositeTypeAttribute.comment",
+ "bytebase.v1.CompositeTypeAttribute.name",
+ "bytebase.v1.CompositeTypeAttribute.type",
+ "bytebase.v1.CompositeTypeMetadata.comment",
+ "bytebase.v1.CompositeTypeMetadata.name",
+ "bytebase.v1.CreateAccessGrantRequest.parent",
+ "bytebase.v1.CreateDatabaseGroupRequest.database_group_id",
+ "bytebase.v1.CreateDatabaseGroupRequest.parent",
+ "bytebase.v1.CreateGroupRequest.group_email",
+ "bytebase.v1.CreateIdentityProviderRequest.identity_provider_id",
+ "bytebase.v1.CreateInstanceRequest.instance_id",
+ "bytebase.v1.CreateInstanceRequest.parent",
+ "bytebase.v1.CreateIssueCommentRequest.parent",
+ "bytebase.v1.CreateIssueRequest.parent",
+ "bytebase.v1.CreatePlanRequest.parent",
+ "bytebase.v1.CreatePolicyRequest.parent",
+ "bytebase.v1.CreateProjectRequest.project_id",
+ "bytebase.v1.CreateReleaseRequest.parent",
+ "bytebase.v1.CreateReleaseRequest.release_id_template",
+ "bytebase.v1.CreateReleaseRequest.release_id_timezone",
+ "bytebase.v1.CreateRevisionRequest.parent",
+ "bytebase.v1.CreateRoleRequest.role_id",
+ "bytebase.v1.CreateRolloutRequest.parent",
+ "bytebase.v1.CreateRolloutRequest.target",
+ "bytebase.v1.CreateSavedQueryRequest.parent",
+ "bytebase.v1.CreateServiceAccountRequest.parent",
+ "bytebase.v1.CreateServiceAccountRequest.service_account_id",
+ "bytebase.v1.CreateSheetRequest.parent",
+ "bytebase.v1.CreateWorkloadIdentityRequest.parent",
+ "bytebase.v1.CreateWorkloadIdentityRequest.workload_identity_id",
+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.DataClassification.id",
+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.DataClassification.title",
+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.Level.title",
+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.id",
+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.title",
+ "bytebase.v1.DataSource.Address.host",
+ "bytebase.v1.DataSource.Address.port",
+ "bytebase.v1.DataSource.AzureCredential.client_id",
+ "bytebase.v1.DataSource.AzureCredential.tenant_id",
+ "bytebase.v1.DataSource.authentication_database",
+ "bytebase.v1.DataSource.database",
+ "bytebase.v1.DataSource.extra_connection_parameters",
+ "bytebase.v1.DataSource.host",
+ "bytebase.v1.DataSource.id",
+ "bytebase.v1.DataSource.instance_id",
+ "bytebase.v1.DataSource.master_name",
+ "bytebase.v1.DataSource.master_username",
+ "bytebase.v1.DataSource.port",
+ "bytebase.v1.DataSource.project_id",
+ "bytebase.v1.DataSource.region",
+ "bytebase.v1.DataSource.replica_set",
+ "bytebase.v1.DataSource.service_name",
+ "bytebase.v1.DataSource.sid",
+ "bytebase.v1.DataSource.ssh_host",
+ "bytebase.v1.DataSource.ssh_port",
+ "bytebase.v1.DataSource.ssh_user",
+ "bytebase.v1.DataSource.username",
+ "bytebase.v1.DataSource.warehouse_id",
+ "bytebase.v1.DataSourceExternalSecret.AppRoleAuthOption.mount_path",
+ "bytebase.v1.DataSourceExternalSecret.engine_name",
+ "bytebase.v1.DataSourceExternalSecret.password_key_name",
+ "bytebase.v1.DataSourceExternalSecret.secret_name",
+ "bytebase.v1.DataSourceExternalSecret.url",
+ "bytebase.v1.Database.effective_environment",
+ "bytebase.v1.Database.environment",
+ "bytebase.v1.Database.labels",
+ "bytebase.v1.Database.name",
+ "bytebase.v1.Database.project",
+ "bytebase.v1.Database.release",
+ "bytebase.v1.Database.sync_error",
+ "bytebase.v1.DatabaseCatalog.name",
+ "bytebase.v1.DatabaseGroup.Database.name",
+ "bytebase.v1.DatabaseGroup.name",
+ "bytebase.v1.DatabaseGroup.title",
+ "bytebase.v1.DatabaseMetadata.character_set",
+ "bytebase.v1.DatabaseMetadata.collation",
+ "bytebase.v1.DatabaseMetadata.name",
+ "bytebase.v1.DatabaseMetadata.owner",
+ "bytebase.v1.DatabaseMetadata.search_path",
+ "bytebase.v1.DatabaseSDLSchema.content_type",
+ "bytebase.v1.DatabaseSDLSchema.schema",
+ "bytebase.v1.DatabaseSchema.schema",
+ "bytebase.v1.DeleteDatabaseGroupRequest.name",
+ "bytebase.v1.DeleteGroupRequest.name",
+ "bytebase.v1.DeleteIdentityProviderRequest.name",
+ "bytebase.v1.DeleteInstanceRequest.name",
+ "bytebase.v1.DeletePolicyRequest.name",
+ "bytebase.v1.DeleteProjectRequest.name",
+ "bytebase.v1.DeleteReleaseRequest.name",
+ "bytebase.v1.DeleteReviewConfigRequest.name",
+ "bytebase.v1.DeleteRevisionRequest.name",
+ "bytebase.v1.DeleteRoleRequest.name",
+ "bytebase.v1.DeleteSavedQueryRequest.name",
+ "bytebase.v1.DeleteServiceAccountRequest.name",
+ "bytebase.v1.DeleteUserRequest.name",
+ "bytebase.v1.DeleteWorkloadIdentityRequest.name",
+ "bytebase.v1.DeleteWorkspaceRequest.name",
+ "bytebase.v1.DependencyColumn.column",
+ "bytebase.v1.DependencyColumn.schema",
+ "bytebase.v1.DependencyColumn.table",
+ "bytebase.v1.DependencyTable.schema",
+ "bytebase.v1.DependencyTable.table",
+ "bytebase.v1.DiffMetadataRequest.name",
+ "bytebase.v1.DiffMetadataResponse.diff",
+ "bytebase.v1.DiffSchemaRequest.changelog",
+ "bytebase.v1.DiffSchemaRequest.name",
+ "bytebase.v1.DiffSchemaRequest.schema",
+ "bytebase.v1.DiffSchemaResponse.diff",
+ "bytebase.v1.DimensionConstraint.dimension",
+ "bytebase.v1.DimensionalConfig.data_type",
+ "bytebase.v1.EmailSetting.SMTPConfig.host",
+ "bytebase.v1.EmailSetting.SMTPConfig.username",
+ "bytebase.v1.EmailSetting.from",
+ "bytebase.v1.EmailSetting.from_name",
+ "bytebase.v1.EnumTypeMetadata.comment",
+ "bytebase.v1.EnumTypeMetadata.name",
+ "bytebase.v1.EnumTypeMetadata.values",
+ "bytebase.v1.EnvironmentSetting.Environment.id",
+ "bytebase.v1.EnvironmentSetting.Environment.name",
+ "bytebase.v1.EnvironmentSetting.Environment.tags",
+ "bytebase.v1.EnvironmentSetting.Environment.title",
+ "bytebase.v1.EventMetadata.character_set_client",
+ "bytebase.v1.EventMetadata.collation_connection",
+ "bytebase.v1.EventMetadata.comment",
+ "bytebase.v1.EventMetadata.definition",
+ "bytebase.v1.EventMetadata.name",
+ "bytebase.v1.EventMetadata.sql_mode",
+ "bytebase.v1.EventMetadata.time_zone",
+ "bytebase.v1.ExchangeTokenRequest.email",
+ "bytebase.v1.ExportAuditLogsRequest.filter",
+ "bytebase.v1.ExportAuditLogsRequest.order_by",
+ "bytebase.v1.ExportAuditLogsRequest.page_token",
+ "bytebase.v1.ExportAuditLogsRequest.parent",
+ "bytebase.v1.ExportAuditLogsResponse.next_page_token",
+ "bytebase.v1.ExportRequest.container",
+ "bytebase.v1.ExportRequest.data_source_id",
+ "bytebase.v1.ExportRequest.name",
+ "bytebase.v1.ExportRequest.schema",
+ "bytebase.v1.ExportRequest.statement",
+ "bytebase.v1.ExportResponse.applied_access_grant",
+ "bytebase.v1.ExtensionMetadata.description",
+ "bytebase.v1.ExtensionMetadata.name",
+ "bytebase.v1.ExtensionMetadata.schema",
+ "bytebase.v1.ExtensionMetadata.version",
+ "bytebase.v1.ExternalTableMetadata.external_database_name",
+ "bytebase.v1.ExternalTableMetadata.external_server_name",
+ "bytebase.v1.ExternalTableMetadata.name",
+ "bytebase.v1.FieldMapping.display_name",
+ "bytebase.v1.FieldMapping.groups",
+ "bytebase.v1.FieldMapping.identifier",
+ "bytebase.v1.FieldMapping.phone",
+ "bytebase.v1.ForeignKeyMetadata.columns",
+ "bytebase.v1.ForeignKeyMetadata.match_type",
+ "bytebase.v1.ForeignKeyMetadata.name",
+ "bytebase.v1.ForeignKeyMetadata.on_delete",
+ "bytebase.v1.ForeignKeyMetadata.on_update",
+ "bytebase.v1.ForeignKeyMetadata.referenced_columns",
+ "bytebase.v1.ForeignKeyMetadata.referenced_schema",
+ "bytebase.v1.ForeignKeyMetadata.referenced_table",
+ "bytebase.v1.FunctionMetadata.character_set_client",
+ "bytebase.v1.FunctionMetadata.collation_connection",
+ "bytebase.v1.FunctionMetadata.comment",
+ "bytebase.v1.FunctionMetadata.database_collation",
+ "bytebase.v1.FunctionMetadata.definition",
+ "bytebase.v1.FunctionMetadata.name",
+ "bytebase.v1.FunctionMetadata.signature",
+ "bytebase.v1.FunctionMetadata.sql_mode",
+ "bytebase.v1.GenerationMetadata.expression",
+ "bytebase.v1.GetAccessGrantRequest.name",
+ "bytebase.v1.GetAuthenticationRestrictionRequest.workspace",
+ "bytebase.v1.GetChangelogRequest.name",
+ "bytebase.v1.GetDatabaseCatalogRequest.name",
+ "bytebase.v1.GetDatabaseGroupRequest.name",
+ "bytebase.v1.GetDatabaseMetadataRequest.filter",
+ "bytebase.v1.GetDatabaseMetadataRequest.name",
+ "bytebase.v1.GetDatabaseRequest.name",
+ "bytebase.v1.GetDatabaseSDLSchemaRequest.name",
+ "bytebase.v1.GetDatabaseSchemaRequest.name",
+ "bytebase.v1.GetGroupRequest.name",
+ "bytebase.v1.GetIamPolicyRequest.resource",
+ "bytebase.v1.GetIdentityProviderRequest.name",
+ "bytebase.v1.GetInstanceRequest.name",
+ "bytebase.v1.GetIssueRequest.name",
+ "bytebase.v1.GetPlanCheckRunRequest.name",
+ "bytebase.v1.GetPlanRequest.name",
+ "bytebase.v1.GetPolicyRequest.name",
+ "bytebase.v1.GetProjectRequest.name",
+ "bytebase.v1.GetQueryHistoryRequest.name",
+ "bytebase.v1.GetReleaseRequest.name",
+ "bytebase.v1.GetReviewConfigRequest.name",
+ "bytebase.v1.GetRevisionRequest.name",
+ "bytebase.v1.GetRoleRequest.name",
+ "bytebase.v1.GetRolloutRequest.name",
+ "bytebase.v1.GetSavedQueryPolicyRequest.resource",
+ "bytebase.v1.GetSavedQueryRequest.name",
+ "bytebase.v1.GetSchemaStringRequest.name",
+ "bytebase.v1.GetSchemaStringRequest.object",
+ "bytebase.v1.GetSchemaStringRequest.schema",
+ "bytebase.v1.GetSchemaStringResponse.schema_string",
+ "bytebase.v1.GetServiceAccountRequest.name",
+ "bytebase.v1.GetSettingRequest.name",
+ "bytebase.v1.GetSheetRequest.name",
+ "bytebase.v1.GetTaskRunLogRequest.parent",
+ "bytebase.v1.GetTaskRunRequest.name",
+ "bytebase.v1.GetTaskRunSessionRequest.parent",
+ "bytebase.v1.GetUserRequest.name",
+ "bytebase.v1.GetWorkloadIdentityRequest.name",
+ "bytebase.v1.GetWorkspaceRequest.name",
+ "bytebase.v1.GridLevel.density",
+ "bytebase.v1.Group.description",
+ "bytebase.v1.Group.email",
+ "bytebase.v1.Group.name",
+ "bytebase.v1.Group.source",
+ "bytebase.v1.Group.title",
+ "bytebase.v1.GroupMember.member",
+ "bytebase.v1.IamPolicy.etag",
+ "bytebase.v1.IdentityProvider.domain",
+ "bytebase.v1.IdentityProvider.name",
+ "bytebase.v1.IdentityProvider.title",
+ "bytebase.v1.IndexMetadata.comment",
+ "bytebase.v1.IndexMetadata.definition",
+ "bytebase.v1.IndexMetadata.expressions",
+ "bytebase.v1.IndexMetadata.name",
+ "bytebase.v1.IndexMetadata.opclass_names",
+ "bytebase.v1.IndexMetadata.parent_index_name",
+ "bytebase.v1.IndexMetadata.parent_index_schema",
+ "bytebase.v1.IndexMetadata.type",
+ "bytebase.v1.Instance.engine_version",
+ "bytebase.v1.Instance.environment",
+ "bytebase.v1.Instance.external_link",
+ "bytebase.v1.Instance.labels",
+ "bytebase.v1.Instance.name",
+ "bytebase.v1.Instance.title",
+ "bytebase.v1.InstanceResource.engine_version",
+ "bytebase.v1.InstanceResource.environment",
+ "bytebase.v1.InstanceResource.name",
+ "bytebase.v1.InstanceResource.title",
+ "bytebase.v1.InstanceRole.attribute",
+ "bytebase.v1.InstanceRole.name",
+ "bytebase.v1.InstanceRole.role_name",
+ "bytebase.v1.InstanceRole.valid_until",
+ "bytebase.v1.Issue.Approver.principal",
+ "bytebase.v1.Issue.access_grant",
+ "bytebase.v1.Issue.creator",
+ "bytebase.v1.Issue.description",
+ "bytebase.v1.Issue.labels",
+ "bytebase.v1.Issue.name",
+ "bytebase.v1.Issue.plan",
+ "bytebase.v1.Issue.title",
+ "bytebase.v1.IssueComment.IssueUpdate.from_description",
+ "bytebase.v1.IssueComment.IssueUpdate.from_labels",
+ "bytebase.v1.IssueComment.IssueUpdate.from_title",
+ "bytebase.v1.IssueComment.IssueUpdate.to_description",
+ "bytebase.v1.IssueComment.IssueUpdate.to_labels",
+ "bytebase.v1.IssueComment.IssueUpdate.to_title",
+ "bytebase.v1.IssueComment.comment",
+ "bytebase.v1.IssueComment.creator",
+ "bytebase.v1.IssueComment.name",
+ "bytebase.v1.IssueComment.payload",
+ "bytebase.v1.KerberosConfig.instance",
+ "bytebase.v1.KerberosConfig.kdc_host",
+ "bytebase.v1.KerberosConfig.kdc_port",
+ "bytebase.v1.KerberosConfig.kdc_transport_protocol",
+ "bytebase.v1.KerberosConfig.primary",
+ "bytebase.v1.KerberosConfig.realm",
+ "bytebase.v1.LDAPIdentityProviderConfig.base_dn",
+ "bytebase.v1.LDAPIdentityProviderConfig.bind_dn",
+ "bytebase.v1.LDAPIdentityProviderConfig.host",
+ "bytebase.v1.LDAPIdentityProviderConfig.user_filter",
+ "bytebase.v1.LDAPIdentityProviderTestRequestContext.username",
+ "bytebase.v1.Label.group",
+ "bytebase.v1.Label.value",
+ "bytebase.v1.LeaveWorkspaceRequest.name",
+ "bytebase.v1.ListAccessGrantsRequest.filter",
+ "bytebase.v1.ListAccessGrantsRequest.order_by",
+ "bytebase.v1.ListAccessGrantsRequest.page_token",
+ "bytebase.v1.ListAccessGrantsRequest.parent",
+ "bytebase.v1.ListAccessGrantsResponse.next_page_token",
+ "bytebase.v1.ListChangelogsRequest.filter",
+ "bytebase.v1.ListChangelogsRequest.page_token",
+ "bytebase.v1.ListChangelogsRequest.parent",
+ "bytebase.v1.ListChangelogsResponse.next_page_token",
+ "bytebase.v1.ListDatabaseGroupsRequest.parent",
+ "bytebase.v1.ListDatabasesRequest.filter",
+ "bytebase.v1.ListDatabasesRequest.order_by",
+ "bytebase.v1.ListDatabasesRequest.page_token",
+ "bytebase.v1.ListDatabasesRequest.parent",
+ "bytebase.v1.ListDatabasesResponse.next_page_token",
+ "bytebase.v1.ListGroupsRequest.filter",
+ "bytebase.v1.ListGroupsRequest.page_token",
+ "bytebase.v1.ListGroupsResponse.next_page_token",
+ "bytebase.v1.ListIdentityProvidersRequest.parent",
+ "bytebase.v1.ListInstanceDatabaseRequest.name",
+ "bytebase.v1.ListInstanceDatabaseResponse.databases",
+ "bytebase.v1.ListInstanceRolesRequest.page_token",
+ "bytebase.v1.ListInstanceRolesRequest.parent",
+ "bytebase.v1.ListInstanceRolesResponse.next_page_token",
+ "bytebase.v1.ListInstancesRequest.filter",
+ "bytebase.v1.ListInstancesRequest.order_by",
+ "bytebase.v1.ListInstancesRequest.page_token",
+ "bytebase.v1.ListInstancesRequest.parent",
+ "bytebase.v1.ListInstancesResponse.next_page_token",
+ "bytebase.v1.ListIssueCommentsRequest.page_token",
+ "bytebase.v1.ListIssueCommentsRequest.parent",
+ "bytebase.v1.ListIssueCommentsResponse.next_page_token",
+ "bytebase.v1.ListIssuesRequest.filter",
+ "bytebase.v1.ListIssuesRequest.order_by",
+ "bytebase.v1.ListIssuesRequest.page_token",
+ "bytebase.v1.ListIssuesRequest.parent",
+ "bytebase.v1.ListIssuesRequest.query",
+ "bytebase.v1.ListIssuesResponse.next_page_token",
+ "bytebase.v1.ListPlansRequest.filter",
+ "bytebase.v1.ListPlansRequest.page_token",
+ "bytebase.v1.ListPlansRequest.parent",
+ "bytebase.v1.ListPlansResponse.next_page_token",
+ "bytebase.v1.ListPoliciesRequest.parent",
+ "bytebase.v1.ListProjectsRequest.filter",
+ "bytebase.v1.ListProjectsRequest.order_by",
+ "bytebase.v1.ListProjectsRequest.page_token",
+ "bytebase.v1.ListProjectsResponse.next_page_token",
+ "bytebase.v1.ListQueryHistoriesRequest.filter",
+ "bytebase.v1.ListQueryHistoriesRequest.page_token",
+ "bytebase.v1.ListQueryHistoriesRequest.parent",
+ "bytebase.v1.ListQueryHistoriesResponse.next_page_token",
+ "bytebase.v1.ListReleaseCategoriesRequest.parent",
+ "bytebase.v1.ListReleaseCategoriesResponse.categories",
+ "bytebase.v1.ListReleasesRequest.filter",
+ "bytebase.v1.ListReleasesRequest.page_token",
+ "bytebase.v1.ListReleasesRequest.parent",
+ "bytebase.v1.ListReleasesResponse.next_page_token",
+ "bytebase.v1.ListRevisionsRequest.page_token",
+ "bytebase.v1.ListRevisionsRequest.parent",
+ "bytebase.v1.ListRevisionsResponse.next_page_token",
+ "bytebase.v1.ListRolloutsRequest.filter",
+ "bytebase.v1.ListRolloutsRequest.page_token",
+ "bytebase.v1.ListRolloutsRequest.parent",
+ "bytebase.v1.ListRolloutsResponse.next_page_token",
+ "bytebase.v1.ListSavedQueriesRequest.filter",
+ "bytebase.v1.ListSavedQueriesRequest.order_by",
+ "bytebase.v1.ListSavedQueriesRequest.page_token",
+ "bytebase.v1.ListSavedQueriesRequest.parent",
+ "bytebase.v1.ListSavedQueriesResponse.next_page_token",
+ "bytebase.v1.ListServiceAccountsRequest.filter",
+ "bytebase.v1.ListServiceAccountsRequest.page_token",
+ "bytebase.v1.ListServiceAccountsRequest.parent",
+ "bytebase.v1.ListServiceAccountsResponse.next_page_token",
+ "bytebase.v1.ListTaskRunsRequest.parent",
+ "bytebase.v1.ListUsersRequest.filter",
+ "bytebase.v1.ListUsersRequest.page_token",
+ "bytebase.v1.ListUsersResponse.next_page_token",
+ "bytebase.v1.ListWorkloadIdentitiesRequest.filter",
+ "bytebase.v1.ListWorkloadIdentitiesRequest.page_token",
+ "bytebase.v1.ListWorkloadIdentitiesRequest.parent",
+ "bytebase.v1.ListWorkloadIdentitiesResponse.next_page_token",
+ "bytebase.v1.LoginRequest.email",
+ "bytebase.v1.LoginRequest.idp_name",
+ "bytebase.v1.LoginRequest.workspace",
+ "bytebase.v1.MCPDelegation.client_id",
+ "bytebase.v1.MCPDelegation.correlation_id",
+ "bytebase.v1.MCPDelegation.resource",
+ "bytebase.v1.MCPDelegation.scope",
+ "bytebase.v1.MaskingExemptionPolicy.Exemption.members",
+ "bytebase.v1.MaskingReason.algorithm",
+ "bytebase.v1.MaskingReason.context",
+ "bytebase.v1.MaskingReason.masking_rule_id",
+ "bytebase.v1.MaskingReason.semantic_type_id",
+ "bytebase.v1.MaskingReason.semantic_type_title",
+ "bytebase.v1.MaskingRulePolicy.MaskingRule.id",
+ "bytebase.v1.MaskingRulePolicy.MaskingRule.semantic_type",
+ "bytebase.v1.MaterializedViewMetadata.comment",
+ "bytebase.v1.MaterializedViewMetadata.definition",
+ "bytebase.v1.MaterializedViewMetadata.name",
+ "bytebase.v1.MoveMySavedQueriesRequest.parent",
+ "bytebase.v1.MoveMySavedQueriesRequest.source_folder",
+ "bytebase.v1.MoveMySavedQueriesRequest.target_folder",
+ "bytebase.v1.OAuth2IdentityProviderConfig.auth_url",
+ "bytebase.v1.OAuth2IdentityProviderConfig.client_id",
+ "bytebase.v1.OAuth2IdentityProviderConfig.scopes",
+ "bytebase.v1.OAuth2IdentityProviderConfig.token_url",
+ "bytebase.v1.OAuth2IdentityProviderConfig.user_info_url",
+ "bytebase.v1.OIDCIdentityProviderConfig.auth_endpoint",
+ "bytebase.v1.OIDCIdentityProviderConfig.client_id",
+ "bytebase.v1.OIDCIdentityProviderConfig.issuer",
+ "bytebase.v1.OIDCIdentityProviderConfig.scopes",
+ "bytebase.v1.ObjectSchema.semantic_type",
+ "bytebase.v1.PackageMetadata.definition",
+ "bytebase.v1.PackageMetadata.name",
+ "bytebase.v1.PaymentInfo.currency",
+ "bytebase.v1.PaymentInfo.invoice_url",
+ "bytebase.v1.PaymentInfo.next_period_price",
+ "bytebase.v1.PaymentInfo.period_end",
+ "bytebase.v1.PaymentInfo.period_start",
+ "bytebase.v1.PaymentInfo.total_price",
+ "bytebase.v1.PermissionDeniedDetail.method",
+ "bytebase.v1.PermissionDeniedDetail.required_permissions",
+ "bytebase.v1.PermissionDeniedDetail.resources",
+ "bytebase.v1.Plan.ChangeDatabaseConfig.release",
+ "bytebase.v1.Plan.ChangeDatabaseConfig.sheet",
+ "bytebase.v1.Plan.ChangeDatabaseConfig.targets",
+ "bytebase.v1.Plan.CreateDatabaseConfig.character_set",
+ "bytebase.v1.Plan.CreateDatabaseConfig.cluster",
+ "bytebase.v1.Plan.CreateDatabaseConfig.collation",
+ "bytebase.v1.Plan.CreateDatabaseConfig.database",
+ "bytebase.v1.Plan.CreateDatabaseConfig.environment",
+ "bytebase.v1.Plan.CreateDatabaseConfig.owner",
+ "bytebase.v1.Plan.CreateDatabaseConfig.table",
+ "bytebase.v1.Plan.CreateDatabaseConfig.target",
+ "bytebase.v1.Plan.RolloutStageSummary.stage",
+ "bytebase.v1.Plan.Spec.id",
+ "bytebase.v1.Plan.creator",
+ "bytebase.v1.Plan.description",
+ "bytebase.v1.Plan.issue",
+ "bytebase.v1.Plan.name",
+ "bytebase.v1.Plan.title",
+ "bytebase.v1.PlanCheckRun.Result.content",
+ "bytebase.v1.PlanCheckRun.Result.target",
+ "bytebase.v1.PlanCheckRun.Result.title",
+ "bytebase.v1.PlanCheckRun.error",
+ "bytebase.v1.PlanCheckRun.name",
+ "bytebase.v1.Policy.name",
+ "bytebase.v1.PrepareSampleProjectInstanceRequest.parent",
+ "bytebase.v1.PreviewTaskRunRollbackRequest.name",
+ "bytebase.v1.PreviewTaskRunRollbackResponse.statement",
+ "bytebase.v1.ProcedureMetadata.character_set_client",
+ "bytebase.v1.ProcedureMetadata.collation_connection",
+ "bytebase.v1.ProcedureMetadata.comment",
+ "bytebase.v1.ProcedureMetadata.database_collation",
+ "bytebase.v1.ProcedureMetadata.definition",
+ "bytebase.v1.ProcedureMetadata.name",
+ "bytebase.v1.ProcedureMetadata.signature",
+ "bytebase.v1.ProcedureMetadata.sql_mode",
+ "bytebase.v1.Project.data_classification_config_id",
+ "bytebase.v1.Project.labels",
+ "bytebase.v1.Project.name",
+ "bytebase.v1.Project.title",
+ "bytebase.v1.QueryHistory.creator",
+ "bytebase.v1.QueryHistory.database",
+ "bytebase.v1.QueryHistory.error",
+ "bytebase.v1.QueryHistory.name",
+ "bytebase.v1.QueryHistory.statement",
+ "bytebase.v1.QueryRequest.container",
+ "bytebase.v1.QueryRequest.data_source_id",
+ "bytebase.v1.QueryRequest.name",
+ "bytebase.v1.QueryRequest.schema",
+ "bytebase.v1.QueryRequest.statement",
+ "bytebase.v1.QueryResponse.applied_access_grant",
+ "bytebase.v1.QueryResult.PostgresError.code",
+ "bytebase.v1.QueryResult.PostgresError.column_name",
+ "bytebase.v1.QueryResult.PostgresError.constraint_name",
+ "bytebase.v1.QueryResult.PostgresError.data_type_name",
+ "bytebase.v1.QueryResult.PostgresError.detail",
+ "bytebase.v1.QueryResult.PostgresError.file",
+ "bytebase.v1.QueryResult.PostgresError.hint",
+ "bytebase.v1.QueryResult.PostgresError.internal_query",
+ "bytebase.v1.QueryResult.PostgresError.message",
+ "bytebase.v1.QueryResult.PostgresError.routine",
+ "bytebase.v1.QueryResult.PostgresError.schema_name",
+ "bytebase.v1.QueryResult.PostgresError.severity",
+ "bytebase.v1.QueryResult.PostgresError.table_name",
+ "bytebase.v1.QueryResult.PostgresError.where",
+ "bytebase.v1.QueryResult.column_names",
+ "bytebase.v1.QueryResult.column_type_names",
+ "bytebase.v1.QueryResult.error",
+ "bytebase.v1.QueryResult.statement",
+ "bytebase.v1.RejectIssueRequest.comment",
+ "bytebase.v1.RejectIssueRequest.name",
+ "bytebase.v1.Release.File.path",
+ "bytebase.v1.Release.File.sheet",
+ "bytebase.v1.Release.File.sheet_sha256",
+ "bytebase.v1.Release.File.version",
+ "bytebase.v1.Release.VCSSource.url",
+ "bytebase.v1.Release.category",
+ "bytebase.v1.Release.creator",
+ "bytebase.v1.Release.name",
+ "bytebase.v1.RemoveDataSourceRequest.name",
+ "bytebase.v1.RequestIssueRequest.comment",
+ "bytebase.v1.RequestIssueRequest.name",
+ "bytebase.v1.RequestMetadata.caller_ip",
+ "bytebase.v1.RequestMetadata.caller_supplied_user_agent",
+ "bytebase.v1.RequestPasswordResetRequest.email",
+ "bytebase.v1.RequestPasswordResetRequest.workspace",
+ "bytebase.v1.ResetPasswordRequest.email",
+ "bytebase.v1.RetryIssueApprovalRequest.name",
+ "bytebase.v1.ReviewConfig.name",
+ "bytebase.v1.ReviewConfig.resources",
+ "bytebase.v1.ReviewConfig.title",
+ "bytebase.v1.Revision.deleter",
+ "bytebase.v1.Revision.file",
+ "bytebase.v1.Revision.name",
+ "bytebase.v1.Revision.release",
+ "bytebase.v1.Revision.sheet",
+ "bytebase.v1.Revision.sheet_sha256",
+ "bytebase.v1.Revision.task_run",
+ "bytebase.v1.Revision.version",
+ "bytebase.v1.RevokeAccessGrantRequest.name",
+ "bytebase.v1.Role.description",
+ "bytebase.v1.Role.name",
+ "bytebase.v1.Role.permissions",
+ "bytebase.v1.Role.title",
+ "bytebase.v1.RoleGrant.role",
+ "bytebase.v1.RoleGrant.user",
+ "bytebase.v1.Rollout.name",
+ "bytebase.v1.Rollout.title",
+ "bytebase.v1.RolloutPolicy.roles",
+ "bytebase.v1.RotateDirectorySyncTokenRequest.name",
+ "bytebase.v1.RunPlanChecksRequest.name",
+ "bytebase.v1.RunPlanChecksRequest.spec_id",
+ "bytebase.v1.SQLEditorThemeSetting.id",
+ "bytebase.v1.SQLEditorThemeSetting.monaco_base",
+ "bytebase.v1.SQLEditorThemeSetting.name",
+ "bytebase.v1.SQLReviewRule.NamingRulePayload.format",
+ "bytebase.v1.SQLReviewRule.StringArrayRulePayload.list",
+ "bytebase.v1.SQLReviewRule.StringRulePayload.value",
+ "bytebase.v1.SavedQuery.creator",
+ "bytebase.v1.SavedQuery.database",
+ "bytebase.v1.SavedQuery.folder",
+ "bytebase.v1.SavedQuery.name",
+ "bytebase.v1.SavedQuery.project",
+ "bytebase.v1.SavedQuery.title",
+ "bytebase.v1.SavedQueryBinding.members",
+ "bytebase.v1.SavedQueryPolicy.etag",
+ "bytebase.v1.SchemaCatalog.name",
+ "bytebase.v1.SchemaMetadata.comment",
+ "bytebase.v1.SchemaMetadata.name",
+ "bytebase.v1.SchemaMetadata.owner",
+ "bytebase.v1.SearchAuditLogsRequest.filter",
+ "bytebase.v1.SearchAuditLogsRequest.order_by",
+ "bytebase.v1.SearchAuditLogsRequest.page_token",
+ "bytebase.v1.SearchAuditLogsRequest.parent",
+ "bytebase.v1.SearchAuditLogsResponse.next_page_token",
+ "bytebase.v1.SearchIssuesRequest.filter",
+ "bytebase.v1.SearchIssuesRequest.order_by",
+ "bytebase.v1.SearchIssuesRequest.page_token",
+ "bytebase.v1.SearchIssuesRequest.parent",
+ "bytebase.v1.SearchIssuesRequest.query",
+ "bytebase.v1.SearchIssuesResponse.next_page_token",
+ "bytebase.v1.SearchMyAccessGrantsRequest.filter",
+ "bytebase.v1.SearchMyAccessGrantsRequest.order_by",
+ "bytebase.v1.SearchMyAccessGrantsRequest.page_token",
+ "bytebase.v1.SearchMyAccessGrantsRequest.parent",
+ "bytebase.v1.SearchMyAccessGrantsResponse.next_page_token",
+ "bytebase.v1.SearchProjectsRequest.filter",
+ "bytebase.v1.SearchProjectsRequest.order_by",
+ "bytebase.v1.SearchProjectsRequest.page_token",
+ "bytebase.v1.SearchProjectsResponse.next_page_token",
+ "bytebase.v1.SearchQueryHistoriesRequest.filter",
+ "bytebase.v1.SearchQueryHistoriesRequest.page_token",
+ "bytebase.v1.SearchQueryHistoriesRequest.parent",
+ "bytebase.v1.SearchQueryHistoriesResponse.next_page_token",
+ "bytebase.v1.SearchSavedQueriesRequest.filter",
+ "bytebase.v1.SearchSavedQueriesRequest.page_token",
+ "bytebase.v1.SearchSavedQueriesRequest.parent",
+ "bytebase.v1.SearchSavedQueriesResponse.next_page_token",
+ "bytebase.v1.SearchSavedQueryFoldersRequest.filter",
+ "bytebase.v1.SearchSavedQueryFoldersRequest.parent",
+ "bytebase.v1.SearchSavedQueryFoldersResponse.folders",
+ "bytebase.v1.SemanticTypeSetting.SemanticType.description",
+ "bytebase.v1.SemanticTypeSetting.SemanticType.icon",
+ "bytebase.v1.SemanticTypeSetting.SemanticType.id",
+ "bytebase.v1.SemanticTypeSetting.SemanticType.title",
+ "bytebase.v1.SendEmailLoginCodeRequest.email",
+ "bytebase.v1.SendEmailLoginCodeRequest.workspace",
+ "bytebase.v1.SequenceMetadata.cache_size",
+ "bytebase.v1.SequenceMetadata.comment",
+ "bytebase.v1.SequenceMetadata.data_type",
+ "bytebase.v1.SequenceMetadata.increment",
+ "bytebase.v1.SequenceMetadata.last_value",
+ "bytebase.v1.SequenceMetadata.max_value",
+ "bytebase.v1.SequenceMetadata.min_value",
+ "bytebase.v1.SequenceMetadata.name",
+ "bytebase.v1.SequenceMetadata.owner_column",
+ "bytebase.v1.SequenceMetadata.owner_table",
+ "bytebase.v1.SequenceMetadata.start",
+ "bytebase.v1.ServiceAccount.email",
+ "bytebase.v1.ServiceAccount.name",
+ "bytebase.v1.ServiceAccount.title",
+ "bytebase.v1.SetIamPolicyRequest.etag",
+ "bytebase.v1.SetIamPolicyRequest.resource",
+ "bytebase.v1.SetSavedQueryPolicyRequest.resource",
+ "bytebase.v1.Setting.name",
+ "bytebase.v1.Sheet.name",
+ "bytebase.v1.SignupRequest.email",
+ "bytebase.v1.SignupRequest.title",
+ "bytebase.v1.SpatialIndexConfig.method",
+ "bytebase.v1.Stage.environment",
+ "bytebase.v1.Stage.id",
+ "bytebase.v1.Stage.name",
+ "bytebase.v1.StorageConfig.buffering",
+ "bytebase.v1.StorageConfig.data_compression",
+ "bytebase.v1.StorageConfig.sort_in_tempdb",
+ "bytebase.v1.StorageConfig.tablespace",
+ "bytebase.v1.StorageConfig.work_tablespace",
+ "bytebase.v1.StreamMetadata.comment",
+ "bytebase.v1.StreamMetadata.definition",
+ "bytebase.v1.StreamMetadata.name",
+ "bytebase.v1.StreamMetadata.owner",
+ "bytebase.v1.StreamMetadata.table_name",
+ "bytebase.v1.Subscription.etag",
+ "bytebase.v1.Subscription.org_name",
+ "bytebase.v1.SwitchWorkspaceRequest.workspace",
+ "bytebase.v1.SyncDatabaseRequest.name",
+ "bytebase.v1.SyncDatabases.databases",
+ "bytebase.v1.SyncInstanceRequest.name",
+ "bytebase.v1.SyncInstanceResponse.databases",
+ "bytebase.v1.TableCatalog.classification",
+ "bytebase.v1.TableCatalog.name",
+ "bytebase.v1.TableMetadata.charset",
+ "bytebase.v1.TableMetadata.collation",
+ "bytebase.v1.TableMetadata.comment",
+ "bytebase.v1.TableMetadata.create_options",
+ "bytebase.v1.TableMetadata.engine",
+ "bytebase.v1.TableMetadata.name",
+ "bytebase.v1.TableMetadata.owner",
+ "bytebase.v1.TableMetadata.primary_key_type",
+ "bytebase.v1.TableMetadata.sharding_info",
+ "bytebase.v1.TableMetadata.sorting_keys",
+ "bytebase.v1.TablePartitionMetadata.expression",
+ "bytebase.v1.TablePartitionMetadata.name",
+ "bytebase.v1.TablePartitionMetadata.use_default",
+ "bytebase.v1.TablePartitionMetadata.value",
+ "bytebase.v1.TagPolicy.tags",
+ "bytebase.v1.Task.DatabaseCreate.sheet",
+ "bytebase.v1.Task.DatabaseUpdate.release",
+ "bytebase.v1.Task.DatabaseUpdate.sheet",
+ "bytebase.v1.Task.name",
+ "bytebase.v1.Task.skipped_reason",
+ "bytebase.v1.Task.spec_id",
+ "bytebase.v1.Task.target",
+ "bytebase.v1.TaskMetadata.comment",
+ "bytebase.v1.TaskMetadata.condition",
+ "bytebase.v1.TaskMetadata.definition",
+ "bytebase.v1.TaskMetadata.id",
+ "bytebase.v1.TaskMetadata.name",
+ "bytebase.v1.TaskMetadata.owner",
+ "bytebase.v1.TaskMetadata.predecessors",
+ "bytebase.v1.TaskMetadata.schedule",
+ "bytebase.v1.TaskMetadata.warehouse",
+ "bytebase.v1.TaskRun.creator",
+ "bytebase.v1.TaskRun.detail",
+ "bytebase.v1.TaskRun.name",
+ "bytebase.v1.TaskRunLog.name",
+ "bytebase.v1.TaskRunLogEntry.CommandExecute.CommandResponse.error",
+ "bytebase.v1.TaskRunLogEntry.CommandExecute.statement",
+ "bytebase.v1.TaskRunLogEntry.ComputeDiff.error",
+ "bytebase.v1.TaskRunLogEntry.DatabaseSync.error",
+ "bytebase.v1.TaskRunLogEntry.GhostMigration.error",
+ "bytebase.v1.TaskRunLogEntry.PriorBackup.PriorBackupDetail.Item.Table.database",
+ "bytebase.v1.TaskRunLogEntry.PriorBackup.PriorBackupDetail.Item.Table.schema",
+ "bytebase.v1.TaskRunLogEntry.PriorBackup.PriorBackupDetail.Item.Table.table",
+ "bytebase.v1.TaskRunLogEntry.PriorBackup.error",
+ "bytebase.v1.TaskRunLogEntry.ReleaseFileExecute.file_path",
+ "bytebase.v1.TaskRunLogEntry.ReleaseFileExecute.version",
+ "bytebase.v1.TaskRunLogEntry.RetryInfo.error",
+ "bytebase.v1.TaskRunLogEntry.SchemaDump.error",
+ "bytebase.v1.TaskRunLogEntry.TransactionControl.error",
+ "bytebase.v1.TaskRunLogEntry.replica_id",
+ "bytebase.v1.TaskRunSession.Postgres.Session.application_name",
+ "bytebase.v1.TaskRunSession.Postgres.Session.blocked_by_pids",
+ "bytebase.v1.TaskRunSession.Postgres.Session.client_addr",
+ "bytebase.v1.TaskRunSession.Postgres.Session.client_port",
+ "bytebase.v1.TaskRunSession.Postgres.Session.datname",
+ "bytebase.v1.TaskRunSession.Postgres.Session.pid",
+ "bytebase.v1.TaskRunSession.Postgres.Session.query",
+ "bytebase.v1.TaskRunSession.Postgres.Session.state",
+ "bytebase.v1.TaskRunSession.Postgres.Session.usename",
+ "bytebase.v1.TaskRunSession.Postgres.Session.wait_event",
+ "bytebase.v1.TaskRunSession.Postgres.Session.wait_event_type",
+ "bytebase.v1.TaskRunSession.name",
+ "bytebase.v1.TessellationConfig.scheme",
+ "bytebase.v1.TestEmailSettingRequest.parent",
+ "bytebase.v1.TestEmailSettingRequest.to",
+ "bytebase.v1.TestEmailSettingResponse.error",
+ "bytebase.v1.TestIdentityProviderResponse.claims",
+ "bytebase.v1.TestIdentityProviderResponse.user_info",
+ "bytebase.v1.TestWebhookRequest.project",
+ "bytebase.v1.TestWebhookResponse.error",
+ "bytebase.v1.TriggerMetadata.body",
+ "bytebase.v1.TriggerMetadata.character_set_client",
+ "bytebase.v1.TriggerMetadata.collation_connection",
+ "bytebase.v1.TriggerMetadata.comment",
+ "bytebase.v1.TriggerMetadata.event",
+ "bytebase.v1.TriggerMetadata.name",
+ "bytebase.v1.TriggerMetadata.sql_mode",
+ "bytebase.v1.TriggerMetadata.timing",
+ "bytebase.v1.UndeleteInstanceRequest.name",
+ "bytebase.v1.UndeleteProjectRequest.name",
+ "bytebase.v1.UndeleteReleaseRequest.name",
+ "bytebase.v1.UndeleteServiceAccountRequest.name",
+ "bytebase.v1.UndeleteUserRequest.name",
+ "bytebase.v1.UndeleteWorkloadIdentityRequest.name",
+ "bytebase.v1.UpdateDataSourceRequest.name",
+ "bytebase.v1.UpdateEmailRequest.email",
+ "bytebase.v1.UpdateEmailRequest.name",
+ "bytebase.v1.UpdateIssueCommentRequest.parent",
+ "bytebase.v1.UpdatePurchaseRequest.etag",
+ "bytebase.v1.UpdateSavedQueryStarRequest.name",
+ "bytebase.v1.User.email",
+ "bytebase.v1.User.name",
+ "bytebase.v1.User.title",
+ "bytebase.v1.User.workspace",
+ "bytebase.v1.VCSUser.display_name",
+ "bytebase.v1.VCSUser.user_id",
+ "bytebase.v1.VCSUser.user_name",
+ "bytebase.v1.VerifyCheckoutSessionResponse.status",
+ "bytebase.v1.ViewMetadata.comment",
+ "bytebase.v1.ViewMetadata.definition",
+ "bytebase.v1.ViewMetadata.name",
+ "bytebase.v1.Webhook.name",
+ "bytebase.v1.Webhook.title",
+ "bytebase.v1.WorkloadIdentity.email",
+ "bytebase.v1.WorkloadIdentity.name",
+ "bytebase.v1.WorkloadIdentity.title",
+ "bytebase.v1.WorkloadIdentityConfig.allowed_audiences",
+ "bytebase.v1.WorkloadIdentityConfig.issuer_url",
+ "bytebase.v1.WorkloadIdentityConfig.subject_pattern",
+ "bytebase.v1.Workspace.logo",
+ "bytebase.v1.Workspace.name",
+ "bytebase.v1.Workspace.title",
+ "bytebase.v1.WorkspaceProfileSetting.domains",
+ "bytebase.v1.WorkspaceProfileSetting.external_url",
+ "bytebase.v1.WorkspaceProfileSetting.sql_editor_theme_id",
+ "google.api.expr.v1alpha1.Constant.bytes_value",
+ "google.api.expr.v1alpha1.Constant.string_value",
+ "google.api.expr.v1alpha1.Expr.Call.function",
+ "google.api.expr.v1alpha1.Expr.Comprehension.accu_var",
+ "google.api.expr.v1alpha1.Expr.Comprehension.iter_var",
+ "google.api.expr.v1alpha1.Expr.Comprehension.iter_var2",
+ "google.api.expr.v1alpha1.Expr.CreateStruct.Entry.field_key",
+ "google.api.expr.v1alpha1.Expr.CreateStruct.message_name",
+ "google.api.expr.v1alpha1.Expr.Ident.name",
+ "google.api.expr.v1alpha1.Expr.Select.field",
+ "google.protobuf.Any.type_url",
+ "google.protobuf.Any.value",
+ "google.protobuf.FieldMask.paths",
+ "google.rpc.Status.message",
+ "google.type.Expr.description",
+ "google.type.Expr.expression",
+ "google.type.Expr.location",
+ "google.type.Expr.title",
+}
+
+// TestAuditRowNeedsNoRedactionBeyondTheAnyPayloads pins the narrowing in
+// createAuditLog. The row is not walked as a whole; service_data and
+// status.details are redacted where they are assigned, and everything else is
+// left alone because it holds nothing annotated.
+//
+// That is a fact about today's protos, not a guarantee, so it is asserted
+// rather than assumed: annotate a field on RequestMetadata or MCPDelegation —
+// caller_supplied_user_agent is caller-controlled and the likeliest candidate —
+// and this fails, because nothing would redact it.
+func TestAuditRowNeedsNoRedactionBeyondTheAnyPayloads(t *testing.T) {
+ for _, message := range []proto.Message{&storepb.RequestMetadata{}, &storepb.MCPDelegation{}} {
+ descriptor := message.ProtoReflect().Descriptor()
+ require.Nil(t, planFor(descriptor),
+ "%s is assigned onto the audit row unredacted; annotating a field under it needs redaction wired up "+
+ "in createAuditLog first", descriptor.FullName())
+ }
+
+ // The row's own plan may cover the two Any fields and nothing else.
+ plan := planFor((&storepb.AuditLog{}).ProtoReflect().Descriptor())
+ require.NotNil(t, plan)
+ fields := (&storepb.AuditLog{}).ProtoReflect().Descriptor().Fields()
+ var covered []string
+ for number := range plan.fields {
+ covered = append(covered, string(fields.ByNumber(number).Name()))
+ }
+ slices.Sort(covered)
+ require.Equal(t, []string{"service_data", "status"}, covered,
+ "a field on the audit row started needing redaction; redact it where it is assigned in createAuditLog")
+}
+
+// TestLintAuditAnyFieldsAreRegistered is the descriptor half. It catches a new
+// google.protobuf.Any FIELD reaching an audit row — a field the walk can see,
+// even though the type packed inside it is chosen in Go and cannot be seen.
+// Without it a new Any field would be governed by nothing: the runtime drops
+// what it cannot place, so the payload would vanish silently.
+func TestLintAuditAnyFieldsAreRegistered(t *testing.T) {
+ found := auditReachableAnyFields(t)
+
+ var unregistered []string
+ for _, field := range found {
+ if _, ok := auditAnyRegistry[protoreflect.FullName(field)]; !ok {
+ unregistered = append(unregistered, field)
+ }
+ }
+ slices.Sort(unregistered)
+ require.Empty(t, unregistered,
+ "these carry a google.protobuf.Any onto an audit row and auditAnyRegistry does not name what may be "+
+ "packed in them, so the runtime would drop whatever they carry: register the permitted types")
+
+ var stale []protoreflect.FullName
+ for field := range auditAnyRegistry {
+ if !slices.Contains(found, string(field)) {
+ stale = append(stale, field)
+ }
+ }
+ slices.SortFunc(stale, func(a, b protoreflect.FullName) int { return strings.Compare(string(a), string(b)) })
+ require.Empty(t, stale, "these no longer reach an audit row; drop them so the registry keeps meaning something")
+}
+
+// auditReachableAnyFields is every google.protobuf.Any-typed field reachable
+// from a message that can reach an audit payload, including the audit row's own
+// directly assigned fields.
+func auditReachableAnyFields(t *testing.T) []string {
+ t.Helper()
+ seen := map[protoreflect.FullName]bool{}
+ var found []string
+ var walk func(protoreflect.MessageDescriptor)
+ walk = func(descriptor protoreflect.MessageDescriptor) {
+ if seen[descriptor.FullName()] {
+ return
+ }
+ seen[descriptor.FullName()] = true
+ fields := descriptor.Fields()
+ for i := range fields.Len() {
+ field := fields.Get(i)
+ // An annotated field never reaches the row, so neither does an Any
+ // at or under it. Skipping matches what the redactor does and what
+ // the inventory walk counts as reachable.
+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {
+ continue
+ }
+ sub := submessageOf(field)
+ if sub == nil {
+ continue
+ }
+ if sub.FullName() == anyFullName {
+ found = append(found, string(field.FullName()))
+ continue
+ }
+ walk(sub)
+ }
+ }
+
+ protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
+ if fd.Package() != "bytebase.v1" {
+ return true
+ }
+ services := fd.Services()
+ for i := range services.Len() {
+ methods := services.Get(i).Methods()
+ for j := range methods.Len() {
+ walk(methods.Get(j).Input())
+ walk(methods.Get(j).Output())
+ }
+ }
+ return true
+ })
+ walk((&storepb.AuditLog{}).ProtoReflect().Descriptor())
+
+ slices.Sort(found)
+ return slices.Compact(found)
+}
diff --git a/backend/api/v1/audit_redact_test.go b/backend/api/v1/audit_redact_test.go
new file mode 100644
index 0000000000..39c5399f1d
--- /dev/null
+++ b/backend/api/v1/audit_redact_test.go
@@ -0,0 +1,1076 @@
+package v1
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "net/http"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/reflect/protodesc"
+ "google.golang.org/protobuf/reflect/protoreflect"
+ "google.golang.org/protobuf/reflect/protoregistry"
+ "google.golang.org/protobuf/types/descriptorpb"
+ "google.golang.org/protobuf/types/dynamicpb"
+ "google.golang.org/protobuf/types/known/anypb"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/component/config"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/generated-go/v1/v1connect"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+// Audited RPCs write their request and response payloads to audit_log, and to
+// stdout when RuntimeEnableAuditLogStdout is set. Anything with
+// bb.auditLogs.search/export, or read access to the log pipeline, can read
+// them. Redaction therefore owes the row two things, and the first group below
+// is exactly those two:
+//
+// - the secret leaves. Proven exhaustively rather than by example:
+// TestAuditRedactionCoversEveryAnnotatedField seeds a sentinel into all 567
+// annotated field paths across 321 request and response types and asserts
+// none survives. Hand-written per-credential tables used to sit here and
+// were removed — every field they covered is in that sweep, on the same
+// generated Go types, and a table is a list someone has to remember to
+// extend.
+//
+// - the substance stays. TestAuditRowKeepsItsSubstance is the only guard for
+// this, and has to be written by hand: the sweep asserts absence, so a
+// redactor that blanked the entire row passes it. Over-redaction is the
+// failure mode nothing else in this package can see.
+//
+// The groups after that are the walker's own invariants — it terminates on the
+// cyclic components, reaches an annotation at any depth, rebuilds rather than
+// aliases, and never touches the caller's message — then the two payloads a
+// descriptor walk cannot see into on its own, a packed Any and the streaming
+// path. Fixtures and machinery are last, so the file reads as claims first.
+
+const secretSentinel = "s3cr3t-sentinel-value"
+
+var encodedSecretSentinel = base64.StdEncoding.EncodeToString([]byte(secretSentinel))
+
+// ---- The contract: the secret leaves, the substance stays ----------------
+
+// TestAuditRedactionCoversEveryAnnotatedField is the sweep. A redactor that
+// stops covering a field fails here rather than silently starting to log it.
+func TestAuditRedactionCoversEveryAnnotatedField(t *testing.T) {
+ population := auditPopulation(t)
+ names := make([]protoreflect.FullName, 0, len(population))
+ for name := range population {
+ names = append(names, name)
+ }
+ slices.SortFunc(names, func(a, b protoreflect.FullName) int { return strings.Compare(string(a), string(b)) })
+
+ covered := 0
+ for _, name := range names {
+ descriptor := messageDescriptorByName(t, name)
+ targets := annotatedTargets(descriptor, string(descriptor.Name()), nil, map[protoreflect.FullName]bool{})
+ if len(targets) == 0 {
+ continue
+ }
+ t.Run(string(name), func(t *testing.T) {
+ for _, target := range targets {
+ covered++
+ filled := dynamicMessage(t, descriptor)
+ require.True(t, seedTarget(filled.ProtoReflect(), target.chain),
+ "%s could not be populated, so every assertion below would pass vacuously", target.path)
+
+ redacted := redactForAudit(filled)
+ assertNoAnnotatedContent(t, redacted.ProtoReflect(), string(descriptor.Name()))
+
+ got, err := protojson.Marshal(redacted)
+ require.NoError(t, err, target.path)
+ require.NotContains(t, string(got), secretSentinel,
+ "%s reached the audit payload of %v", target.path, population[name])
+ require.NotContains(t, string(got), encodedSecretSentinel,
+ "%s reached the audit payload of %v, base64-encoded", target.path, population[name])
+ }
+ })
+ }
+ require.NotZero(t, covered, "the sweep found nothing to assert on")
+ t.Logf("swept %d annotated field paths across %d request/response types", covered, len(names))
+}
+
+// The over-redaction guard, and the half TestAuditRedactionCoversEveryAnnotatedField
+// structurally cannot be: that sweep proves annotated fields are gone, and has
+// no notion of what the row must still say. Every value below carries a
+// sentinel, so the loop asserts both halves — the secret leaves, the substance
+// stays.
+//
+// The rows naming a role, a user and a result count are the leaks this design
+// was written for. The sweep already covers their credentials by field path;
+// what only lives here is that redacting them did not take the surrounding
+// record with it.
+func TestAuditRowKeepsItsSubstance(t *testing.T) {
+ batchParent := "projects/project-a"
+ otpCode := secretSentinel
+ mfaTempToken := secretSentinel
+ queryResult := func() *v1pb.QueryResult {
+ return &v1pb.QueryResult{
+ RowsCount: 7,
+ ColumnNames: []string{"card_number"},
+ Rows: []*v1pb.QueryRow{{Values: []*v1pb.RowValue{{Kind: &v1pb.RowValue_StringValue{StringValue: secretSentinel}}}}},
+ }
+ }
+ for _, tt := range []struct {
+ name string
+ value any
+ response bool
+ want []string
+ }{
+ {
+ name: "webhook request retains identifiers",
+ value: &v1pb.AddWebhookRequest{Project: "projects/project-a", Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel}},
+ want: []string{"projects/project-a", "projects/project-a/webhooks/webhook-a", "Webhook A"},
+ },
+ {
+ // The row has to stay useful: which instances were retargeted, and
+ // which kind of IAM credential was supplied.
+ name: "batch instance update retains targets and credential type",
+ value: &v1pb.BatchUpdateInstancesRequest{
+ Parent: &batchParent,
+ Requests: []*v1pb.UpdateInstanceRequest{{Instance: &v1pb.Instance{
+ Name: "instances/instance-a",
+ DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},
+ }}},
+ },
+ want: []string{"projects/project-a", "instances/instance-a", "db.example.com", "gcpCredential"},
+ },
+ {
+ name: "azure iam credential retains the principal it names",
+ value: &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{
+ IamExtension: &v1pb.DataSource_AzureCredential_{AzureCredential: &v1pb.DataSource_AzureCredential{
+ TenantId: "tenant-a", ClientId: "client-a", ClientSecret: secretSentinel,
+ }},
+ }},
+ want: []string{"azureCredential", "tenant-a", "client-a"},
+ },
+ {
+ name: "aws iam credential records that one was supplied",
+ value: &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{
+ IamExtension: &v1pb.DataSource_AwsCredential{AwsCredential: &v1pb.DataSource_AWSCredential{
+ AccessKeyId: secretSentinel, SecretAccessKey: secretSentinel,
+ }},
+ }},
+ want: []string{"awsCredential"},
+ },
+ {
+ name: "audit export retains page token",
+ value: &v1pb.ExportAuditLogsResponse{Content: []byte(secretSentinel), NextPageToken: "next-page-token"},
+ response: true,
+ want: []string{"next-page-token"},
+ },
+ {
+ // INPUT_ONLY on the password, OUTPUT_ONLY only on its container, so
+ // clients are meant to send it. Which role was written is the point
+ // of the row; redactInstance used to pass the password with it.
+ name: "instance role update retains the role it names",
+ value: &v1pb.UpdateInstanceRequest{Instance: instanceWithRolePassword()},
+ want: []string{"instances/instance-a", "role-a"},
+ },
+ {
+ // A failed validation neither consumes the code nor suppresses the
+ // row, so this used to write a still-live OTP.
+ name: "user update retains who was updated",
+ value: &v1pb.UpdateUserRequest{User: &v1pb.User{Name: "users/[email protected]"}, OtpCode: &otpCode},
+ want: []string{"users/[email protected]"},
+ },
+ {
+ // The two hand-written redactors disagreed here: Query kept
+ // rows_count and AdminExecute dropped it. One annotation on `rows`
+ // settles it, and the count is what makes the row worth reading.
+ name: "admin execute retains the shape of the result",
+ value: &v1pb.AdminExecuteResponse{Results: []*v1pb.QueryResult{queryResult()}},
+ response: true,
+ want: []string{`"rowsCount":"7"`, "card_number"},
+ },
+ {
+ name: "query retains the shape of the result",
+ value: &v1pb.QueryResponse{Results: []*v1pb.QueryResult{queryResult()}},
+ response: true,
+ want: []string{`"rowsCount":"7"`, "card_number"},
+ },
+ {
+ // Who tried to log in is the whole point of a login row, and the
+ // password, OTP and MFA token beside it are all annotated.
+ name: "login retains who attempted it",
+ value: &v1pb.LoginRequest{
+ Email: "[email protected]",
+ Password: secretSentinel,
+ OtpCode: &otpCode,
+ MfaTempToken: &mfaTempToken,
+ },
+ want: []string{"[email protected]"},
+ },
+ {
+ name: "login response retains the user it authenticated",
+ value: &v1pb.LoginResponse{Token: secretSentinel, User: &v1pb.User{Name: "users/[email protected]"}},
+ response: true,
+ want: []string{"users/[email protected]"},
+ },
+ {
+ // The Signup password leak Codex flagged on #20024, which surfaced
+ // only once SetAuditWorkspaceID re-enabled this audit path.
+ name: "signup retains the account being created",
+ value: &v1pb.SignupRequest{Email: "[email protected]", Password: secretSentinel, Title: "bob"},
+ want: []string{"[email protected]", "bob"},
+ },
+ {
+ // The other half of #20024. An external OIDC token can be replayed
+ // against the issuing IdP, so it must not reach a row that
+ // bb.auditLogs.search can read — but which workload presented it is
+ // exactly what the row is for. ExchangeTokenResponse.access_token
+ // leaves nothing behind to assert on, so the sweep covers it alone.
+ name: "token exchange retains the workload it correlates",
+ value: &v1pb.ExchangeTokenRequest{Token: secretSentinel, Email: "[email protected]"},
+ want: []string{"[email protected]"},
+ },
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ got := marshalAuditPayload(tt.value)
+ for _, want := range tt.want {
+ require.Contains(t, got, want)
+ }
+ require.NotContains(t, got, secretSentinel, "a credential reached the audit log")
+ require.NotContains(t, got, encodedSecretSentinel, "an encoded credential reached the audit log")
+ })
+ }
+}
+
+// ---- The walker's invariants ---------------------------------------------
+
+// Redaction must not mutate the caller's message: it runs on the live request
+// and response objects, so masking in place would corrupt what the handler
+// returns to the client or what a later interceptor reads.
+//
+// Asserted as "the message is unchanged afterwards" rather than by reading back
+// the one field a case cares about. A per-field check passes while redaction
+// quietly clears the neighbour nobody thought to read, and the earlier version
+// of this spent forty lines of type switch picking that field out.
+//
+// The last three rows are the repeated-message half, and the leak this design
+// opens with: Instance.roles[].password and instance_resource.data_sources[]
+// both cross a repeated message field, where copy-and-share is easiest to get
+// wrong.
+func TestAuditRedactionDoesNotMutateTheCallersMessage(t *testing.T) {
+ for name, message := range map[string]proto.Message{
+ "service account response": &v1pb.ServiceAccount{ServiceKey: secretSentinel},
+ "project webhook response": &v1pb.Project{Webhooks: []*v1pb.Webhook{{Url: secretSentinel}}},
+ "release response": &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}},
+ "saved query response": &v1pb.SavedQuery{Content: []byte(secretSentinel)},
+ "audit export response": &v1pb.ExportAuditLogsResponse{Content: []byte(secretSentinel), NextPageToken: "next-page-token"},
+ "ai setting request": &v1pb.UpdateSettingRequest{Setting: &v1pb.Setting{Value: &v1pb.SettingValue{Value: &v1pb.SettingValue_Ai{Ai: &v1pb.AISetting{ApiKey: secretSentinel}}}}},
+ "add data source request": &v1pb.AddDataSourceRequest{DataSource: dataSourceWithEverySecret()},
+ "update data source request": &v1pb.UpdateDataSourceRequest{DataSource: dataSourceWithEverySecret()},
+ "remove data source request": &v1pb.RemoveDataSourceRequest{DataSource: dataSourceWithEverySecret()},
+ "create instance request": &v1pb.CreateInstanceRequest{Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}}},
+ "update instance request": &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}}},
+ "create release request": &v1pb.CreateReleaseRequest{Release: &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}},
+ "update release request": &v1pb.UpdateReleaseRequest{Release: &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}},
+ "create saved query request": &v1pb.CreateSavedQueryRequest{SavedQuery: &v1pb.SavedQuery{Content: []byte(secretSentinel)}},
+ "reset password request": &v1pb.ResetPasswordRequest{Email: "[email protected]", Code: secretSentinel, NewPassword: secretSentinel},
+ "instance roles": &v1pb.Instance{Roles: []*v1pb.InstanceRole{{Name: "r", Password: proto.String(secretSentinel)}}},
+ "database instance resource": &v1pb.Database{InstanceResource: &v1pb.InstanceResource{DataSources: []*v1pb.DataSource{{Id: "admin", Password: secretSentinel}}}},
+ "batch update instances": &v1pb.BatchUpdateInstancesRequest{Requests: []*v1pb.UpdateInstanceRequest{{
+ Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}},
+ }}},
+ } {
+ t.Run(name, func(t *testing.T) {
+ before := proto.Clone(message)
+ got := marshalAuditPayload(message)
+
+ require.NotContains(t, got, secretSentinel, "a credential reached the audit log")
+ require.NotContains(t, got, encodedSecretSentinel, "an encoded credential reached the audit log")
+ require.True(t, proto.Equal(before, message),
+ "redaction mutated the caller's message\nbefore: %v\nafter: %v", before, message)
+ })
+ }
+}
+
+// TestAuditPlanHandlesCyclicDescriptors pins the fixed-point builder against
+// the four cyclic components in the descriptor graph. An unguarded descend runs
+// until the stack does, inside the interceptor, taking the RPC with it — and a
+// guard keyed on "the field's type equals the parent's" catches only the
+// self-recursive one.
+//
+// Termination is the easy half. The harder one is that breaking a cycle by
+// provisionally answering "no plan" leaves the resulting plan with no descend
+// arm for the fields that resolved that way, so an annotation inside the cycle
+// would be applied at depth 0 and nowhere else — with the coverage sweep still
+// passing, because the top-level copy is clean.
+func TestAuditPlanHandlesCyclicDescriptors(t *testing.T) {
+ for name, root := range map[string]proto.Message{
+ // TablePartitionMetadata.subpartitions, the only self-recursive field,
+ // and the one component not on an audited path.
+ "self-recursive partitions": &v1pb.DiffMetadataRequest{},
+ // ObjectSchema <-> StructKind <-> ArrayKind, whose cycle runs through a
+ // map. Reached from UpdateDatabaseCatalog, which is audited.
+ "object schema through a map": &v1pb.UpdateDatabaseCatalogRequest{},
+ // google.protobuf.Value <-> Struct <-> ListValue, reached from
+ // QueryResponse.results[].rows[].values[].value_value on every audited
+ // Query and AdminExecute.
+ "protobuf Value": &v1pb.QueryResponse{},
+ // google.api.expr Expr and its nested types, reached from
+ // SetIamPolicyRequest.policy.bindings[].parsed_expr.
+ "CEL expression": &v1pb.SetIamPolicyRequest{},
+ } {
+ t.Run(name, func(t *testing.T) {
+ done := make(chan *redactPlan, 1)
+ go func() { done <- planFor(root.ProtoReflect().Descriptor()) }()
+ select {
+ case plan := <-done:
+ // The plan may legitimately be nil; what matters is that
+ // building it terminated rather than recursing forever.
+ _ = plan
+ case <-time.After(30 * time.Second):
+ t.Fatal("plan construction did not terminate on a cyclic descriptor")
+ }
+ // A second call has to answer from the cache, not rebuild.
+ require.NotPanics(t, func() { planFor(root.ProtoReflect().Descriptor()) })
+ })
+ }
+}
+
+// TestAuditRedactionFollowsACycleToAnyDepth is the half termination alone does
+// not give: that an annotation INSIDE a cycle is still reached. Breaking a
+// cycle by provisionally answering "no plan" leaves the plan with no descend
+// arm for the fields that resolved that way, so the annotation is applied at
+// depth 0 and nowhere else — with the coverage sweep still green, because the
+// top-level copy is clean.
+//
+// An earlier build did something subtler: it cached plan VALUES in the descend
+// arm, freezing a finite chain, so everything below its end was shared
+// unredacted. A 10-deep chain leaked 8 of its 11 secrets while a one-level
+// fixture stayed green — which is why the depth here is far past three rather
+// than the single hop that would read as sufficient. Descend arms hold a
+// DESCRIPTOR now and resolve their plan at each depth.
+//
+// The cycle is built rather than found in the surface, because no annotation
+// sits inside one today — exactly why either bug would go unnoticed until one
+// did.
+func TestAuditRedactionFollowsACycleToAnyDepth(t *testing.T) {
+ descriptor := cyclicSensitiveDescriptor(t)
+ secret := descriptor.Fields().ByName("secret")
+ child := descriptor.Fields().ByName("child")
+
+ plan := planFor(descriptor)
+ require.NotNil(t, plan, "a message with an annotated field needs a plan")
+ require.Equal(t, actionDescend, plan.fields[child.Number()].kind,
+ "the recursive field has to carry a descend arm, or the annotation is applied at depth 0 and nowhere else")
+
+ // Far deeper than any chain a value-caching builder could freeze, which was
+ // three links.
+ const depth = 12
+ root := dynamicpb.NewMessage(descriptor)
+ level := root
+ for range depth {
+ level.Set(secret, protoreflect.ValueOfString(secretSentinel))
+ next := dynamicpb.NewMessage(descriptor)
+ level.Set(child, protoreflect.ValueOfMessage(next))
+ level = next
+ }
+ level.Set(secret, protoreflect.ValueOfString(secretSentinel))
+
+ got, err := protojson.Marshal(redactForAudit(proto.Message(root)))
+ require.NoError(t, err)
+ require.NotContains(t, string(got), secretSentinel,
+ "an annotated field deep inside a cycle survived redaction")
+ require.Equal(t, depth, strings.Count(string(got), `"child"`),
+ "the fixture has to actually reach that depth for the assertion above to mean anything")
+ require.Equal(t, secretSentinel, root.Get(secret).String(), "redaction mutated the caller's message")
+}
+
+// TestAuditRedactionRebuildsMapsRatherThanSharing pins the map rule. Setting a
+// map field from the value Range yields shares the Go map itself and its
+// message values, so clearing a field in an entry writes through to the
+// caller's live message — measured, the source loses the field.
+//
+// The fixture is built rather than found: no map in the v1 surface has a value
+// type with a plan today, so this violation of "redaction does not mutate the
+// caller's message" would otherwise go uncaught until one appeared.
+func TestAuditRedactionRebuildsMapsRatherThanSharing(t *testing.T) {
+ descriptor := mapOfSensitiveDescriptor(t)
+ entryValue := dynamicpb.NewMessage(descriptor.Fields().ByName("entries").MapValue().Message())
+ entryValue.Set(entryValue.Descriptor().Fields().ByName("secret"), protoreflect.ValueOfString(secretSentinel))
+
+ message := dynamicpb.NewMessage(descriptor)
+ entries := message.Mutable(descriptor.Fields().ByName("entries")).Map()
+ entries.Set(protoreflect.ValueOfString("k").MapKey(), protoreflect.ValueOfMessage(entryValue))
+
+ plan := planFor(descriptor)
+ require.NotNil(t, plan)
+ redacted := redactMessage(message, plan)
+
+ got := redacted.Get(descriptor.Fields().ByName("entries")).Map().
+ Get(protoreflect.ValueOfString("k").MapKey()).Message()
+ require.Empty(t, got.Get(got.Descriptor().Fields().ByName("secret")).String(),
+ "a credential inside a map value survived redaction")
+ require.Equal(t, secretSentinel, entryValue.Get(entryValue.Descriptor().Fields().ByName("secret")).String(),
+ "redaction wrote through the shared map and cleared the caller's own message")
+}
+
+// TestAuditRedactionKeepsSensitiveOneofArmsPresent pins the blank-versus-clear
+// rule. Clearing a scalar arm unsets the oneof and erases which arm was
+// supplied: DataSourceExternalSecret.token is a string arm whose sibling
+// app_role is a message that would survive as {}, so clearing would make token
+// auth indistinguishable from unconfigured while AppRole stayed legible.
+func TestAuditRedactionKeepsSensitiveOneofArmsPresent(t *testing.T) {
+ request := &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{
+ ExternalSecret: &v1pb.DataSourceExternalSecret{
+ Url: "https://vault.example.com",
+ AuthOption: &v1pb.DataSourceExternalSecret_Token{Token: secretSentinel},
+ },
+ }}
+ got := marshalAuditPayload(request)
+ require.NotContains(t, got, secretSentinel)
+ require.Contains(t, got, `"token":""`,
+ "the arm must stay present with no value, so the row still records that token auth was configured")
+ require.Contains(t, got, "vault.example.com", "the secret store the caller named is the point of the row")
+
+ // The message arm keeps its own shape, minus the credentials inside it.
+ appRole := &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{
+ ExternalSecret: &v1pb.DataSourceExternalSecret{
+ AuthOption: &v1pb.DataSourceExternalSecret_AppRole{
+ AppRole: &v1pb.DataSourceExternalSecret_AppRoleAuthOption{
+ RoleId: secretSentinel, SecretId: secretSentinel, MountPath: "approle",
+ },
+ },
+ },
+ }}
+ got = marshalAuditPayload(appRole)
+ require.NotContains(t, got, secretSentinel)
+ require.Contains(t, got, "approle", "which auth method was configured stays readable")
+}
+
+// TestAuditRedactionSharesUnannotatedSubtrees pins the performance property the
+// whole design rests on: a subtree holding nothing annotated is shared by
+// pointer, never copied. A 5 MB sheet inside an audited batch used to be cloned
+// only to have its content nulled.
+func TestAuditRedactionSharesUnannotatedSubtrees(t *testing.T) {
+ labels := map[string]string{"env": "prod"}
+ instance := &v1pb.Instance{
+ Name: "instances/instance-a",
+ Labels: labels,
+ Roles: []*v1pb.InstanceRole{{Name: "instances/instance-a/roles/r", Password: proto.String(secretSentinel)}},
+ DataSources: []*v1pb.DataSource{{Id: "admin", Password: secretSentinel}},
+ }
+ redacted := redactForAudit(instance)
+ require.NotSame(t, instance, redacted, "a message with a plan is copied, not returned as itself")
+ require.Equal(t, labels, redacted.GetLabels(), "an unannotated map is shared, not rebuilt")
+
+ // A message with nothing annotated anywhere beneath it is returned as
+ // itself, which is what makes the sheet case free.
+ sheet := &v1pb.CreateSheetRequest{Parent: "projects/project-a", Sheet: &v1pb.Sheet{Content: []byte(secretSentinel)}}
+ require.NotSame(t, sheet, redactForAudit(sheet), "the sheet content is annotated, so the request is copied")
+ plain := &v1pb.GetSheetRequest{Name: "projects/project-a/sheets/sheet-a"}
+ require.Same(t, plain, redactForAudit(plain), "nothing under this message is annotated, so it is not copied")
+}
+
+// A rejected request is audited too, so redaction runs on messages the handler
+// was about to refuse. A hand-written redactor used to read through the nil
+// data source in one of these and panic the interceptor; the descriptor walk
+// visits only populated fields, so there is nothing to read through — these
+// pin that.
+func TestAuditRedactionSurvivesAnIncompleteRequest(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ request any
+ }{
+ {"add data source without one", &v1pb.AddDataSourceRequest{Name: "instances/instance-a"}},
+ {"update data source without one", &v1pb.UpdateDataSourceRequest{Name: "instances/instance-a"}},
+ {"remove data source without one", &v1pb.RemoveDataSourceRequest{Name: "instances/instance-a"}},
+ {"create instance without one", &v1pb.CreateInstanceRequest{InstanceId: "instance-a"}},
+ {"update instance without one", &v1pb.UpdateInstanceRequest{}},
+ {"batch update instances without any", &v1pb.BatchUpdateInstancesRequest{
+ Requests: []*v1pb.UpdateInstanceRequest{{}},
+ }},
+ {"instance carrying an empty data source", &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{
+ DataSources: []*v1pb.DataSource{{}},
+ }}},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ got := marshalAuditPayload(tt.request)
+ require.NotEmpty(t, got, "a rejected request still gets a row")
+ })
+ }
+}
+
+// ---- The two payloads a descriptor walk cannot see on its own ------------
+
+// TestAuditRedactsPackedAny pins the Any half. The descriptor walk sees only
+// type_url and value, so a credential inside a packed message is invisible to
+// it: the packed type is resolved and redacted at runtime, against the
+// registry, or the Any is dropped.
+func TestAuditRedactsPackedAny(t *testing.T) {
+ setting, err := anypb.New(&v1pb.Setting{Value: &v1pb.SettingValue{Value: &v1pb.SettingValue_Ai{
+ Ai: &v1pb.AISetting{Endpoint: "https://ai.example.com", ApiKey: secretSentinel},
+ }}})
+ require.NoError(t, err)
+
+ row := &storepb.AuditLog{
+ Method: "/bytebase.v1.SettingService/UpdateSetting",
+ ServiceData: redactAuditServiceData(setting),
+ }
+ got, err := protojson.Marshal(row)
+ require.NoError(t, err)
+ require.NotContains(t, string(got), secretSentinel, "a credential inside the service_data before-image was recorded")
+ require.Contains(t, string(got), "ai.example.com", "the rest of the before-image is the point of the row")
+ require.Contains(t, string(setting.GetValue()), secretSentinel, "redaction mutated the caller's Any")
+
+ t.Run("an unregistered packed type is dropped, not logged", func(t *testing.T) {
+ // protojson.Marshal fails the ENTIRE row on an unresolvable Any, so
+ // dropping is what keeps the record rather than losing it.
+ stranger, err := anypb.New(&v1pb.Sheet{Content: []byte(secretSentinel)})
+ require.NoError(t, err)
+ row := &storepb.AuditLog{
+ Method: "/bytebase.v1.SheetService/CreateSheet",
+ ServiceData: redactAuditServiceData(stranger),
+ }
+ require.Nil(t, row.GetServiceData())
+ got, err := protojson.Marshal(row)
+ require.NoError(t, err)
+ require.NotContains(t, string(got), encodedSecretSentinel)
+ })
+
+ t.Run("a registered type with nothing to redact keeps its type_url verbatim", func(t *testing.T) {
+ // connect's ErrorDetail.Type() returns a bare full name, so details are
+ // stored without the type.googleapis.com/ prefix. A round-trip through
+ // anypb.New would rewrite it and silently change what SearchAuditLogs
+ // emits as @type.
+ detail, err := connect.NewErrorDetail(&v1pb.PermissionDeniedDetail{Method: "/bytebase.v1.UserService/GetUser"})
+ require.NoError(t, err)
+ connectErr := connect.NewError(connect.CodePermissionDenied, errors.New("denied"))
+ connectErr.AddDetail(detail)
+
+ row := &storepb.AuditLog{Status: redactAuditStatus(convertErrToStatus(connectErr))}
+ require.Len(t, row.GetStatus().GetDetails(), 1)
+ require.Equal(t, "bytebase.v1.PermissionDeniedDetail", row.GetStatus().GetDetails()[0].GetTypeUrl())
+ })
+}
+
+// TestStreamingAuditRedactsRows is the end-to-end assertion the streaming path
+// needs. AdminExecute is the only streaming RPC, it is audited, and its
+// response carries every row of an admin-mode query — and Send builds its own
+// auditEntry and calls createAuditLog directly, so a redaction walk that lived
+// in WrapUnary would silently skip it. Everything else about streaming
+// persistence is exercised through the createAuditLogFunc stub, which bypasses
+// the real path; this one writes a real row and reads it back.
+func TestStreamingAuditRedactsRows(t *testing.T) {
+ st := newAuditLiveStore(t)
+ t.Cleanup(func() { require.NoError(t, st.Close()) })
+ interceptor := NewAuditInterceptor(st, "test-secret", &config.Profile{})
+
+ handler := interceptor.WrapStreamingHandler(func(_ context.Context, conn connect.StreamingHandlerConn) error {
+ if err := conn.Receive(&v1pb.AdminExecuteRequest{
+ Name: "instances/instance-a/databases/db",
+ Statement: "SELECT card_number FROM payments",
+ }); err != nil {
+ return err
+ }
+ return conn.Send(&v1pb.AdminExecuteResponse{Results: []*v1pb.QueryResult{{
+ ColumnNames: []string{"card_number"},
+ RowsCount: 1,
+ Statement: "SELECT card_number FROM payments",
+ Rows: []*v1pb.QueryRow{{Values: []*v1pb.RowValue{
+ {Kind: &v1pb.RowValue_StringValue{StringValue: secretSentinel}},
+ }}},
+ }}})
+ })
+
+ ctx := context.WithValue(context.Background(), common.AuthContextKey, &common.AuthContext{
+ Audit: true,
+ Resources: []*common.Resource{{Type: common.ResourceTypeWorkspace, ID: auditTestWorkspace}},
+ })
+ require.NoError(t, handler(ctx, &auditStreamingConn{}))
+
+ rows, err := st.SearchAuditLogs(ctx, &store.AuditLogFind{})
+ require.NoError(t, err)
+ require.Len(t, rows, 1)
+ require.NotContains(t, rows[0].Payload.Response, secretSentinel, "an admin-mode result row reached the audit log")
+ require.Contains(t, rows[0].Payload.Response, "card_number", "the column names and the statement are the point of the row")
+ require.Contains(t, rows[0].Payload.Response, `"rowsCount":"1"`,
+ "how many rows the query returned survives; only the rows themselves are dropped")
+}
+
+// ---- Guards on the fixtures the sweep builds for itself ------------------
+
+// TestNoAnnotationOutsideBytebaseProtos is what makes the fill's well-known-type
+// skip safe. The skip exists because google.protobuf.Value rejects an empty
+// instance at marshal time; it would be a hole if an annotation could ever sit
+// inside one.
+func TestNoAnnotationOutsideBytebaseProtos(t *testing.T) {
+ var annotated []string
+ protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
+ if strings.HasPrefix(string(fd.Package()), "bytebase.") {
+ return true
+ }
+ var walk func(protoreflect.MessageDescriptors)
+ walk = func(messages protoreflect.MessageDescriptors) {
+ for i := range messages.Len() {
+ md := messages.Get(i)
+ fields := md.Fields()
+ for j := range fields.Len() {
+ if field := fields.Get(j); auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {
+ annotated = append(annotated, string(field.FullName()))
+ }
+ }
+ walk(md.Messages())
+ }
+ }
+ walk(fd.Messages())
+ return true
+ })
+ require.Empty(t, annotated,
+ "the coverage sweep skips non-bytebase messages when building fixtures; an annotation inside one would go untested")
+}
+
+// ---- Fixtures and machinery ----------------------------------------------
+
+func instanceWithRolePassword() *v1pb.Instance {
+ password := secretSentinel
+ return &v1pb.Instance{
+ Name: "instances/instance-a",
+ Roles: []*v1pb.InstanceRole{{Name: "instances/instance-a/roles/role-a", RoleName: "role-a", Password: &password}},
+ }
+}
+
+// dataSourceWithEverySecret is the readable fixture: every credential a data
+// source can carry, on one message, so a case reads as the request a caller
+// sends. It is a hand-kept list and cannot see a field added later —
+// TestAuditRedactionCoversEveryAnnotatedField is the net that can.
+func dataSourceWithEverySecret() *v1pb.DataSource {
+ return &v1pb.DataSource{
+ Id: "admin-ds",
+ Host: "db.example.com",
+ Username: "admin",
+ Password: secretSentinel,
+ SslCa: secretSentinel,
+ SslCaPath: secretSentinel,
+ SslCert: secretSentinel,
+ SslCertPath: secretSentinel,
+ SslKey: secretSentinel,
+ SslKeyPath: secretSentinel,
+ SshPassword: secretSentinel,
+ SshPrivateKey: secretSentinel,
+ AuthenticationPrivateKey: secretSentinel,
+ AuthenticationPrivateKeyPassphrase: secretSentinel,
+ MasterPassword: secretSentinel,
+ ExternalSecret: &v1pb.DataSourceExternalSecret{
+ AuthOption: &v1pb.DataSourceExternalSecret_Token{Token: secretSentinel},
+ },
+ SaslConfig: &v1pb.SASLConfig{Mechanism: &v1pb.SASLConfig_KrbConfig{
+ KrbConfig: &v1pb.KerberosConfig{Keytab: []byte(secretSentinel)},
+ }},
+ IamExtension: &v1pb.DataSource_GcpCredential{
+ GcpCredential: &v1pb.DataSource_GCPCredential{Content: secretSentinel},
+ },
+ }
+}
+
+// auditStreamingConn is a minimal AdminExecute stream.
+type auditStreamingConn struct {
+ connect.StreamingHandlerConn
+}
+
+func (*auditStreamingConn) Spec() connect.Spec {
+ return connect.Spec{Procedure: v1connect.SQLServiceAdminExecuteProcedure}
+}
+func (*auditStreamingConn) Peer() connect.Peer { return connect.Peer{} }
+func (*auditStreamingConn) RequestHeader() http.Header { return http.Header{} }
+func (*auditStreamingConn) Receive(any) error { return nil }
+func (*auditStreamingConn) Send(any) error { return nil }
+
+// cyclicSensitiveDescriptor builds `message Node { string secret = 1
+// [SENSITIVE]; Node child = 2; }` — a self-recursive message with an
+// annotation inside the cycle.
+func cyclicSensitiveDescriptor(t *testing.T) protoreflect.MessageDescriptor {
+ t.Helper()
+ return buildTestMessage(t, "audit_cycle.proto", "audittest.cycle", &descriptorpb.DescriptorProto{
+ Name: proto.String("Node"),
+ Field: []*descriptorpb.FieldDescriptorProto{
+ {
+ Name: proto.String("secret"),
+ Number: proto.Int32(1),
+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),
+ Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(),
+ Options: sensitiveFieldOptions(),
+ },
+ {
+ Name: proto.String("child"),
+ Number: proto.Int32(2),
+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),
+ Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(),
+ TypeName: proto.String(".audittest.cycle.Node"),
+ },
+ },
+ })
+}
+
+// mapOfSensitiveDescriptor builds `message Holder { map<string, Entry> entries
+// = 1; }` where Entry carries a SENSITIVE field.
+func mapOfSensitiveDescriptor(t *testing.T) protoreflect.MessageDescriptor {
+ t.Helper()
+ return buildTestMessage(t, "audit_map.proto", "audittest.maps", &descriptorpb.DescriptorProto{
+ Name: proto.String("Holder"),
+ NestedType: []*descriptorpb.DescriptorProto{
+ {
+ Name: proto.String("EntriesEntry"),
+ Field: []*descriptorpb.FieldDescriptorProto{
+ {
+ Name: proto.String("key"),
+ Number: proto.Int32(1),
+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),
+ Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(),
+ },
+ {
+ Name: proto.String("value"),
+ Number: proto.Int32(2),
+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),
+ Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(),
+ TypeName: proto.String(".audittest.maps.Entry"),
+ },
+ },
+ Options: &descriptorpb.MessageOptions{MapEntry: proto.Bool(true)},
+ },
+ },
+ Field: []*descriptorpb.FieldDescriptorProto{
+ {
+ Name: proto.String("entries"),
+ Number: proto.Int32(1),
+ Label: descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum(),
+ Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(),
+ TypeName: proto.String(".audittest.maps.Holder.EntriesEntry"),
+ },
+ },
+ }, &descriptorpb.DescriptorProto{
+ Name: proto.String("Entry"),
+ Field: []*descriptorpb.FieldDescriptorProto{
+ {
+ Name: proto.String("secret"),
+ Number: proto.Int32(1),
+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),
+ Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(),
+ Options: sensitiveFieldOptions(),
+ },
+ },
+ })
+}
+
+func sensitiveFieldOptions() *descriptorpb.FieldOptions {
+ options := &descriptorpb.FieldOptions{}
+ proto.SetExtension(options, v1pb.E_AuditBehavior, v1pb.AuditBehavior_SENSITIVE)
+ return options
+}
+
+// buildTestMessage compiles a throwaway file descriptor and returns its first
+// message. Building the fixture rather than borrowing one from the surface is
+// the point: the shapes it covers — an annotation inside a cycle, and one
+// inside a map value — do not exist in the v1 protos today, which is precisely
+// why nothing would notice if the redactor stopped handling them.
+func buildTestMessage(t *testing.T, path, pkg string, messages ...*descriptorpb.DescriptorProto) protoreflect.MessageDescriptor {
+ t.Helper()
+ file, err := protodesc.NewFile(&descriptorpb.FileDescriptorProto{
+ Name: proto.String(path),
+ Package: proto.String(pkg),
+ Syntax: proto.String("proto3"),
+ MessageType: messages,
+ }, nil)
+ require.NoError(t, err)
+ return file.Messages().Get(0)
+}
+
+// auditPopulation is every (procedure, direction) pair whose message can reach
+// marshalAuditPayload, keyed by message type: 210 methods resolve to far fewer
+// distinct types, and the redactor is per type.
+func auditPopulation(t *testing.T) map[protoreflect.FullName][]string {
+ t.Helper()
+ byType := map[protoreflect.FullName][]string{}
+ protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {
+ if fd.Package() != "bytebase.v1" {
+ return true
+ }
+ services := fd.Services()
+ for i := range services.Len() {
+ sd := services.Get(i)
+ methods := sd.Methods()
+ for j := range methods.Len() {
+ md := methods.Get(j)
+ procedure := fmt.Sprintf("/%s/%s", sd.FullName(), md.Name())
+ byType[md.Input().FullName()] = append(byType[md.Input().FullName()], procedure+" request")
+ byType[md.Output().FullName()] = append(byType[md.Output().FullName()], procedure+" response")
+ }
+ }
+ return true
+ })
+ require.NotEmpty(t, byType)
+ return byType
+}
+
+// messageDescriptorByName resolves a name the population produced.
+func messageDescriptorByName(t *testing.T, name protoreflect.FullName) protoreflect.MessageDescriptor {
+ t.Helper()
+ descriptor, err := protoregistry.GlobalFiles.FindDescriptorByName(name)
+ require.NoError(t, err)
+ md, ok := descriptor.(protoreflect.MessageDescriptor)
+ require.True(t, ok, "%s is not a message", name)
+ return md
+}
+
+// isWellKnown reports whether a message comes from outside our own protos.
+// The fill skips these: google.protobuf.Value rejects an empty instance at
+// marshal time, and building one just to prove it holds no credential is
+// wasted work. TestNoAnnotationOutsideBytebaseProtos is what makes the skip
+// safe rather than assumed.
+func isWellKnown(descriptor protoreflect.MessageDescriptor) bool {
+ return !strings.HasPrefix(string(descriptor.FullName()), "bytebase.")
+}
+
+// auditTarget is one annotated field and the chain of fields that reaches it
+// from a top-level request or response message.
+//
+// One message per FIELD rather than one message with every field set: setting a
+// second arm of a oneof clears the first, so a single-message sweep would
+// exercise only the last-numbered arm while every other assertion passed
+// vacuously. Driving off the target path also reaches a oneof nested inside
+// another oneof's arm — every AppIMSetting payload arm sits under SettingValue's
+// app_im arm.
+type auditTarget struct {
+ path string
+ chain []protoreflect.FieldDescriptor
+}
+
+// annotatedTargets enumerates every annotated field reachable from a
+// descriptor, through repeated fields, map values and every arm of every oneof
+// at once. Recursion into a type already on the path stops, which bounds the
+// four cyclic components in the descriptor graph.
+func annotatedTargets(descriptor protoreflect.MessageDescriptor, prefix string, chain []protoreflect.FieldDescriptor, onPath map[protoreflect.FullName]bool) []auditTarget {
+ if onPath[descriptor.FullName()] || isWellKnown(descriptor) {
+ return nil
+ }
+ onPath[descriptor.FullName()] = true
+ defer delete(onPath, descriptor.FullName())
+
+ var targets []auditTarget
+ for i := range descriptor.Fields().Len() {
+ field := descriptor.Fields().Get(i)
+ path := fmt.Sprintf("%s.%s", prefix, field.Name())
+ next := append(append([]protoreflect.FieldDescriptor{}, chain...), field)
+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {
+ targets = append(targets, auditTarget{path: path, chain: next})
+ continue
+ }
+ if sub := submessageOf(field); sub != nil {
+ targets = append(targets, annotatedTargets(sub, path, next, onPath)...)
+ }
+ }
+ return targets
+}
+
+// seedTarget populates exactly one annotated field, walking the chain and
+// creating the containers on the way. It reports whether a sentinel value
+// actually landed: a target it cannot seed is a target the assertions below
+// would pass vacuously, so the caller fails instead of skipping.
+func seedTarget(m protoreflect.Message, chain []protoreflect.FieldDescriptor) bool {
+ field := chain[0]
+ if len(chain) == 1 {
+ return seedAnnotatedField(m, field)
+ }
+ switch {
+ case field.IsMap():
+ entry := m.Mutable(field).Map().NewValue()
+ if !seedTarget(entry.Message(), chain[1:]) {
+ return false
+ }
+ m.Mutable(field).Map().Set(protoreflect.ValueOfString("key").MapKey(), entry)
+ return true
+ case field.IsList():
+ list := m.Mutable(field).List()
+ element := list.NewElement()
+ if !seedTarget(element.Message(), chain[1:]) {
+ return false
+ }
+ list.Append(element)
+ return true
+ default:
+ return seedTarget(m.Mutable(field).Message(), chain[1:])
+ }
+}
+
+// seedAnnotatedField puts the sentinel into the annotated field itself. A
+// scalar takes it directly; an annotated message or container takes it on the
+// first scalar reachable inside, so an OMIT subtree like User.profile or
+// QueryResult.rows is populated rather than merely present.
+func seedAnnotatedField(m protoreflect.Message, field protoreflect.FieldDescriptor) bool {
+ if sub := submessageOf(field); sub != nil {
+ seeded := false
+ switch {
+ case field.IsMap():
+ entry := m.Mutable(field).Map().NewValue()
+ seeded = seedFirstScalar(entry.Message(), map[protoreflect.FullName]bool{})
+ m.Mutable(field).Map().Set(protoreflect.ValueOfString("key").MapKey(), entry)
+ case field.IsList():
+ list := m.Mutable(field).List()
+ element := list.NewElement()
+ seeded = seedFirstScalar(element.Message(), map[protoreflect.FullName]bool{})
+ list.Append(element)
+ default:
+ seeded = seedFirstScalar(m.Mutable(field).Message(), map[protoreflect.FullName]bool{})
+ }
+ return seeded
+ }
+ value, ok := sentinelValue(field)
+ if !ok {
+ return false
+ }
+ if field.IsList() {
+ m.Mutable(field).List().Append(value)
+ return true
+ }
+ m.Set(field, value)
+ return true
+}
+
+// seedFirstScalar puts the sentinel on the first string or bytes field
+// reachable inside a message, descending until it finds one.
+func seedFirstScalar(m protoreflect.Message, onPath map[protoreflect.FullName]bool) bool {
+ descriptor := m.Descriptor()
+ if descriptor.FullName() == anyFullName {
+ // An Any carries no scalar of its own worth seeding, but its type_url
+ // is a string and is what an annotated Any field has to be shown
+ // dropping. It never reaches protojson, because the annotation that
+ // made it a target is what removes it.
+ m.Set(descriptor.Fields().ByName("type_url"), protoreflect.ValueOfString(secretSentinel))
+ return true
+ }
+ if onPath[descriptor.FullName()] || isWellKnown(descriptor) {
+ return false
+ }
+ onPath[descriptor.FullName()] = true
+ defer delete(onPath, descriptor.FullName())
+
+ for i := range descriptor.Fields().Len() {
+ field := descriptor.Fields().Get(i)
+ if submessageOf(field) != nil {
+ continue
+ }
+ if value, ok := sentinelValue(field); ok {
+ if field.IsList() {
+ m.Mutable(field).List().Append(value)
+ } else {
+ m.Set(field, value)
+ }
+ return true
+ }
+ }
+ for i := range descriptor.Fields().Len() {
+ field := descriptor.Fields().Get(i)
+ sub := submessageOf(field)
+ if sub == nil {
+ continue
+ }
+ switch {
+ case field.IsMap():
+ entry := m.Mutable(field).Map().NewValue()
+ if seedFirstScalar(entry.Message(), onPath) {
+ m.Mutable(field).Map().Set(protoreflect.ValueOfString("key").MapKey(), entry)
+ return true
+ }
+ case field.IsList():
+ list := m.Mutable(field).List()
+ element := list.NewElement()
+ if seedFirstScalar(element.Message(), onPath) {
+ list.Append(element)
+ return true
+ }
+ default:
+ if seedFirstScalar(m.Mutable(field).Message(), onPath) {
+ return true
+ }
+ m.Clear(field)
+ }
+ }
+ return false
+}
+
+func sentinelValue(field protoreflect.FieldDescriptor) (protoreflect.Value, bool) {
+ switch field.Kind() {
+ case protoreflect.StringKind:
+ return protoreflect.ValueOfString(secretSentinel), true
+ case protoreflect.BytesKind:
+ return protoreflect.ValueOfBytes([]byte(secretSentinel)), true
+ default:
+ return protoreflect.Value{}, false
+ }
+}
+
+// assertNoAnnotatedContent walks every populated field of a redacted message and
+// requires that annotated ones carry no content. It is the whole-annotation
+// assertion the string check below cannot make: a field the fixture never
+// populated passes a "does not contain" check for free.
+//
+// Blank rather than absent, because a SENSITIVE oneof arm stays present with an
+// empty value — that is how the row still records which credential type the
+// caller supplied.
+func assertNoAnnotatedContent(t *testing.T, m protoreflect.Message, path string) {
+ t.Helper()
+ m.Range(func(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {
+ fieldPath := fmt.Sprintf("%s.%s", path, field.Name())
+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {
+ require.True(t, isBlankValue(field, value), "annotated field %s survived redaction", fieldPath)
+ return true
+ }
+ switch {
+ case field.IsMap():
+ if isMessageKind(field.MapValue().Kind()) {
+ value.Map().Range(func(key protoreflect.MapKey, entry protoreflect.Value) bool {
+ assertNoAnnotatedContent(t, entry.Message(), fmt.Sprintf("%s[%v]", fieldPath, key))
+ return true
+ })
+ }
+ case isMessageKind(field.Kind()) && field.IsList():
+ for i := range value.List().Len() {
+ assertNoAnnotatedContent(t, value.List().Get(i).Message(), fmt.Sprintf("%s[%d]", fieldPath, i))
+ }
+ case isMessageKind(field.Kind()):
+ assertNoAnnotatedContent(t, value.Message(), fieldPath)
+ default:
+ }
+ return true
+ })
+}
+
+func isBlankValue(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {
+ switch {
+ case field.IsMap():
+ return value.Map().Len() == 0
+ case field.IsList():
+ return value.List().Len() == 0
+ case field.Kind() == protoreflect.BytesKind:
+ return len(value.Bytes()) == 0
+ case field.Kind() == protoreflect.StringKind:
+ return value.String() == ""
+ case isMessageKind(field.Kind()):
+ return !value.Message().IsValid() || proto.Size(value.Message().Interface()) == 0
+ default:
+ return false
+ }
+}
+
+// dynamicMessage builds an empty instance of a descriptor through the global
+// type registry, so the sweep needs no per-type Go literal.
+func dynamicMessage(t *testing.T, descriptor protoreflect.MessageDescriptor) proto.Message {
+ t.Helper()
+ messageType, err := protoregistry.GlobalTypes.FindMessageByName(descriptor.FullName())
+ require.NoError(t, err)
+ return messageType.New().Interface()
+}
diff --git a/backend/api/v1/audit_redaction_test.go b/backend/api/v1/audit_redaction_test.go
deleted file mode 100644
index d8ccf3cecd..0000000000
--- a/backend/api/v1/audit_redaction_test.go
+++ /dev/null
@@ -1,754 +0,0 @@
-package v1
-
-import (
- "encoding/base64"
- "fmt"
- "slices"
- "testing"
-
- "github.com/stretchr/testify/require"
- "google.golang.org/genproto/googleapis/api/annotations"
- "google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/reflect/protoreflect"
- "google.golang.org/protobuf/types/descriptorpb"
-
- v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
-)
-
-// Audited RPCs write their request and response payloads to audit_log, and to
-// stdout when RuntimeEnableAuditLogStdout is set. Anything with
-// bb.auditLogs.search/export, or read access to the log pipeline, can read them.
-// So no live credential may survive into either payload.
-//
-// These are behavioral: each case populates a secret and asserts the marshaled
-// audit string does not contain it. A redactor that stops covering a field fails
-// here rather than silently starting to log it.
-
-const secretSentinel = "s3cr3t-sentinel-value"
-
-var encodedSecretSentinel = base64.StdEncoding.EncodeToString([]byte(secretSentinel))
-
-func TestAuditResponseRedactsCredentials(t *testing.T) {
- for _, tt := range []struct {
- name string
- response any
- }{
- {
- // Only create and key rotation populate this; the read path never
- // does. It is a live API key.
- name: "service account key",
- response: &v1pb.ServiceAccount{Name: "serviceAccounts/[email protected]", ServiceKey: secretSentinel},
- },
- {
- name: "smtp password in a returned setting",
- response: &v1pb.Setting{Value: &v1pb.SettingValue{Value: &v1pb.SettingValue_Email{
- Email: &v1pb.EmailSetting{Config: &v1pb.EmailSetting_Smtp{
- Smtp: &v1pb.EmailSetting_SMTPConfig{Password: secretSentinel},
- }},
- }}},
- },
- {
- name: "idp client secret in a returned provider",
- response: &v1pb.IdentityProvider{Config: &v1pb.IdentityProviderConfig{
- Config: &v1pb.IdentityProviderConfig_Oauth2Config{
- Oauth2Config: &v1pb.OAuth2IdentityProviderConfig{ClientSecret: secretSentinel},
- },
- }},
- },
- {
- name: "project webhook URL",
- response: &v1pb.Project{Webhooks: []*v1pb.Webhook{{
- Name: "projects/project-a/webhooks/webhook-a",
- Url: secretSentinel,
- }}},
- },
- {
- name: "release response SQL",
- response: &v1pb.Release{Files: []*v1pb.Release_File{nil, {
- Statement: []byte(secretSentinel),
- }}},
- },
- {
- name: "saved query response SQL",
- response: &v1pb.SavedQuery{Content: []byte(secretSentinel)},
- },
- {
- name: "purchase checkout credentials",
- response: &v1pb.PurchaseResponse{
- PaymentUrl: secretSentinel,
- SessionId: secretSentinel,
- },
- },
- {
- name: "VCS provider user export",
- response: &v1pb.ExportVCSProviderUsersResponse{
- Content: []byte(secretSentinel),
- },
- },
- {
- name: "exported audit log content",
- response: &v1pb.ExportAuditLogsResponse{
- Content: []byte(secretSentinel),
- NextPageToken: "next-page-token",
- },
- },
- {
- // The single-instance response is redacted as defense in depth even
- // though the converter blanks these on reads; its batch twin gets
- // the same treatment rather than the default arm.
- name: "batch instance update response",
- response: &v1pb.BatchUpdateInstancesResponse{Instances: []*v1pb.Instance{{
- Name: "instances/instance-a",
- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},
- }}},
- },
- } {
- t.Run(tt.name, func(t *testing.T) {
- got, err := getResponseString(tt.response)
- require.NoError(t, err)
- require.NotContains(t, got, secretSentinel, "credential written to the audit log")
- require.NotContains(t, got, encodedSecretSentinel, "encoded content written to the audit log")
- })
- }
-}
-
-// dataSourceWithEverySecret is the readable fixture: every secret redactDataSource
-// masks today, on one data source, so a case reads as the request a caller sends.
-// It is a hand-kept list and cannot see a field added later —
-// TestAuditRedactsEveryInputOnlyDataSourceField is the net that can.
-func dataSourceWithEverySecret() *v1pb.DataSource {
- return &v1pb.DataSource{
- Id: "admin",
- Type: v1pb.DataSourceType_ADMIN,
- Username: "bytebase",
- Host: "db.example.com",
- Password: secretSentinel,
- SslCa: secretSentinel,
- SslCaPath: secretSentinel,
- SslCert: secretSentinel,
- SslCertPath: secretSentinel,
- SslKey: secretSentinel,
- SslKeyPath: secretSentinel,
- SshPassword: secretSentinel,
- SshPrivateKey: secretSentinel,
- AuthenticationPrivateKey: secretSentinel,
- AuthenticationPrivateKeyPassphrase: secretSentinel,
- MasterPassword: secretSentinel,
- ExternalSecret: &v1pb.DataSourceExternalSecret{
- AuthOption: &v1pb.DataSourceExternalSecret_Token{Token: secretSentinel},
- },
- SaslConfig: &v1pb.SASLConfig{Mechanism: &v1pb.SASLConfig_KrbConfig{
- KrbConfig: &v1pb.KerberosConfig{Keytab: []byte(secretSentinel)},
- }},
- IamExtension: &v1pb.DataSource_GcpCredential{
- GcpCredential: &v1pb.DataSource_GCPCredential{Content: secretSentinel},
- },
- }
-}
-
-// assertDataSourceIntact checks the fields a redactor rewrites rather than
-// clears, then returns the password for the caller's own comparison.
-func assertDataSourceIntact(t *testing.T, d *v1pb.DataSource) string {
- t.Helper()
- require.Equal(t, secretSentinel, d.GetAuthenticationPrivateKeyPassphrase(), "redaction mutated the passphrase")
- require.Equal(t, secretSentinel, d.GetGcpCredential().GetContent(), "redaction mutated the IAM credential")
- require.Equal(t, secretSentinel, string(d.GetSaslConfig().GetKrbConfig().GetKeytab()), "redaction mutated the keytab")
- return d.GetPassword()
-}
-
-// fillInputOnlyFields sets every INPUT_ONLY string and bytes field reachable
-// from m to the sentinel, singular or repeated. Each oneof takes its arm-th
-// variant, so calling this for every arm up to maxOneofArms covers every branch
-// of every oneof declared alongside its siblings.
-//
-// This reads the same annotation the read path is pinned against
-// (assertNoInputOnlyValues), so a credential added to the proto lands in the
-// fixture without anyone remembering to add it here. Two shapes it cannot build:
-// a map, and a oneof nested inside another oneof's arm, which one global arm
-// counter only ever constructs on one value. Neither is silent — inputOnlyLeaves
-// counts both, so a fill that cannot reach one fails the test rather than
-// quietly asking less of the redactor.
-func fillInputOnlyFields(m protoreflect.Message, arm int, onPath map[protoreflect.FullName]bool) {
- descriptor := m.Descriptor()
- if onPath[descriptor.FullName()] {
- return
- }
- onPath[descriptor.FullName()] = true
- defer delete(onPath, descriptor.FullName())
-
- skip := map[protoreflect.FieldNumber]bool{}
- for i := 0; i < descriptor.Oneofs().Len(); i++ {
- oneof := descriptor.Oneofs().Get(i)
- if oneof.IsSynthetic() {
- continue
- }
- for j := 0; j < oneof.Fields().Len(); j++ {
- skip[oneof.Fields().Get(j).Number()] = true
- }
- fillOneField(m, oneof.Fields().Get(arm%oneof.Fields().Len()), arm, onPath)
- }
- for i := 0; i < descriptor.Fields().Len(); i++ {
- if field := descriptor.Fields().Get(i); !skip[field.Number()] {
- fillOneField(m, field, arm, onPath)
- }
- }
-}
-
-func fillOneField(m protoreflect.Message, field protoreflect.FieldDescriptor, arm int, onPath map[protoreflect.FullName]bool) {
- if field.IsMap() {
- return
- }
- if field.Kind() == protoreflect.MessageKind {
- if field.IsList() {
- list := m.Mutable(field).List()
- element := list.NewElement()
- fillInputOnlyFields(element.Message(), arm, onPath)
- list.Append(element)
- return
- }
- fillInputOnlyFields(m.Mutable(field).Message(), arm, onPath)
- return
- }
- if !isInputOnly(field) {
- return
- }
- var value protoreflect.Value
- switch field.Kind() {
- case protoreflect.StringKind:
- value = protoreflect.ValueOfString(secretSentinel)
- case protoreflect.BytesKind:
- value = protoreflect.ValueOfBytes([]byte(secretSentinel))
- default:
- return
- }
- if field.IsList() {
- m.Mutable(field).List().Append(value)
- return
- }
- m.Set(field, value)
-}
-
-// maxOneofArms returns the widest non-synthetic oneof reachable from the
-// descriptor. Driving the arm loop off this rather than off a literal is the
-// point: a fourth iam_extension variant raises the bound by declaring itself.
-func maxOneofArms(descriptor protoreflect.MessageDescriptor, onPath map[protoreflect.FullName]bool) int {
- if onPath[descriptor.FullName()] {
- return 0
- }
- onPath[descriptor.FullName()] = true
- defer delete(onPath, descriptor.FullName())
-
- widest := 1
- for i := 0; i < descriptor.Oneofs().Len(); i++ {
- if oneof := descriptor.Oneofs().Get(i); !oneof.IsSynthetic() {
- widest = max(widest, oneof.Fields().Len())
- }
- }
- for i := 0; i < descriptor.Fields().Len(); i++ {
- if field := descriptor.Fields().Get(i); field.Kind() == protoreflect.MessageKind && !field.IsMap() {
- widest = max(widest, maxOneofArms(field.Message(), onPath))
- }
- }
- return widest
-}
-
-// inputOnlyLeaves is the yardstick: every INPUT_ONLY scalar reachable from the
-// descriptor, down every arm of every oneof at once and through map values. The
-// fill can only take one arm per pass and cannot build a map at all, so the
-// union over passes has to equal this — which is what makes a fill that quietly
-// reaches less than it claims fail instead of pass. Every other assertion in the
-// test is negative, and a field nobody populated passes them all.
-//
-// Counting what the fill cannot yet build is deliberate. DataSource already
-// carries a map (extra_connection_parameters); the day one like it is declared
-// INPUT_ONLY, this fails and names it, rather than the gap staying a comment.
-func inputOnlyLeaves(descriptor protoreflect.MessageDescriptor, prefix string, onPath map[protoreflect.FullName]bool) []string {
- if onPath[descriptor.FullName()] {
- return nil
- }
- onPath[descriptor.FullName()] = true
- defer delete(onPath, descriptor.FullName())
-
- var paths []string
- for i := 0; i < descriptor.Fields().Len(); i++ {
- field := descriptor.Fields().Get(i)
- path := fmt.Sprintf("%s.%s", prefix, field.Name())
- switch {
- case field.IsMap():
- if isInputOnly(field) {
- paths = append(paths, path)
- }
- if field.MapValue().Kind() == protoreflect.MessageKind {
- paths = append(paths, inputOnlyLeaves(field.MapValue().Message(), path, onPath)...)
- }
- case field.Kind() == protoreflect.MessageKind:
- paths = append(paths, inputOnlyLeaves(field.Message(), path, onPath)...)
- case isInputOnly(field):
- paths = append(paths, path)
- default:
- }
- }
- return paths
-}
-
-// sentinelPaths reports which fields the fill actually reached.
-func sentinelPaths(m protoreflect.Message, prefix string) []string {
- var paths []string
- m.Range(func(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {
- path := fmt.Sprintf("%s.%s", prefix, field.Name())
- switch {
- case field.IsMap():
- case field.Kind() == protoreflect.MessageKind && field.IsList():
- for i := range value.List().Len() {
- paths = append(paths, sentinelPaths(value.List().Get(i).Message(), path)...)
- }
- case field.Kind() == protoreflect.MessageKind:
- paths = append(paths, sentinelPaths(value.Message(), path)...)
- case field.IsList():
- for i := range value.List().Len() {
- if isSentinel(field, value.List().Get(i)) {
- paths = append(paths, path)
- break
- }
- }
- default:
- if isSentinel(field, value) {
- paths = append(paths, path)
- }
- }
- return true
- })
- return paths
-}
-
-func isSentinel(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {
- switch field.Kind() {
- case protoreflect.StringKind:
- return value.String() == secretSentinel
- case protoreflect.BytesKind:
- return string(value.Bytes()) == secretSentinel
- default:
- return false
- }
-}
-
-func isInputOnly(field protoreflect.FieldDescriptor) bool {
- options, ok := field.Options().(*descriptorpb.FieldOptions)
- if !ok || options == nil {
- return false
- }
- behaviors, ok := proto.GetExtension(options, annotations.E_FieldBehavior).([]annotations.FieldBehavior)
- return ok && slices.Contains(behaviors, annotations.FieldBehavior_INPUT_ONLY)
-}
-
-// A hand-kept fixture cannot fail for a credential nobody added to it. This one
-// is derived from the proto instead: every INPUT_ONLY field on a DataSource,
-// every arm of every oneof under it, on each request that carries one. A new
-// credential field, or a fourth iam_extension variant, is covered on the day it
-// is declared rather than on the day someone notices the audit row.
-func TestAuditRedactsEveryInputOnlyDataSourceField(t *testing.T) {
- dataSource := (&v1pb.DataSource{}).ProtoReflect().Descriptor()
- reached := map[string]bool{}
- for arm := range maxOneofArms(dataSource, map[protoreflect.FullName]bool{}) {
- filled := &v1pb.DataSource{}
- fillInputOnlyFields(filled.ProtoReflect(), arm, map[protoreflect.FullName]bool{})
- for _, path := range sentinelPaths(filled.ProtoReflect(), "dataSource") {
- reached[path] = true
- }
-
- for _, tt := range []struct {
- name string
- request any
- }{
- {"create instance", &v1pb.CreateInstanceRequest{Instance: &v1pb.Instance{
- DataSources: []*v1pb.DataSource{filled},
- }}},
- {"update instance", &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{
- DataSources: []*v1pb.DataSource{filled},
- }}},
- {"batch update instances", &v1pb.BatchUpdateInstancesRequest{
- Requests: []*v1pb.UpdateInstanceRequest{{Instance: &v1pb.Instance{
- DataSources: []*v1pb.DataSource{filled},
- }}},
- }},
- {"add data source", &v1pb.AddDataSourceRequest{DataSource: filled}},
- {"update data source", &v1pb.UpdateDataSourceRequest{DataSource: filled}},
- {"remove data source", &v1pb.RemoveDataSourceRequest{DataSource: filled}},
- // The seventh carrier, and the one that reaches the audit log only
- // because MCP denials are recorded: ListInstanceDatabase is
- // EXCLUDED and unaudited, so its request was never written down
- // until the ceiling gate started recording its refusals.
- {"list instance database", &v1pb.ListInstanceDatabaseRequest{Instance: &v1pb.Instance{
- DataSources: []*v1pb.DataSource{filled},
- }}},
- } {
- t.Run(fmt.Sprintf("%s/oneof arm %d", tt.name, arm), func(t *testing.T) {
- got, err := getRequestString(tt.request)
- require.NoError(t, err)
- require.NotContains(t, got, secretSentinel, "credential written to the audit log")
- require.NotContains(t, got, encodedSecretSentinel, "encoded credential written to the audit log")
- })
- }
-
- t.Run(fmt.Sprintf("redacted message/oneof arm %d", arm), func(t *testing.T) {
- // The string check above only catches what the fixture happened to
- // populate. This one holds the whole annotation: nothing declared
- // INPUT_ONLY may carry a value after redaction.
- assertNoInputOnlyValues(t, redactDataSource(filled).ProtoReflect(), "dataSource")
- })
- }
-
- // The positive half. Everything above is an assertion that something is
- // absent, which a field the fill never populated passes for free — so under-
- // coverage would read as success. This one fails instead.
- t.Run("the fill reaches every INPUT_ONLY field", func(t *testing.T) {
- var missed []string
- for _, path := range inputOnlyLeaves(dataSource, "dataSource", map[protoreflect.FullName]bool{}) {
- if !reached[path] {
- missed = append(missed, path)
- }
- }
- require.Empty(t, missed, "never populated, so the redaction assertions above say nothing about them")
- })
-}
-
-// A rejected request is audited too, so every redactor runs on messages the
-// handler was about to refuse. redactDataSource used to read through the nil
-// data source in one of these and panic the interceptor.
-func TestAuditRedactionSurvivesAnIncompleteRequest(t *testing.T) {
- for _, tt := range []struct {
- name string
- request any
- }{
- {"add data source without one", &v1pb.AddDataSourceRequest{Name: "instances/instance-a"}},
- {"update data source without one", &v1pb.UpdateDataSourceRequest{Name: "instances/instance-a"}},
- {"remove data source without one", &v1pb.RemoveDataSourceRequest{Name: "instances/instance-a"}},
- {"create instance without one", &v1pb.CreateInstanceRequest{InstanceId: "instance-a"}},
- {"update instance without one", &v1pb.UpdateInstanceRequest{}},
- {"batch update instances without any", &v1pb.BatchUpdateInstancesRequest{
- Requests: []*v1pb.UpdateInstanceRequest{{}},
- }},
- {"instance carrying an empty data source", &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{
- DataSources: []*v1pb.DataSource{{}},
- }}},
- } {
- t.Run(tt.name, func(t *testing.T) {
- got, err := getRequestString(tt.request)
- require.NoError(t, err)
- require.NotEmpty(t, got, "a rejected request still gets a row")
- })
- }
-}
-
-func TestPrepareSampleProjectInstanceAuditRequestContainsOnlyParent(t *testing.T) {
- request := &v1pb.PrepareSampleProjectInstanceRequest{Parent: "projects/project-a"}
-
- got, err := getRequestString(request)
- require.NoError(t, err)
- require.Contains(t, got, "projects/project-a")
- require.NotContains(t, got, secretSentinel)
- require.Equal(
- t,
- "projects/project-a",
- getRequestResource(request, "/bytebase.v1.InstanceService/PrepareSampleProjectInstance"),
- )
-}
-
-func TestAuditRequestRedactsCredentials(t *testing.T) {
- batchParent := "projects/project-a"
- settingRequest := func(v *v1pb.SettingValue) *v1pb.UpdateSettingRequest {
- return &v1pb.UpdateSettingRequest{Setting: &v1pb.Setting{Value: v}}
- }
- imSetting := func(s *v1pb.AppIMSetting_IMSetting) *v1pb.UpdateSettingRequest {
- return settingRequest(&v1pb.SettingValue{Value: &v1pb.SettingValue_AppIm{
- AppIm: &v1pb.AppIMSetting{Settings: []*v1pb.AppIMSetting_IMSetting{s}},
- }})
- }
-
- for _, tt := range []struct {
- name string
- request any
- }{
- {"smtp password", settingRequest(&v1pb.SettingValue{Value: &v1pb.SettingValue_Email{
- Email: &v1pb.EmailSetting{Config: &v1pb.EmailSetting_Smtp{
- Smtp: &v1pb.EmailSetting_SMTPConfig{Password: secretSentinel},
- }},
- }})},
- {"ai api key", settingRequest(&v1pb.SettingValue{Value: &v1pb.SettingValue_Ai{
- Ai: &v1pb.AISetting{ApiKey: secretSentinel},
- }})},
- {"slack token", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Slack{Slack: &v1pb.AppIMSetting_Slack{Token: secretSentinel}}})},
- {"feishu app secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Feishu{Feishu: &v1pb.AppIMSetting_Feishu{AppSecret: secretSentinel}}})},
- {"wecom secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Wecom{Wecom: &v1pb.AppIMSetting_Wecom{Secret: secretSentinel}}})},
- {"lark app secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Lark{Lark: &v1pb.AppIMSetting_Lark{AppSecret: secretSentinel}}})},
- {"dingtalk client secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Dingtalk{Dingtalk: &v1pb.AppIMSetting_DingTalk{ClientSecret: secretSentinel}}})},
- {"teams client secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Teams{Teams: &v1pb.AppIMSetting_Teams{ClientSecret: secretSentinel}}})},
- {"oauth2 client secret", &v1pb.CreateIdentityProviderRequest{IdentityProvider: &v1pb.IdentityProvider{
- Config: &v1pb.IdentityProviderConfig{Config: &v1pb.IdentityProviderConfig_Oauth2Config{
- Oauth2Config: &v1pb.OAuth2IdentityProviderConfig{ClientSecret: secretSentinel},
- }},
- }}},
- {"oidc client secret", &v1pb.CreateIdentityProviderRequest{IdentityProvider: &v1pb.IdentityProvider{
- Config: &v1pb.IdentityProviderConfig{Config: &v1pb.IdentityProviderConfig_OidcConfig{
- OidcConfig: &v1pb.OIDCIdentityProviderConfig{ClientSecret: secretSentinel},
- }},
- }}},
- {"ldap bind password", &v1pb.UpdateIdentityProviderRequest{IdentityProvider: &v1pb.IdentityProvider{
- Config: &v1pb.IdentityProviderConfig{Config: &v1pb.IdentityProviderConfig_LdapConfig{
- LdapConfig: &v1pb.LDAPIdentityProviderConfig{BindPassword: secretSentinel},
- }},
- }}},
- {"create project webhook URL", &v1pb.CreateProjectRequest{Project: &v1pb.Project{
- Webhooks: []*v1pb.Webhook{{Name: "projects/project-a/webhooks/webhook-a", Url: secretSentinel}},
- }}},
- {"update project webhook URL", &v1pb.UpdateProjectRequest{Project: &v1pb.Project{
- Name: "projects/project-a",
- Webhooks: []*v1pb.Webhook{{Name: "projects/project-a/webhooks/webhook-a", Url: secretSentinel}},
- }}},
- {"release SQL", &v1pb.CreateReleaseRequest{Release: &v1pb.Release{
- Files: []*v1pb.Release_File{nil, {Statement: []byte(secretSentinel)}},
- }}},
- {"updated release SQL", &v1pb.UpdateReleaseRequest{Release: &v1pb.Release{
- Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}},
- }}},
- {"saved query SQL", &v1pb.CreateSavedQueryRequest{SavedQuery: &v1pb.SavedQuery{
- Content: []byte(secretSentinel),
- }}},
- // The data-source secrets are INPUT_ONLY, so a write request is the only
- // place they appear — which is exactly what an audit row records.
- {"instance data source secrets", &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{
- Name: "instances/instance-a",
- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},
- }}},
- {"batch instance data source secrets", &v1pb.BatchUpdateInstancesRequest{
- Parent: &batchParent,
- Requests: []*v1pb.UpdateInstanceRequest{nil, {Instance: &v1pb.Instance{
- Name: "instances/instance-a",
- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},
- }}},
- }},
- {"created instance data source secrets", &v1pb.CreateInstanceRequest{
- InstanceId: "instance-a",
- Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}},
- }},
- {"added data source secrets", &v1pb.AddDataSourceRequest{
- Name: "instances/instance-a",
- DataSource: dataSourceWithEverySecret(),
- }},
- {"updated data source secrets", &v1pb.UpdateDataSourceRequest{
- Name: "instances/instance-a",
- DataSource: dataSourceWithEverySecret(),
- }},
- {"removed data source secrets", &v1pb.RemoveDataSourceRequest{
- Name: "instances/instance-a",
- DataSource: dataSourceWithEverySecret(),
- }},
- {"azure iam client secret", &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{
- IamExtension: &v1pb.DataSource_AzureCredential_{AzureCredential: &v1pb.DataSource_AzureCredential{
- TenantId: "tenant-a", ClientId: "client-a", ClientSecret: secretSentinel,
- }},
- }}},
- {"aws iam access keys", &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{
- IamExtension: &v1pb.DataSource_AwsCredential{AwsCredential: &v1pb.DataSource_AWSCredential{
- AccessKeyId: secretSentinel, SecretAccessKey: secretSentinel, SessionToken: secretSentinel,
- RoleArn: secretSentinel, ExternalId: secretSentinel,
- }},
- }}},
- {"password reset code and password", &v1pb.ResetPasswordRequest{
- Email: "[email protected]",
- Code: secretSentinel,
- NewPassword: secretSentinel,
- }},
- {"enterprise license", &v1pb.UploadLicenseRequest{License: secretSentinel}},
- {"add webhook URL", &v1pb.AddWebhookRequest{
- Project: "projects/project-a",
- Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel},
- }},
- {"update webhook URL", &v1pb.UpdateWebhookRequest{
- Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel},
- }},
- {"remove webhook URL", &v1pb.RemoveWebhookRequest{
- Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel},
- }},
- } {
- t.Run(tt.name, func(t *testing.T) {
- got, err := getRequestString(tt.request)
- require.NoError(t, err)
- require.NotContains(t, got, secretSentinel, "credential written to the audit log")
- require.NotContains(t, got, encodedSecretSentinel, "encoded content written to the audit log")
- if _, ok := tt.request.(*v1pb.ResetPasswordRequest); ok {
- require.Contains(t, got, "[email protected]", "email is required for audit attribution")
- }
- })
- }
-}
-
-func TestSensitiveOutboundAuditMetadata(t *testing.T) {
- batchParent := "projects/project-a"
- for _, tt := range []struct {
- name string
- value any
- response bool
- want []string
- }{
- {
- name: "webhook request retains identifiers",
- value: &v1pb.AddWebhookRequest{Project: "projects/project-a", Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel}},
- want: []string{"projects/project-a", "projects/project-a/webhooks/webhook-a", "Webhook A"},
- },
- {
- // The row has to stay useful: which instances were retargeted, and
- // which kind of IAM credential was supplied.
- name: "batch instance update retains targets and credential type",
- value: &v1pb.BatchUpdateInstancesRequest{
- Parent: &batchParent,
- Requests: []*v1pb.UpdateInstanceRequest{{Instance: &v1pb.Instance{
- Name: "instances/instance-a",
- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},
- }}},
- },
- want: []string{"projects/project-a", "instances/instance-a", "db.example.com", "gcpCredential"},
- },
- {
- name: "azure iam credential retains the principal it names",
- value: &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{
- IamExtension: &v1pb.DataSource_AzureCredential_{AzureCredential: &v1pb.DataSource_AzureCredential{
- TenantId: "tenant-a", ClientId: "client-a", ClientSecret: secretSentinel,
- }},
- }},
- want: []string{"azureCredential", "tenant-a", "client-a"},
- },
- {
- name: "aws iam credential records that one was supplied",
- value: &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{
- IamExtension: &v1pb.DataSource_AwsCredential{AwsCredential: &v1pb.DataSource_AWSCredential{
- AccessKeyId: secretSentinel, SecretAccessKey: secretSentinel,
- }},
- }},
- want: []string{"awsCredential"},
- },
- {
- name: "audit export retains page token",
- value: &v1pb.ExportAuditLogsResponse{Content: []byte(secretSentinel), NextPageToken: "next-page-token"},
- response: true,
- want: []string{"next-page-token"},
- },
- } {
- t.Run(tt.name, func(t *testing.T) {
- var (
- got string
- err error
- )
- if tt.response {
- got, err = getResponseString(tt.value)
- } else {
- got, err = getRequestString(tt.value)
- }
- require.NoError(t, err)
- for _, want := range tt.want {
- require.Contains(t, got, want)
- }
- })
- }
-}
-
-// Redaction must not mutate the caller's message: these run on the live request
-// and response objects, so masking in place would corrupt what the handler
-// returns to the client or what a later interceptor reads.
-func TestAuditRedactionDoesNotMutateInput(t *testing.T) {
- account := &v1pb.ServiceAccount{ServiceKey: secretSentinel}
- _, err := getResponseString(account)
- require.NoError(t, err)
- require.Equal(t, secretSentinel, account.GetServiceKey(), "redaction mutated the response")
-
- request := &v1pb.UpdateSettingRequest{Setting: &v1pb.Setting{Value: &v1pb.SettingValue{
- Value: &v1pb.SettingValue_Ai{Ai: &v1pb.AISetting{ApiKey: secretSentinel}},
- }}}
- _, err = getRequestString(request)
- require.NoError(t, err)
- require.Equal(t, secretSentinel, request.GetSetting().GetValue().GetAi().GetApiKey(),
- "redaction mutated the request")
-
- project := &v1pb.Project{Webhooks: []*v1pb.Webhook{{Url: secretSentinel}}}
- _, err = getResponseString(project)
- require.NoError(t, err)
- require.Equal(t, secretSentinel, project.GetWebhooks()[0].GetUrl(), "redaction mutated the project response")
-
- release := &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}
- _, err = getResponseString(release)
- require.NoError(t, err)
- require.Equal(t, secretSentinel, string(release.GetFiles()[0].GetStatement()), "redaction mutated the release response")
-
- savedQuery := &v1pb.SavedQuery{Content: []byte(secretSentinel)}
- _, err = getResponseString(savedQuery)
- require.NoError(t, err)
- require.Equal(t, secretSentinel, string(savedQuery.GetContent()), "redaction mutated the saved query response")
-
- auditExport := &v1pb.ExportAuditLogsResponse{Content: []byte(secretSentinel), NextPageToken: "next-page-token"}
- _, err = getResponseString(auditExport)
- require.NoError(t, err)
- require.Equal(t, secretSentinel, string(auditExport.GetContent()), "redaction mutated the audit export response")
-
- for _, tt := range []struct {
- name string
- request any
- }{
- {
- name: "create instance request",
- request: &v1pb.CreateInstanceRequest{Instance: &v1pb.Instance{
- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},
- }},
- },
- {
- name: "update instance request",
- request: &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{
- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},
- }},
- },
- {
- name: "batch update instances request",
- request: &v1pb.BatchUpdateInstancesRequest{Requests: []*v1pb.UpdateInstanceRequest{{
- Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}},
- }}},
- },
- {name: "add data source request", request: &v1pb.AddDataSourceRequest{DataSource: dataSourceWithEverySecret()}},
- {name: "update data source request", request: &v1pb.UpdateDataSourceRequest{DataSource: dataSourceWithEverySecret()}},
- {name: "remove data source request", request: &v1pb.RemoveDataSourceRequest{DataSource: dataSourceWithEverySecret()}},
- {name: "create release request", request: &v1pb.CreateReleaseRequest{Release: &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}}},
- {name: "update release request", request: &v1pb.UpdateReleaseRequest{Release: &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}}},
- {name: "create saved query request", request: &v1pb.CreateSavedQueryRequest{SavedQuery: &v1pb.SavedQuery{Content: []byte(secretSentinel)}}},
- {name: "reset password request", request: &v1pb.ResetPasswordRequest{Email: "[email protected]", Code: secretSentinel, NewPassword: secretSentinel}},
- } {
- t.Run(tt.name, func(t *testing.T) {
- _, err := getRequestString(tt.request)
- require.NoError(t, err)
- var sensitiveValue string
- switch request := tt.request.(type) {
- case *v1pb.CreateInstanceRequest:
- sensitiveValue = assertDataSourceIntact(t, request.GetInstance().GetDataSources()[0])
- case *v1pb.UpdateInstanceRequest:
- sensitiveValue = assertDataSourceIntact(t, request.GetInstance().GetDataSources()[0])
- case *v1pb.BatchUpdateInstancesRequest:
- sensitiveValue = assertDataSourceIntact(t, request.GetRequests()[0].GetInstance().GetDataSources()[0])
- case *v1pb.AddDataSourceRequest:
- sensitiveValue = assertDataSourceIntact(t, request.GetDataSource())
- case *v1pb.UpdateDataSourceRequest:
- sensitiveValue = assertDataSourceIntact(t, request.GetDataSource())
- case *v1pb.RemoveDataSourceRequest:
- sensitiveValue = assertDataSourceIntact(t, request.GetDataSource())
- case *v1pb.CreateReleaseRequest:
- sensitiveValue = string(request.GetRelease().GetFiles()[0].GetStatement())
- case *v1pb.UpdateReleaseRequest:
- sensitiveValue = string(request.GetRelease().GetFiles()[0].GetStatement())
- case *v1pb.CreateSavedQueryRequest:
- sensitiveValue = string(request.GetSavedQuery().GetContent())
- case *v1pb.ResetPasswordRequest:
- sensitiveValue = request.GetCode()
- require.Equal(t, secretSentinel, request.GetNewPassword())
- default:
- t.Fatalf("unexpected request type %T", request)
- }
- require.Equal(t, secretSentinel, sensitiveValue, "redaction mutated the request")
- })
- }
-}
diff --git a/backend/api/v1/audit_test.go b/backend/api/v1/audit_test.go
index eeedd09a8b..ef3dc9df33 100644
--- a/backend/api/v1/audit_test.go
+++ b/backend/api/v1/audit_test.go
@@ -135,77 +135,6 @@ func TestLogAuditToStdoutFormat(t *testing.T) {
a.Equal("client-A", delegated["mcp_client_id"])
}
-// TestAuditRedactsCredentials covers the request/response redactors that
-// strip secrets before the audit pipeline serializes payloads. Specifically
-// guards against regressions like the Signup password leak and the
-// ExchangeToken OIDC/access-token leak that Codex flagged on #20024 — both
-// only surfaced once the corresponding audit path was re-enabled by the
-// SetAuditWorkspaceID callback.
-func TestAuditRedactsCredentials(t *testing.T) {
- a := require.New(t)
-
- t.Run("LoginRequest redacts password and MFA secrets", func(_ *testing.T) {
- otp := "123456"
- mfa := "mfa-temp-jwt"
- reqStr, err := getRequestString(&v1pb.LoginRequest{
- Email: "[email protected]",
- Password: "hunter2",
- OtpCode: &otp,
- MfaTempToken: &mfa,
- })
- a.NoError(err)
- a.Contains(reqStr, "[email protected]", "non-sensitive email stays")
- a.NotContains(reqStr, "hunter2", "plaintext password must not appear")
- a.NotContains(reqStr, "123456", "OTP must not appear")
- a.NotContains(reqStr, "mfa-temp-jwt", "MFA temp token must not appear")
- })
-
- t.Run("LoginResponse drops token", func(_ *testing.T) {
- respStr, err := getResponseString(&v1pb.LoginResponse{
- Token: "secret-access-token",
- User: &v1pb.User{Name: "users/[email protected]"},
- })
- a.NoError(err)
- a.Contains(respStr, "users/[email protected]", "user info is retained")
- a.NotContains(respStr, "secret-access-token", "access token must not appear")
- })
-
- t.Run("SignupRequest redacts password", func(_ *testing.T) {
- reqStr, err := getRequestString(&v1pb.SignupRequest{
- Email: "[email protected]",
- Password: "signup-password",
- Title: "bob",
- })
- a.NoError(err)
- a.Contains(reqStr, "[email protected]")
- a.NotContains(reqStr, "signup-password",
- "plaintext password must not appear in Signup audit")
- })
-
- t.Run("ExchangeTokenRequest redacts OIDC token", func(_ *testing.T) {
- reqStr, err := getRequestString(&v1pb.ExchangeTokenRequest{
- Token: "oidc.jwt.payload",
- Email: "[email protected]",
- })
- a.NoError(err)
- a.Contains(reqStr, "[email protected]",
- "workload email retained for audit correlation")
- a.NotContains(reqStr, "oidc.jwt.payload",
- "external OIDC token must not appear in audit — it can be replayed "+
- "against the original IdP or reveal workload claims")
- })
-
- t.Run("ExchangeTokenResponse drops issued access token", func(_ *testing.T) {
- respStr, err := getResponseString(&v1pb.ExchangeTokenResponse{
- AccessToken: "issued-bytebase-api-token",
- })
- a.NoError(err)
- a.NotContains(respStr, "issued-bytebase-api-token",
- "issued Bytebase access token must never be logged — anyone with "+
- "audit read access could use it as a valid API token")
- })
-}
-
func TestFailedLoginWithoutWorkspaceIsSkipped(t *testing.T) {
ctx := context.WithValue(context.Background(), common.AuthContextKey, &common.AuthContext{Audit: true})
st := newAuditLiveStore(t)
@@ -398,6 +327,7 @@ func TestLifecycleAuditResource(t *testing.T) {
{name: "update webhook", request: &v1pb.UpdateWebhookRequest{Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a"}}, method: v1connect.ProjectServiceUpdateWebhookProcedure, want: "projects/project-a/webhooks/webhook-a"},
{name: "remove webhook", request: &v1pb.RemoveWebhookRequest{Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a"}}, method: v1connect.ProjectServiceRemoveWebhookProcedure, want: "projects/project-a/webhooks/webhook-a"},
{name: "export audit logs", request: &v1pb.ExportAuditLogsRequest{Parent: "projects/project-a"}, method: v1connect.AuditLogServiceExportAuditLogsProcedure, want: "projects/project-a"},
+ {name: "prepare sample project instance", request: &v1pb.PrepareSampleProjectInstanceRequest{Parent: "projects/project-a"}, method: "/bytebase.v1.InstanceService/PrepareSampleProjectInstance", want: "projects/project-a"},
} {
t.Run(test.name, func(t *testing.T) {
require.Equal(t, test.want, getRequestResource(test.request, test.method))
diff --git a/backend/api/v1/audit_validate_only_test.go b/backend/api/v1/audit_validate_only_test.go
index 5c4403615b..53f482fb7d 100644
--- a/backend/api/v1/audit_validate_only_test.go
+++ b/backend/api/v1/audit_validate_only_test.go
@@ -91,13 +91,12 @@ func TestValidateOnlyAuditSkipAppliesOnlyToSuccess(t *testing.T) {
require.Equal(t, int32(connect.CodePermissionDenied), rows[0].Payload.GetStatus().GetCode(),
"the row must carry the denial, not a blank status")
- // The newly-logged row goes through the same getRequestString
+ // The newly-logged row goes through the same marshalAuditPayload
// redaction as every other UpdateDataSource row — that is why
// narrowing the skip exposes no field the plain path did not already
- // write. It pins that the path runs, not that redactDataSource covers
- // every secret: the iam_extension oneof and
- // authentication_private_key_passphrase are unmasked there today, on
- // this row and on every UpdateDataSource row that preceded it.
+ // write. It pins that the path runs on a row the skip used to swallow;
+ // which fields the redaction covers is the annotation's job, pinned by
+ // TestAuditRedactionCoversEveryAnnotatedField.
request := rows[0].Payload.GetRequest()
require.NotEmpty(t, request)
require.NotContains(t, request, storedPassword,
diff --git a/backend/api/v1/instance_service_converter_test.go b/backend/api/v1/instance_service_converter_test.go
index a0a0689776..3cec259f4b 100644
--- a/backend/api/v1/instance_service_converter_test.go
+++ b/backend/api/v1/instance_service_converter_test.go
@@ -442,57 +442,88 @@ func TestRetainStoredKeytabs(t *testing.T) {
require.Empty(t, moved[1].GetSaslConfig().GetKrbConfig().Keytab)
}
-// assertNoInputOnlyValues walks every populated message field and requires
-// that fields annotated INPUT_ONLY carry no value. It pins the read-path
-// contract: whatever the proto declares write-only must be blanked by the
-// store->v1 converters.
-func assertNoInputOnlyValues(t *testing.T, m protoreflect.Message, path string) {
+// assertNoSensitiveValues walks every populated message field and requires that
+// fields the proto declares write-only or credential-bearing carry no value. It
+// pins the read-path contract: whatever is INPUT_ONLY or SENSITIVE must be
+// blanked by the store->v1 converters. A converter that legitimately mints one
+// — CreateServiceAccount returning the key it just made — is exercised by its
+// own test rather than exempted here, so this walk stays unconditional.
+//
+// SENSITIVE as well as INPUT_ONLY, because the two answer different questions
+// and neither subsumes the other: ServiceAccount.service_key is OUTPUT_ONLY and
+// is a live credential, while IdP client_secret and Login password carry no
+// field_behavior at all.
+func assertNoSensitiveValues(t *testing.T, m protoreflect.Message, path string) {
fields := m.Descriptor().Fields()
for i := 0; i < fields.Len(); i++ {
fd := fields.Get(i)
fieldPath := fmt.Sprintf("%s.%s", path, fd.Name())
- if opts, ok := fd.Options().(*descriptorpb.FieldOptions); ok && opts != nil {
- behaviors, ok := proto.GetExtension(opts, annotations.E_FieldBehavior).([]annotations.FieldBehavior)
- if ok && slices.Contains(behaviors, annotations.FieldBehavior_INPUT_ONLY) {
- // Oneof members may stay present as an is-configured signal
- // (e.g. the Vault token), so require blank content, not
- // absence.
- if m.Has(fd) {
- blank := false
- switch {
- case fd.IsList():
- blank = m.Get(fd).List().Len() == 0
- case fd.Kind() == protoreflect.BytesKind:
- blank = len(m.Get(fd).Bytes()) == 0
- case fd.Kind() == protoreflect.StringKind:
- blank = m.Get(fd).String() == ""
- default:
- }
- require.True(t, blank, "INPUT_ONLY field %s must be blank on reads", fieldPath)
- }
- continue
+ if isInputOnly(fd) || auditBehaviorOf(fd) == v1pb.AuditBehavior_SENSITIVE {
+ // Oneof members may stay present as an is-configured signal
+ // (e.g. the Vault token), so require blank content, not
+ // absence.
+ if m.Has(fd) {
+ require.True(t, isBlankValue(fd, m.Get(fd)), "write-only field %s must be blank on reads", fieldPath)
}
+ continue
}
if fd.IsMap() {
+ // Four converter-reachable maps carry message values, and a
+ // credential inside one would reach a read response with CI green
+ // if the walk stopped here.
+ if isMessageKind(fd.MapValue().Kind()) && m.Has(fd) {
+ m.Get(fd).Map().Range(func(key protoreflect.MapKey, entry protoreflect.Value) bool {
+ assertNoSensitiveValues(t, entry.Message(), fmt.Sprintf("%s[%v]", fieldPath, key))
+ return true
+ })
+ }
continue
}
- if fd.Kind() != protoreflect.MessageKind {
+ if !isMessageKind(fd.Kind()) {
continue
}
if fd.IsList() {
list := m.Get(fd).List()
for j := 0; j < list.Len(); j++ {
- assertNoInputOnlyValues(t, list.Get(j).Message(), fmt.Sprintf("%s[%d]", fieldPath, j))
+ assertNoSensitiveValues(t, list.Get(j).Message(), fmt.Sprintf("%s[%d]", fieldPath, j))
}
continue
}
if m.Has(fd) {
- assertNoInputOnlyValues(t, m.Get(fd).Message(), fieldPath)
+ assertNoSensitiveValues(t, m.Get(fd).Message(), fieldPath)
}
}
}
-func TestConvertDataSourcesBlanksEveryInputOnlyField(t *testing.T) {
+// isInputOnly reports whether the proto declares a field write-only.
+func isInputOnly(field protoreflect.FieldDescriptor) bool {
+ options, ok := field.Options().(*descriptorpb.FieldOptions)
+ if !ok || options == nil {
+ return false
+ }
+ behaviors, ok := proto.GetExtension(options, annotations.E_FieldBehavior).([]annotations.FieldBehavior)
+ return ok && slices.Contains(behaviors, annotations.FieldBehavior_INPUT_ONLY)
+}
+
+// InstanceRole.password is INPUT_ONLY and SENSITIVE, and the store message has
+// no password field at all — so reads cannot populate it. This pins that: the
+// audit-side leak this design opens with was the request path, and the read
+// path staying clean is what lets the annotation be SENSITIVE rather than OMIT.
+func TestConvertInstanceRolesBlanksEveryWriteOnlyField(t *testing.T) {
+ instance := &store.InstanceMessage{ResourceID: "instance-a"}
+ roles := []*storepb.InstanceRole{
+ {Name: "admin", Attribute: proto.String("SUPERUSER"), ConnectionLimit: proto.Int32(10)},
+ {Name: "readonly"},
+ }
+ got := convertInstanceRoles(instance, roles)
+ require.Len(t, got, len(roles))
+ for _, role := range got {
+ assertNoSensitiveValues(t, role.ProtoReflect(), fmt.Sprintf("instance_role(%s)", role.GetRoleName()))
+ }
+ require.Equal(t, "instances/instance-a/roles/admin", got[0].GetName())
+}
+
+func TestConvertDataSourcesBlanksEveryWriteOnlyField(t *testing.T) {
// One store data source per authentication/secret shape, each with every
// secret populated, so the INPUT_ONLY walk covers all converter branches.
dataSources := []*storepb.DataSource{
@@ -593,7 +624,7 @@ func TestConvertDataSourcesBlanksEveryInputOnlyField(t *testing.T) {
got := convertDataSources(dataSources)
require.Len(t, got, len(dataSources))
for _, ds := range got {
- assertNoInputOnlyValues(t, ds.ProtoReflect(), fmt.Sprintf("data_source(%s)", ds.GetId()))
+ assertNoSensitiveValues(t, ds.ProtoReflect(), fmt.Sprintf("data_source(%s)", ds.GetId()))
}
// The AWS credential keeps signaling presence without content.
diff --git a/backend/api/v1/mcp_gate.go b/backend/api/v1/mcp_gate.go
index 4226c15974..294c3ab9ca 100644
--- a/backend/api/v1/mcp_gate.go
+++ b/backend/api/v1/mcp_gate.go
@@ -313,7 +313,8 @@ func mcpSettingsFromContext(ctx context.Context) (store.MCPSettings, bool) {
// EXCLUDED ones — and TestEmailSetting and TestIdentityProvider are the rows an
// operator would most want, since each would have carried a stored secret to an
// address the agent chose. Recording requests that were never recorded is why
-// getRequestString grew redactors in the same change (audit.go).
+// redaction has to cover more than the audited RPCs (audit.go): a denial must
+// not transcribe the secret it refused.
//
// A ceiling the gate cannot act on splits in two, and the split is the same one
// the /mcp connection gate makes. A stored value this build cannot interpret —
diff --git a/backend/api/v1/mcp_gate_test.go b/backend/api/v1/mcp_gate_test.go
index fd4dd8dbc2..739cdb6210 100644
--- a/backend/api/v1/mcp_gate_test.go
+++ b/backend/api/v1/mcp_gate_test.go
@@ -590,8 +590,8 @@ func TestForbiddenClassLeavesReadsAlone(t *testing.T) {
// (project_service_converter.go), which copied the incoming-webhook URL out
// verbatim. That URL is a bearer credential — whoever holds it posts into
// the customer's Slack, Feishu, WeCom or Teams as Bytebase — and this repo
-// already treated it as one: redactWebhook masks it in audit rows
-// (audit.go) beside the OIDC client secret and the LDAP bind password.
+// already treated it as one: it is annotated SENSITIVE, so audit rows drop
+// it beside the OIDC client secret and the LDAP bind password.
// ListProjects pages the whole workspace, so one call took the lot. The url
// is write-only now, the way a data source's password is: reads leave it
// empty. TestWebhook takes the stored url when the request names a webhook
@@ -1348,27 +1348,27 @@ var mcpDenialRequestsUnderReview = map[string]mcpDenialRequestReview{
// masks, and the gate refuses before dispatch, so it was never even used.
v1connect.AIServiceChatProcedure: {
[]string{"messages", "tool_definitions"},
- "redactAIChatRequest drops the conversation body, which is unbounded and stored untruncated",
+ "messages and tool_definitions are annotated OMIT, so the unbounded conversation body is dropped",
},
v1connect.AuthServiceSwitchWorkspaceProcedure: {
[]string{"mfa_temp_token", "otp_code", "recovery_code", "web", "workspace"},
- "redactSwitchWorkspaceRequest masks the three MFA proofs; the workspace it targeted and the cookie flag stay",
+ "the three MFA proofs are annotated SENSITIVE; the workspace it targeted and the cookie flag stay",
},
v1connect.IdentityProviderServiceTestIdentityProviderProcedure: {
[]string{"identity_provider", "ldap_context", "oauth2_context", "oidc_context"},
- "redactTestIdentityProviderRequest masks the provider secret and the test credential",
+ "the provider secret and the test credential are annotated SENSITIVE, wherever they appear",
},
v1connect.InstanceServiceListInstanceDatabaseProcedure: {
[]string{"instance"},
- "redactInstance masks every data-source credential; pinned by TestAuditRedactsEveryInputOnlyDataSourceField",
+ "every data-source credential is annotated SENSITIVE; pinned by TestAuditRedactionCoversEveryAnnotatedField",
},
v1connect.ProjectServiceTestWebhookProcedure: {
[]string{"project", "webhook"},
- "redactWebhook masks the URL, which is a bearer credential; the project stays",
+ "Webhook.url is annotated SENSITIVE because it is a bearer credential; the project stays",
},
v1connect.SettingServiceTestEmailSettingProcedure: {
[]string{"email_setting", "to"},
- "redactEmailSetting masks the SMTP password; the address the agent chose to mail stays, which is the point of the row",
+ "the SMTP password is annotated SENSITIVE; the address the agent chose to mail stays, which is the point of the row",
},
// Recorded as sent. None of it is a credential, and what the row keeps is
@@ -1488,11 +1488,16 @@ func mcpRequestFieldNeedsReview(field protoreflect.FieldDescriptor) bool {
// on one, joins the population above without anyone deciding what its denial
// row may carry.
//
-// It cannot check that a redactor exists — getRequestString's type switch is
-// not readable from here — so it checks the thing that failed twice: that
-// somebody looked, at each field rather than at the method as a whole. A new
-// unaudited RPC, or a new field on one already reviewed, breaks the build; the
-// fix is a redactor plus a row here, or a row here saying why none is needed.
+// It checks the thing that failed twice: that somebody looked, at each field
+// rather than at the method as a whole. A new unaudited RPC, or a new field on
+// one already reviewed, breaks the build; the fix is an audit_behavior
+// annotation plus a row here, or a row here saying why none is needed.
+//
+// It sits alongside the inventory in audit_redact_inventory_test.go rather than
+// being replaced by it. The inventory is per (message, string-or-bytes field)
+// across the whole audited surface; this is per (method, field) and covers the
+// message-typed fields the inventory's scalar scope does not — DiffMetadata's
+// target_metadata is a whole schema, and no string field names it.
func TestLintDenialRequestsAreReviewedForRedaction(t *testing.T) {
needsReview := map[string][]string{}
for _, row := range mcpClassificationsFromDescriptors(t) {
diff --git a/backend/api/v1/setting_service_directory_sync_test.go b/backend/api/v1/setting_service_directory_sync_test.go
index f23c589c6a..58f9fd8cdb 100644
--- a/backend/api/v1/setting_service_directory_sync_test.go
+++ b/backend/api/v1/setting_service_directory_sync_test.go
@@ -80,8 +80,7 @@ func TestRotateDirectorySyncTokenRejectsMalformedNames(t *testing.T) {
func TestRotateDirectorySyncTokenResponseIsRedactedForAudit(t *testing.T) {
const token = "b17c0d9e-2222-4000-8000-fedcbafedcba"
- got, err := getResponseString(&v1pb.RotateDirectorySyncTokenResponse{Token: token})
- require.NoError(t, err)
+ got := marshalAuditPayload(&v1pb.RotateDirectorySyncTokenResponse{Token: token})
require.NotContains(t, got, token, "minted SCIM token written to the audit log")
}
diff --git a/docs/design/audit-payload-redaction-from-proto.md b/docs/design/audit-payload-redaction-from-proto.md
index 4a438199e4..f0c4cb5de1 100644
--- a/docs/design/audit-payload-redaction-from-proto.md
+++ b/docs/design/audit-payload-redaction-from-proto.md
@@ -1,6 +1,6 @@
# Audit payload redaction from proto annotations
-Status: proposal · 2026-08-20 · addresses BYT-10090
+Status: implemented · 2026-08-21 · addresses BYT-10090
## Background
@@ -157,7 +157,8 @@ lints are the whole enforcement story, and an annotation implying otherwise is w
### Redaction
-The hook point is `getRequestString` and `getResponseString` themselves, not `WrapUnary`. Unary and
+The hook point is `marshalAuditPayload` (originally two functions, `getRequestString` and
+`getResponseString`), not `WrapUnary`. Unary and
streaming rows reach redaction only through those two: `auditConnectStreamingConn.Send`
(`audit.go:171-193`) builds its own `auditEntry` and calls `createAuditLog` directly. `AdminExecute`
is the only streaming RPC, is audited, and its response carries every row of an admin-mode query, so
@@ -253,7 +254,7 @@ Constraints:
- **Every `Any` that reaches the row is registered and redacted.** The descriptor walk sees
`type_url` and `value`, never the packed message, so an annotated field inside one is invisible
to the redactor, the coverage lint and the inventory alike. Three paths put an `Any` on an audit
- row and none passes `getRequestString` or `getResponseString`:
+ row and none passes `marshalAuditPayload`:
| source | call sites today | packed types |
|---|---|---|
@@ -361,8 +362,8 @@ walking each of the ten rebuilds for what it newly admits, is a precondition for
- **Inventory** — the half that makes goal 4 real. A sentinel sweep cannot distinguish an
unannotated credential from an ordinary field the audit row intentionally keeps; both survive
redaction identically, so the sweep alone has no oracle. The oracle is a checked-in inventory of
- every string and bytes field reachable from a message that can reach `getRequestString` or
- `getResponseString`. A field missing from it fails the build; clearing that failure means
+ every string and bytes field reachable from a message that can reach `marshalAuditPayload`.
+ A field missing from it fails the build; clearing that failure means
annotating the field or recording that it is not a credential. Same shape as
`mcpDenialRequestsUnderReview` (`mcp_gate_test.go:1322`), which it replaces once the population
matches.
@@ -494,3 +495,95 @@ darwin/arm64, one run, with `INPUT_ONLY` standing in for the annotation set.
- **AIP-147** prescribes patterns, not an annotation: `INPUT_ONLY` plus `OUTPUT_ONLY bool
<name>_set`. We follow it inconsistently — `ssl_ca_set` does, `directory_sync_token_configured`
does not, and that field is in no tagged release yet.
+
+## Implementation
+
+- **Annotation** — `AuditBehavior` and `audit_behavior = 100010` in
+ `proto/v1/v1/annotation.proto`; 77 fields annotated across 17 service protos.
+ `OMIT` is defined as "must not be recorded, for any reason", which is the
+ reading the rest of this design assumes: it carries `User.phone`, `groups` and
+ `profile`, `MaskingReason.semantic_type_icon` and `AIChatRequest.messages`
+ alongside the bulk content fields, so deleting the ten rebuilds admits nothing
+ new.
+- **Redactor** — `backend/api/v1/audit_redact.go`. Plan per message type, cached
+ on descriptor identity, with `descend` arms holding the sub-descriptor rather
+ than a nested plan so a cycle is followed lazily to any depth; copy-and-share
+ by `Range`; maps and repeated messages rebuilt; `SENSITIVE` oneof arms blanked
+ and `OMIT` ones dropped. All 35 hand-written redactors are gone. With nothing
+ left to switch on, `getRequestString` and `getResponseString` collapse into one
+ `marshalAuditPayload`: the annotation says what to do with a field, and which
+ side of the call it arrived on does not change that. The hook point is
+ unchanged — it is still called from `createAuditLog`, not `WrapUnary`, which is
+ what keeps the streaming path covered. The row itself is **not** walked as a
+ whole: `service_data` and `status` are redacted where they are assigned, by
+ `redactAuditServiceData` and `redactAuditStatus`, and `request_metadata` and
+ `mcp_delegation` are assigned untouched because nothing under them is
+ annotated. `TestAuditRowNeedsNoRedactionBeyondTheAnyPayloads` asserts that
+ stays true.
+- **`Any` registry** — `backend/api/v1/audit_redact.go`, keyed by
+ destination field rather than by type, so `Status.details` and `service_data`
+ permit different things. `TestLintAuditAnyFieldsAreRegistered` enforces the
+ descriptor half: a new `Any` field reaching the row must name what may be
+ packed in it. The other half — a new type packed into an existing field —
+ changes no descriptor, so it is caught at runtime instead, by the `slog.Warn`
+ `redactPackedAny` emits naming the type it dropped.
+- **Coverage** — the redactor owes the row two things, and the tests are split
+ on that line rather than by message type, and both halves live in
+ `audit_redact_test.go`. `TestAuditRedactionCoversEveryAnnotatedField`
+ sweeps 567 annotated field paths across 321 request/response types, one
+ message per field so every oneof arm is exercised on its own rather than only
+ the last-numbered one — that is the whole of "the secret leaves", so the
+ hand-written per-credential tables that used to assert it by example are gone.
+ `TestAuditRowKeepsItsSubstance` in the same file is the other half
+ and has to stay hand-written: the sweep asserts absence, so a redactor that
+ blanked the entire row would pass it. Mutation-checked in both directions —
+ disabling redaction fails 91 subtests, replacing the output with an empty
+ message fails all 13 survival rows and leaves the sweep green.
+- **Inventory** — `audit_redact_inventory_test.go` holds the declaration lints,
+ as against `audit_redact_test.go` which tests the code: 866 recorded
+ `(message, string-or-bytes field)` pairs. Annotated fields are absent by
+ design, so the list reads as "what the audit row writes down" and a new
+ credential lands in it as one line of diff.
+- **Read path** — `assertNoInputOnlyValues` is generalized to
+ `assertNoSensitiveValues` (INPUT_ONLY ∪ SENSITIVE, traversing map values) and
+ runs unconditionally on the converters a test can construct. A static
+ call-graph lint over every `v1pb`-producing function in `backend/api/v1` was
+ prototyped and dropped: the read path is a different surface from the audit
+ row, and a Go AST check that reports a credential leak when a converter is
+ merely *renamed* is the wrong price to pay inside this change. If we want it,
+ it belongs in its own PR.
+- **Kept, not replaced** — `mcpDenialRequestsUnderReview` stays alongside the
+ inventory. It is per `(method, field)` and covers the message-typed request
+ fields the inventory's scalar scope does not: `DiffMetadata.target_metadata`
+ is a whole schema, and no string field names it.
+- **Deliberately not fail-open** — `AWSCredential.role_arn` and `external_id`
+ are annotated `SENSITIVE`. They are not credentials, and the Redaction section
+ above notes that deleting `redactIAMExtension` would start recording them;
+ annotating them keeps today's behavior instead of taking that trade silently.
+- **Widened on purpose, recorded here rather than discovered later** —
+ `redactDataSource` used to replace the whole `external_secret` submessage with
+ an empty one; only its credential leaves are annotated, so
+ `DataSourceExternalSecret`'s `url`, `engine_name`, `secret_name`,
+ `password_key_name`, `mount_path` and the type enums now reach the row. Those
+ say WHERE a secret lives, not what it is, and knowing which Vault path an
+ instance was pointed at is the kind of thing the row exists for — but a
+ customer's internal Vault endpoint and secret path are newly readable by
+ anyone with `bb.auditLogs.search`, so this is a sign-off, not a side effect.
+- **`QueryResult.messages` is `OMIT`** — the deleted `redactQueryResponse` was
+ an allowlist that predated the field, so it dropped it by omission. Engine
+ output (`RAISE NOTICE`, `PRINT`, `DBMS_OUTPUT`) is a result-data channel like
+ `rows`, and column masking does not reach it.
+- **`v1.AuditLog.service_data` is `OMIT`** — otherwise SearchAuditLogs' own
+ audit row unpacks, redacts and re-transcribes the before-image of every row
+ the caller read, which is the one place cost scaled with payload size.
+- **A SENSITIVE scalar with explicit presence is blanked, not dropped.** The
+ Redaction section above argues for blanking a oneof arm so the row still
+ records which credential was supplied; a proto3 `optional` credential is the
+ same question, and `LoginRequest.otp_code` is the case that matters — a failed
+ login has to stay distinguishable from one where no second factor was
+ presented. Non-optional scalars are still dropped, which protojson renders
+ identically.
+- **Still uncovered** — `Status.message`, exactly as the Redaction section says.
+ `idp_service.go:283` still interpolates an oauth2 error whose `Error()` can
+ embed the IdP's raw response body. It has no descriptor, so nothing here
+ reaches it; fixing it means deciding what that error should say instead.