Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(api): redact audit payloads from proto annotations (BYT-10090) (#21226)
Danny Xu37339d73c8a7
diff --git a/backend/api/v1/audit.go b/backend/api/v1/audit.goindex f3bb93b7a4..7deeaddbb9 100644--- a/backend/api/v1/audit.go+++ b/backend/api/v1/audit.go@@ -11,7 +11,6 @@ import ("connectrpc.com/connect""google.golang.org/protobuf/encoding/protojson""google.golang.org/protobuf/proto"- "google.golang.org/protobuf/reflect/protoreflect""google.golang.org/protobuf/types/known/anypb""google.golang.org/protobuf/types/known/durationpb"@@ -29,15 +28,14 @@ import ("github.com/bytebase/bytebase/backend/store")-// used for replacing sensitive fields.-var (- maskedString string-)-const (// maxAuditPayloadChars is the maximum characters for request/response payloads in stdout logs.// Set to 100KB (102400 chars) to match AWS CloudTrail industry standard for audit logs.maxAuditPayloadChars = 102400++ // unencodablePayload stands in for a payload protojson refused, so the row+ // records that something was there and could not be written down.+ unencodablePayload = `{"_bytebase":"payload could not be encoded"}`)// AuditInterceptor is the v1 audit interceptor for gRPC server.@@ -88,10 +86,12 @@ func (in *AuditInterceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc// and a refused agent is interesting either way. Only the internal MCP// chain runs the gate, so the public chain is unaffected.//- // Recording a request that was never recorded is why getRequestString- // grew redactors below: a denial must not transcribe the secret it- // refused. The rule for a new one is the population above, not the four- // named methods.+ // Recording a request that was never recorded is why redaction has to+ // cover more than the audited RPCs: a denial must not transcribe the+ // secret it refused. Since redaction is driven by the field annotation+ // rather than by a per-RPC redactor, a gate-refused method is covered+ // the moment its fields are annotated — the population is the one above,+ // not the four named methods.mcpPolicyDenied := falsectx = common.WithSetMCPPolicyDenied(ctx, func() { mcpPolicyDenied = true })@@ -237,14 +237,8 @@ func (in *AuditInterceptor) createAuditLog(ctx context.Context, e *auditEntry) ereturn nil}- requestString, err := getRequestString(e.request)- if err != nil {- return errors.Wrapf(err, "failed to get request string")- }- responseString, err := getResponseString(e.response)- if err != nil {- return errors.Wrapf(err, "failed to get response string")- }+ requestString := marshalAuditPayload(e.request)+ responseString := marshalAuditPayload(e.response)var user stringif u, ok := GetUserFromContext(ctx); ok {@@ -349,6 +343,23 @@ func (in *AuditInterceptor) createAuditLog(ctx context.Context, e *auditEntry) e}}+ // service_data and status.details are the only fields on the row that need+ // redacting, and neither passes marshalAuditPayload. Both carry+ // google.protobuf.Any values whose packed type the descriptor cannot see:+ // service_data holds a before-image Setting or an IAM policy delta, and+ // status.details holds whatever a failing handler attached.+ //+ // The rest of the row is left alone deliberately. request_metadata and+ // mcp_delegation are ordinary store messages with nothing annotated under+ // them, and request and response are already redacted strings.+ // TestAuditRowNeedsNoRedactionBeyondTheAnyPayloads fails if that stops+ // being true.+ //+ // Computed once: none of the three varies by parent.+ serviceData := redactAuditServiceData(e.serviceData)+ auditStatus := redactAuditStatus(convertErrToStatus(e.rerr))+ mcpDelegation := mcpDelegationFromAuthContext(authContext)+createAuditLogCtx := context.WithoutCancel(ctx)for _, ap := range parents {resource := getRequestResource(e.request, e.method)@@ -373,11 +384,11 @@ func (in *AuditInterceptor) createAuditLog(ctx context.Context, e *auditEntry) eUser: user,Request: requestString,Response: responseString,- Status: convertErrToStatus(e.rerr),+ Status: auditStatus,Latency: durationpb.New(e.latency),- ServiceData: e.serviceData,+ ServiceData: serviceData,RequestMetadata: requestMetadata,- McpDelegation: mcpDelegationFromAuthContext(authContext),+ McpDelegation: mcpDelegation,}// Resolve workspace for audit log.workspaceIDForAudit := ap.auditWorkspaceID@@ -469,7 +480,7 @@ func logAuditToStdout(ctx context.Context, p *storepb.AuditLog) {attrs = append(attrs, mcpDelegationAttrs(p.McpDelegation)...)// Include request payload (truncated to 100KB for log manageability)- // Request is already redacted for sensitive data by getRequestString()+ // Request is already redacted for sensitive data by marshalAuditPayload()if p.Request != "" {request := p.Requestif truncated, wasTruncated := common.TruncateString(p.Request, maxAuditPayloadChars); wasTruncated {@@ -479,7 +490,7 @@ func logAuditToStdout(ctx context.Context, p *storepb.AuditLog) {}// Include response payload (truncated to 100KB for log manageability)- // Response is already redacted for sensitive data by getResponseString()+ // Response is already redacted for sensitive data by marshalAuditPayload()if p.Response != "" {response := p.Responseif truncated, wasTruncated := common.TruncateString(p.Response, maxAuditPayloadChars); wasTruncated {@@ -563,843 +574,44 @@ func getRequestResource(request any, method string) string {return getResourceFromSingleRequest(message.ProtoReflect(), shortMethod)}-func getRequestString(request any) (string, error) {- m := func() protoreflect.ProtoMessage {- if request == nil || reflect.ValueOf(request).IsNil() {- return nil- }- if redacted, ok := redactMCPDenialRequest(request); ok {- return redacted- }- switch r := request.(type) {- case *v1pb.ExportRequest:- return redactExportRequest(r)- case *v1pb.CreateUserRequest:- return redactCreateUserRequest(r)- case *v1pb.UpdateUserRequest:- return redactUpdateUserRequest(r)- case *v1pb.LoginRequest:- return redactLoginRequest(r)- case *v1pb.SignupRequest:- return redactSignupRequest(r)- case *v1pb.ExchangeTokenRequest:- return redactExchangeTokenRequest(r)- case *v1pb.ResetPasswordRequest:- return redactResetPasswordRequest(r)- case *v1pb.UploadLicenseRequest:- return redactUploadLicenseRequest(r)- case *v1pb.CreateProjectRequest:- r = proto.CloneOf(r)- r.Project = redactProject(r.Project)- return r- case *v1pb.UpdateProjectRequest:- r = proto.CloneOf(r)- r.Project = redactProject(r.Project)- return r- case *v1pb.AddWebhookRequest:- r = proto.CloneOf(r)- r.Webhook = redactWebhook(r.Webhook)- return r- case *v1pb.UpdateWebhookRequest:- r = proto.CloneOf(r)- r.Webhook = redactWebhook(r.Webhook)- return r- case *v1pb.RemoveWebhookRequest:- r = proto.CloneOf(r)- r.Webhook = redactWebhook(r.Webhook)- return r- case *v1pb.CreateReleaseRequest:- r = proto.CloneOf(r)- r.Release = redactRelease(r.Release)- return r- case *v1pb.UpdateReleaseRequest:- r = proto.CloneOf(r)- r.Release = redactRelease(r.Release)- return r- case *v1pb.CreateSavedQueryRequest:- r = proto.CloneOf(r)- r.SavedQuery = redactSavedQuery(r.SavedQuery)- return r- case *v1pb.UpdateSavedQueryRequest:- r = proto.CloneOf(r)- r.SavedQuery = redactSavedQuery(r.SavedQuery)- return r-- case *v1pb.CreateInstanceRequest:- r = proto.CloneOf(r)- r.Instance = redactInstance(r.Instance)- return r- case *v1pb.UpdateInstanceRequest:- r = proto.CloneOf(r)- r.Instance = redactInstance(r.Instance)- return r- case *v1pb.BatchUpdateInstancesRequest:- r = proto.CloneOf(r)- for _, ur := range r.Requests {- ur.Instance = redactInstance(ur.Instance)- }- return r- case *v1pb.AddDataSourceRequest:- r = proto.CloneOf(r)- r.DataSource = redactDataSource(r.DataSource)- return r- case *v1pb.UpdateDataSourceRequest:- r = proto.CloneOf(r)- r.DataSource = redactDataSource(r.DataSource)- return r- case *v1pb.RemoveDataSourceRequest:- r = proto.CloneOf(r)- r.DataSource = redactDataSource(r.DataSource)- return r- case *v1pb.UpdateSettingRequest:- r = proto.CloneOf(r)- r.Setting = redactSetting(r.Setting)- return r- case *v1pb.CreateIdentityProviderRequest:- r = proto.CloneOf(r)- r.IdentityProvider = redactIdentityProvider(r.IdentityProvider)- return r- case *v1pb.CreateSheetRequest:- // The clone is already private; drop the content in place instead- // of cloning the potentially large sheet a second time.- r = proto.CloneOf(r)- if r.Sheet != nil {- r.Sheet.Content = nil- }- return r- case *v1pb.BatchCreateSheetsRequest:- r = proto.CloneOf(r)- for _, cr := range r.Requests {- if cr.Sheet != nil {- cr.Sheet.Content = nil- }- }- return r- case *v1pb.UpdateIdentityProviderRequest:- r = proto.CloneOf(r)- r.IdentityProvider = redactIdentityProvider(r.IdentityProvider)- return r- default:- if p, ok := r.(protoreflect.ProtoMessage); ok {- return p- }- return nil- }- }()- if m == nil {- return "", nil- }-- b, err := protojson.Marshal(m)- if err != nil {- return "", err- }- return string(b), nil-}--// redactMCPDenialRequest handles the requests that reach the audit log only-// because MCP denials are recorded. The methods themselves are unaudited; what-// arrives here is a refusal, and a refusal that transcribed the secret it-// refused would be worse than the silence it replaced. Every secret masked here-// is already masked on a sibling method that has always been audited.-//-// It is a function rather than six more arms of getRequestString's switch-// because it is one population with one reason for existing, and that reason is-// worth stating once where a reader can find it.-//-// Which six is not a judgment to repeat by eye — the first pass of it missed-// ListInstanceDatabase, whose request carries a whole Instance and therefore-// every data-source credential the product has. The population is derived from-// the descriptors instead, and TestLintDenialRequestsAreReviewedForRedaction-// fails when a method joins it.-//-// The bool reports whether this was one of them, so an unrelated request falls-// through to the main switch untouched.-func redactMCPDenialRequest(request any) (protoreflect.ProtoMessage, bool) {- switch r := request.(type) {- case *v1pb.TestWebhookRequest:- r = proto.CloneOf(r)- r.Webhook = redactWebhook(r.Webhook)- return r, true- case *v1pb.TestEmailSettingRequest:- r = proto.CloneOf(r)- r.EmailSetting = redactEmailSetting(r.EmailSetting)- return r, true- case *v1pb.TestIdentityProviderRequest:- return redactTestIdentityProviderRequest(r), true- case *v1pb.SwitchWorkspaceRequest:- return redactSwitchWorkspaceRequest(r), true- case *v1pb.AIChatRequest:- return redactAIChatRequest(r), true- case *v1pb.ListInstanceDatabaseRequest:- // The request carries a whole Instance — "we need to set this- // field if the target instance is not created yet" — so it holds- // every data-source credential: password, ssl_key,- // ssh_private_key, the Kerberos keytab, the AWS/Azure/GCP- // credentials, the external-secret token and master_password.- // redactInstance is the same helper CreateInstance and- // UpdateInstance already use.- r = proto.CloneOf(r)- r.Instance = redactInstance(r.Instance)- return r, true- default:- return nil, false- }-}--func getResponseString(response any) (string, error) {- m := func() protoreflect.ProtoMessage {- if response == nil || reflect.ValueOf(response).IsNil() {- return nil- }- switch r := response.(type) {- case *v1pb.QueryResponse:- return redactQueryResponse(r)- case *v1pb.AdminExecuteResponse:- return redactAdminExecuteResponse(r)- case *v1pb.ExportResponse:- return redactExportResponse(r)- case *v1pb.LoginResponse:- return redactLoginResponse(r)- case *v1pb.ExchangeTokenResponse:- return redactExchangeTokenResponse(r)- case *v1pb.RotateDirectorySyncTokenResponse:- return redactRotateDirectorySyncTokenResponse(r)- case *v1pb.ServiceAccount:- return redactServiceAccount(r)- case *v1pb.Setting:- return redactSetting(r)- case *v1pb.IdentityProvider:- return redactIdentityProvider(r)- case *v1pb.User:- return redactUser(r)- case *v1pb.Instance:- return redactInstance(r)- case *v1pb.Project:- return redactProject(r)- case *v1pb.Release:- return redactRelease(r)- case *v1pb.SavedQuery:- return redactSavedQuery(r)- case *v1pb.PurchaseResponse:- return redactPurchaseResponse(r)- case *v1pb.ExportVCSProviderUsersResponse:- return redactExportVCSProviderUsersResponse(r)- case *v1pb.ExportAuditLogsResponse:- return redactExportAuditLogsResponse(r)- case *v1pb.Sheet:- return redactSheet(r)- case *v1pb.BatchCreateSheetsResponse:- n := &v1pb.BatchCreateSheetsResponse{}- for _, sheet := range r.Sheets {- n.Sheets = append(n.Sheets, redactSheet(sheet))- }- return n- case *v1pb.BatchUpdateInstancesResponse:- n := &v1pb.BatchUpdateInstancesResponse{}- for _, instance := range r.Instances {- n.Instances = append(n.Instances, redactInstance(instance))- }- return n- default:- if p, ok := r.(protoreflect.ProtoMessage); ok {- return p- }- return nil- }- }()- if m == nil {- return "", nil- }- b, err := protojson.Marshal(m)- if err != nil {- return "", err- }- return string(b), nil-}--func redactResetPasswordRequest(r *v1pb.ResetPasswordRequest) *v1pb.ResetPasswordRequest {- if r == nil {- return nil- }- cloned := proto.CloneOf(r)- cloned.Code = maskedString- cloned.NewPassword = maskedString- return cloned-}--func redactUploadLicenseRequest(r *v1pb.UploadLicenseRequest) *v1pb.UploadLicenseRequest {- if r == nil {- return nil- }- return &v1pb.UploadLicenseRequest{License: maskedString}-}--func redactPurchaseResponse(r *v1pb.PurchaseResponse) *v1pb.PurchaseResponse {- if r == nil {- return nil- }- return &v1pb.PurchaseResponse{}-}--func redactExportVCSProviderUsersResponse(r *v1pb.ExportVCSProviderUsersResponse) *v1pb.ExportVCSProviderUsersResponse {- if r == nil {- return nil- }- return &v1pb.ExportVCSProviderUsersResponse{}-}--func redactExportAuditLogsResponse(r *v1pb.ExportAuditLogsResponse) *v1pb.ExportAuditLogsResponse {- if r == nil {- return nil- }- return &v1pb.ExportAuditLogsResponse{NextPageToken: r.NextPageToken}-}--func redactExportRequest(r *v1pb.ExportRequest) *v1pb.ExportRequest {- if r == nil {- return nil- }- r = proto.CloneOf(r)- if r.Password != "" {- r.Password = maskedString- }- return r-}--// redactExportResponse drops the exported file content but keeps the applied-// access grant so audit logs record which grant authorized the export.-func redactExportResponse(r *v1pb.ExportResponse) *v1pb.ExportResponse {- if r == nil || r.AppliedAccessGrant == "" {- return nil- }- return &v1pb.ExportResponse{- AppliedAccessGrant: r.AppliedAccessGrant,- }-}--func redactLoginRequest(r *v1pb.LoginRequest) *v1pb.LoginRequest {- if r == nil {- return nil- }-- // Clone to avoid mutating original- r = proto.CloneOf(r)-- // Mask sensitive fields.- if r.Password != "" {- r.Password = maskedString- }- maskOptionalString(&r.OtpCode)- maskOptionalString(&r.RecoveryCode)- maskOptionalString(&r.MfaTempToken)- maskOptionalString(&r.EmailCode)- if r.IdpContext != nil {- r.IdpContext = nil- }- return r-}--func redactSignupRequest(r *v1pb.SignupRequest) *v1pb.SignupRequest {- if r == nil {- return nil- }- r = proto.CloneOf(r)- if r.Password != "" {- r.Password = maskedString- }- return r-}--// redactExchangeTokenRequest masks the external OIDC JWT. The token is a-// credential — it could be replayed against the original IdP or, if logged,-// reveal workload identity claims. The caller's email is kept for audit-// correlation.-func redactExchangeTokenRequest(r *v1pb.ExchangeTokenRequest) *v1pb.ExchangeTokenRequest {- if r == nil {- return nil- }- r = proto.CloneOf(r)- if r.Token != "" {- r.Token = maskedString- }- return r-}--// redactExchangeTokenResponse drops the issued Bytebase access token. Logging-// it would give anyone with audit-log read access a valid API token for the-// workload identity. Returns an empty response so the audit entry still-// records that the call happened.-func redactExchangeTokenResponse(r *v1pb.ExchangeTokenResponse) *v1pb.ExchangeTokenResponse {- if r == nil {- return nil- }- return &v1pb.ExchangeTokenResponse{}-}--// redactRotateDirectorySyncTokenResponse drops the newly minted SCIM token. The-// whole point of hashing it at rest is that the plaintext exists only in the-// single response to the admin who rotated it; writing it to the audit log would-// hand a working credential to anyone who can read that log. Returns an empty-// response so the audit entry still records that the rotation happened.-func redactRotateDirectorySyncTokenResponse(r *v1pb.RotateDirectorySyncTokenResponse) *v1pb.RotateDirectorySyncTokenResponse {- if r == nil {- return nil- }- return &v1pb.RotateDirectorySyncTokenResponse{}-}--// redactServiceAccount drops the API key. Create and key rotation are the only-// responses that carry it — the read path never populates it — and it is a live-// credential, so logging it would hand a working key to anyone who can read the-// audit log or its stdout stream.-func redactServiceAccount(r *v1pb.ServiceAccount) *v1pb.ServiceAccount {- if r == nil {- return nil- }- r = proto.CloneOf(r)- if r.ServiceKey != "" {- r.ServiceKey = maskedString- }- return r-}--// redactIdentityProvider masks the IdP credentials. The read path already blanks-// these (convertToIdentityProvider), so they only reach the audit log through-// the create/update request payload.-func redactIdentityProvider(r *v1pb.IdentityProvider) *v1pb.IdentityProvider {- if r == nil {- return nil- }- r = proto.CloneOf(r)- switch config := r.GetConfig().GetConfig().(type) {- case *v1pb.IdentityProviderConfig_Oauth2Config:- if config.Oauth2Config.GetClientSecret() != "" {- config.Oauth2Config.ClientSecret = maskedString- }- case *v1pb.IdentityProviderConfig_OidcConfig:- if config.OidcConfig.GetClientSecret() != "" {- config.OidcConfig.ClientSecret = maskedString- }- case *v1pb.IdentityProviderConfig_LdapConfig:- if config.LdapConfig.GetBindPassword() != "" {- config.LdapConfig.BindPassword = maskedString- }- default:- }- return r-}--// redactEmailSetting masks the SMTP password, the same field redactSetting-// masks when the same value arrives through UpdateSetting.-func redactEmailSetting(e *v1pb.EmailSetting) *v1pb.EmailSetting {- if e == nil {- return nil- }- e = proto.CloneOf(e)- if smtp := e.GetSmtp(); smtp.GetPassword() != "" {- smtp.Password = maskedString- }- return e-}--// redactTestIdentityProviderRequest masks both halves of a connection test: the-// provider's own client secret or LDAP bind password, which redactIdentityProvider-// already masks on Create and Update, and the credential supplied to run the-// test — an authorization code, or a directory user's password. Each arm checks-// the wrapped message: a oneof wrapper can be set with a nil payload, and this-// runs on the audit path, where a panic would cost the row.-func redactTestIdentityProviderRequest(r *v1pb.TestIdentityProviderRequest) *v1pb.TestIdentityProviderRequest {- if r == nil {- return nil- }- r = proto.CloneOf(r)- r.IdentityProvider = redactIdentityProvider(r.IdentityProvider)- switch context := r.GetContext().(type) {- case *v1pb.TestIdentityProviderRequest_Oauth2Context:- if context.Oauth2Context != nil {- context.Oauth2Context.Code = maskedString- }- case *v1pb.TestIdentityProviderRequest_OidcContext:- if context.OidcContext != nil {- context.OidcContext.Code = maskedString- }- case *v1pb.TestIdentityProviderRequest_LdapContext:- if context.LdapContext != nil {- context.LdapContext.Password = maskedString- }- default:- }- return r-}--// redactSwitchWorkspaceRequest masks the MFA proofs, the same three fields-// redactLoginRequest masks when they arrive on Login.-func redactSwitchWorkspaceRequest(r *v1pb.SwitchWorkspaceRequest) *v1pb.SwitchWorkspaceRequest {- if r == nil {- return nil- }- r = proto.CloneOf(r)- maskOptionalString(&r.OtpCode)- maskOptionalString(&r.RecoveryCode)- maskOptionalString(&r.MfaTempToken)- return r-}--// redactAIChatRequest drops the conversation instead of masking a field. The-// request has no secret in it; it has an unbounded body — every message the-// caller sent, plus the tool definitions — and the audit row stores it whole-// (the size cap applies to the stdout logger only). AIService/Chat is EXCLUDED-// and unaudited, so the only rows it ever produces are the gate's denials, and-// a denial needs the fact of the call, not its transcript.-func redactAIChatRequest(r *v1pb.AIChatRequest) *v1pb.AIChatRequest {- if r == nil {- return nil- }- return &v1pb.AIChatRequest{}-}--// maskOptionalString masks an optional string field in place when it is set,-// so the several credential fields spelled `optional string` mask the same way-// wherever they appear.-func maskOptionalString(field **string) {- if *field != nil {- *field = &maskedString- }-}--func redactWebhook(w *v1pb.Webhook) *v1pb.Webhook {- if w == nil {- return nil- }- cloned := proto.CloneOf(w)- if cloned.Url != "" {- cloned.Url = maskedString- }- return cloned-}--func redactProject(p *v1pb.Project) *v1pb.Project {- if p == nil {- return nil- }- cloned := proto.CloneOf(p)- for i, webhook := range cloned.Webhooks {- cloned.Webhooks[i] = redactWebhook(webhook)- }- return cloned-}--func redactRelease(r *v1pb.Release) *v1pb.Release {- if r == nil {- return nil- }- cloned := proto.CloneOf(r)- for _, file := range cloned.Files {- if file != nil {- file.Statement = nil- }- }- return cloned-}--func redactSavedQuery(r *v1pb.SavedQuery) *v1pb.SavedQuery {- if r == nil {- return nil- }- cloned := proto.CloneOf(r)- cloned.Content = nil- return cloned-}--// redactSetting masks every credential a settings payload can carry. The read-// path blanks these, so they reach the audit log only through UpdateSetting's-// request. Each secret is masked rather than dropped so the log still records-// that the field was being written.-func redactSetting(r *v1pb.Setting) *v1pb.Setting {- if r == nil {- return nil- }- r = proto.CloneOf(r)- switch value := r.GetValue().GetValue().(type) {- case *v1pb.SettingValue_Email:- if smtp := value.Email.GetSmtp(); smtp.GetPassword() != "" {- smtp.Password = maskedString- }- case *v1pb.SettingValue_Ai:- if value.Ai.GetApiKey() != "" {- value.Ai.ApiKey = maskedString- }- case *v1pb.SettingValue_AppIm:- maskAppIMSecrets(value.AppIm)- default:- }- return r-}--func maskAppIMSecrets(s *v1pb.AppIMSetting) {- for _, setting := range s.GetSettings() {- if v := setting.GetSlack(); v.GetToken() != "" {- v.Token = maskedString- }- if v := setting.GetFeishu(); v.GetAppSecret() != "" {- v.AppSecret = maskedString- }- if v := setting.GetWecom(); v.GetSecret() != "" {- v.Secret = maskedString- }- if v := setting.GetLark(); v.GetAppSecret() != "" {- v.AppSecret = maskedString- }- if v := setting.GetDingtalk(); v.GetClientSecret() != "" {- v.ClientSecret = maskedString- }- if v := setting.GetTeams(); v.GetClientSecret() != "" {- v.ClientSecret = maskedString- }- }-}--func redactCreateUserRequest(r *v1pb.CreateUserRequest) *v1pb.CreateUserRequest {- if r == nil {- return nil- }- return &v1pb.CreateUserRequest{- User: redactUser(r.User),- }-}--func redactUpdateUserRequest(r *v1pb.UpdateUserRequest) *v1pb.UpdateUserRequest {- if r == nil {- return nil- }- return &v1pb.UpdateUserRequest{- User: redactUser(r.User),- UpdateMask: r.UpdateMask,- OtpCode: r.OtpCode,- RegenerateTempMfaSecret: r.RegenerateTempMfaSecret,- RegenerateRecoveryCodes: r.RegenerateRecoveryCodes,- }-}--func redactUser(r *v1pb.User) *v1pb.User {- if r == nil {- return nil- }- return &v1pb.User{- Name: r.Name,- Email: r.Email,- Title: r.Title,- }-}--// redactSheet strips the sheet content from audit payloads. Sheets carry full-// SQL statements (potentially megabytes); the audit record keeps the resource-// name and content size.-func redactSheet(s *v1pb.Sheet) *v1pb.Sheet {- if s == nil {- return nil- }- cloned := proto.CloneOf(s)- cloned.Content = nil- return cloned-}--func redactInstance(i *v1pb.Instance) *v1pb.Instance {- if i == nil {- return nil- }- // Clone the instance to avoid modifying the original response- cloned := proto.CloneOf(i)- var dataSources []*v1pb.DataSource- for _, d := range cloned.DataSources {- dataSources = append(dataSources, redactDataSource(d))- }- cloned.DataSources = dataSources- return cloned-}--func redactDataSource(d *v1pb.DataSource) *v1pb.DataSource {- if d == nil {- return nil- }- // Clone the datasource to avoid modifying the original- cloned := proto.CloneOf(d)- if cloned.Password != "" {- cloned.Password = maskedString- }- if cloned.SslCa != "" {- cloned.SslCa = maskedString- }- if cloned.SslCaPath != "" {- cloned.SslCaPath = maskedString- }- if cloned.SslCert != "" {- cloned.SslCert = maskedString- }- if cloned.SslCertPath != "" {- cloned.SslCertPath = maskedString- }- if cloned.SslKey != "" {- cloned.SslKey = maskedString- }- if cloned.SslKeyPath != "" {- cloned.SslKeyPath = maskedString- }- if cloned.SshPassword != "" {- cloned.SshPassword = maskedString- }- if cloned.SshPrivateKey != "" {- cloned.SshPrivateKey = maskedString- }- if cloned.AuthenticationPrivateKey != "" {- cloned.AuthenticationPrivateKey = maskedString- }- if cloned.AuthenticationPrivateKeyPassphrase != "" {- cloned.AuthenticationPrivateKeyPassphrase = maskedString- }- redactIAMExtension(cloned)- if cloned.ExternalSecret != nil {- cloned.ExternalSecret = new(v1pb.DataSourceExternalSecret)- }- if cloned.SaslConfig != nil {- if krbConf := cloned.SaslConfig.GetKrbConfig(); krbConf != nil {- krbConf.Keytab = []byte(maskedString)- cloned.SaslConfig.Mechanism = &v1pb.SASLConfig_KrbConfig{KrbConfig: krbConf}- }- }- if cloned.MasterPassword != "" {- cloned.MasterPassword = maskedString- }- return cloned-}--// redactIAMExtension strips the IAM credential while keeping the variant that-// was set, so the row still records which credential type the caller supplied.-// It keys off the oneof rather than authentication_type, which a request is-// free to leave unset or contradict.+// marshalAuditPayload renders one side of an audited call. Redaction is driven+// by the (bytebase.v1.audit_behavior) annotation on the message's own+// descriptor — see audit_redact.go — so nothing here switches on a type, and a+// field is protected on every RPC that carries it rather than on the ones+// someone wrote a redactor for.//-// What survives mirrors the read path (instance_service_converter.go): the-// Azure tenant and client IDs name the principal and are not INPUT_ONLY, so-// they stay; every AWSCredential field is, including the role ARN, so none-// does; and the GCP content is a whole service-account key.-//-// Keeping the ARN was considered — it names a principal rather than-// authenticating as one, and the row could then answer which role a data-// source was pointed at. It is dropped anyway, because "no INPUT_ONLY field-// survives redaction" is an invariant a descriptor walk can check forever-// (TestAuditRedactsEveryInputOnlyDataSourceField), and a hand-kept exception-// list is the drift this function exists to stop.-func redactIAMExtension(d *v1pb.DataSource) {- switch extension := d.GetIamExtension().(type) {- case *v1pb.DataSource_AzureCredential_:- d.IamExtension = &v1pb.DataSource_AzureCredential_{- AzureCredential: &v1pb.DataSource_AzureCredential{- TenantId: extension.AzureCredential.GetTenantId(),- ClientId: extension.AzureCredential.GetClientId(),- },- }- case *v1pb.DataSource_AwsCredential:- d.IamExtension = &v1pb.DataSource_AwsCredential{- AwsCredential: &v1pb.DataSource_AWSCredential{},- }- case *v1pb.DataSource_GcpCredential:- d.IamExtension = &v1pb.DataSource_GcpCredential{- GcpCredential: &v1pb.DataSource_GCPCredential{},- }- default:- // A variant added to the oneof after this function was written is- // dropped whole rather than logged. Unset lands here too and stays nil.- d.IamExtension = nil- }-}--func redactAdminExecuteResponse(r *v1pb.AdminExecuteResponse) *v1pb.AdminExecuteResponse {- if r == nil {- return nil- }- n := &v1pb.AdminExecuteResponse{- Results: nil,- }- for _, result := range r.Results {- if result == nil {- n.Results = append(n.Results, &v1pb.QueryResult{})- continue- }- n.Results = append(n.Results, &v1pb.QueryResult{- ColumnNames: result.ColumnNames,- ColumnTypeNames: result.ColumnTypeNames,- Rows: nil, // Redacted- Error: result.Error,- Latency: result.Latency,- Statement: result.Statement,- DetailedError: result.DetailedError,- Masked: redactMaskingReasons(result.Masked), // Redact icon data- })- }-- return n-}--func redactQueryResponse(r *v1pb.QueryResponse) *v1pb.QueryResponse {- if r == nil {- return nil- }- n := &v1pb.QueryResponse{- Results: nil,- AppliedAccessGrant: r.AppliedAccessGrant,- }- for _, result := range r.Results {- n.Results = append(n.Results, &v1pb.QueryResult{- ColumnNames: result.ColumnNames,- ColumnTypeNames: result.ColumnTypeNames,- Rows: nil, // Redacted- RowsCount: result.RowsCount,- Error: result.Error,- Latency: result.Latency,- Statement: result.Statement,- DetailedError: result.DetailedError,- Masked: redactMaskingReasons(result.Masked), // Redact icon data- })- }- return n-}--func redactMaskingReasons(reasons []*v1pb.MaskingReason) []*v1pb.MaskingReason {- if reasons == nil {- return nil- }- var redacted []*v1pb.MaskingReason- for _, reason := range reasons {- if reason == nil {- redacted = append(redacted, nil)- continue- }- redacted = append(redacted, &v1pb.MaskingReason{- SemanticTypeId: reason.SemanticTypeId,- SemanticTypeTitle: reason.SemanticTypeTitle,- MaskingRuleId: reason.MaskingRuleId,- Algorithm: reason.Algorithm,- Context: reason.Context,- ClassificationLevel: reason.ClassificationLevel,- // Omit SemanticTypeIcon to avoid polluting audit logs with base64 data- })- }- return redacted-}--func redactLoginResponse(r *v1pb.LoginResponse) *v1pb.LoginResponse {- if r == nil {- return nil+// One function for both directions: the annotation says what to do with a+// field, and which side of the call it arrived on does not change that. It is+// called from createAuditLog rather than from WrapUnary because the streaming+// path builds its own auditEntry and calls createAuditLog directly, so a walk+// that lived in the unary interceptor would skip AdminExecute entirely.+func marshalAuditPayload(payload any) string {+ if payload == nil {+ return ""}-- n := &v1pb.LoginResponse{- RequireResetPassword: r.RequireResetPassword,+ // IsNil panics on a kind that cannot be nil, and Receive/Send stash+ // whatever the stream hands them, so the kind is checked first.+ if value := reflect.ValueOf(payload); value.Kind() == reflect.Pointer && value.IsNil() {+ return ""}- if r.User != nil {- n.User = redactUser(r.User)+ message, ok := payload.(proto.Message)+ if !ok {+ return ""}- return n+ b, err := protojson.Marshal(redactForAudit(message))+ if err != nil {+ // A payload that will not encode must not cost the record. protojson+ // rejects invalid UTF-8, and QueryResult.error and .statement are+ // filled straight from the driver — so one bad byte from an Oracle or+ // MySQL connection in a non-UTF-8 charset used to mean either no audit+ // row at all (unary, where WrapUnary swallows the error) or a killed+ // AdminExecute stream. Degrade the payload instead; everything else on+ // the row, including the method and the outcome, still gets written.+ slog.Warn("audit: payload could not be encoded, recording a placeholder",+ log.BBError(err), slog.String("type", string(message.ProtoReflect().Descriptor().FullName())))+ return unencodablePayload+ }+ return string(b)}func needAudit(ctx context.Context) bool {diff --git a/backend/api/v1/audit_redact.go b/backend/api/v1/audit_redact.gonew file mode 100644index 0000000000..d92a92b753--- /dev/null+++ b/backend/api/v1/audit_redact.go@@ -0,0 +1,560 @@+package v1++import (+ "log/slog"+ "sync"++ "google.golang.org/protobuf/encoding/protowire"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/reflect/protoreflect"+ "google.golang.org/protobuf/reflect/protoregistry"+ "google.golang.org/protobuf/types/descriptorpb"+ "google.golang.org/protobuf/types/known/anypb"++ spb "google.golang.org/genproto/googleapis/rpc/status"++ "github.com/bytebase/bytebase/backend/common/log"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// Redaction is driven by the (bytebase.v1.audit_behavior) field option rather+// than by a per-RPC redactor. A field marked SENSITIVE or OMIT in the .proto is+// dropped wherever it appears, at any depth, on every request and response the+// audit interceptor records.+//+// It runs from createAuditLog rather than from WrapUnary because the streaming+// path builds its own auditEntry and calls createAuditLog directly, so a walk+// in the unary interceptor would skip AdminExecute — the one streaming RPC, and+// the one carrying every row of an admin-mode query.+//+// It is a denylist, so an unannotated field is logged. The inventory lint in+// audit_redact_inventory_test.go moves that failure to CI.++const anyFullName protoreflect.FullName = "google.protobuf.Any"++// The two Any-carrying fields on an audit row, redacted where they are+// assigned rather than by walking the row.+const (+ auditServiceDataField protoreflect.FullName = "bytebase.store.AuditLog.service_data"+ auditStatusDetailsField protoreflect.FullName = "google.rpc.Status.details"+)++// auditAnyRegistry names the types each Any field on an audit row may carry.+// An Any whose packed type is not listed for its field is DROPPED rather than+// logged: protojson.Marshal fails the entire row on an unresolvable Any, so+// keeping it would lose the record.+//+// The descriptor walk sees only type_url and value, so a field annotated inside+// a packed message is invisible to the redactor, the coverage sweep and the+// inventory alike. Registering a type is what pulls its fields into the+// inventory for classification.+//+// TestLintAuditAnyFieldsAreRegistered enforces the descriptor half: a new Any+// FIELD reaching the row must be registered. The other half — a new TYPE packed+// into an existing field — changes no descriptor, so nothing fails the build;+// redactPackedAny logs the dropped type at runtime instead.+var auditAnyRegistry = map[protoreflect.FullName][]protoreflect.FullName{+ // UpdateSetting's before-image and the locked re-capture in the+ // workspace-profile merge (setting_service.go:174, :658), plus SetIamPolicy+ // on projects and on the workspace (project_service.go:621,+ // workspace_service.go:509).+ //+ // Setting is the one that carries credentials; convertToSettingMessage+ // blanks its own, but that is a property of the call site rather than an+ // enforced one, which is why it is classified through the inventory like+ // any other field. v1.AuditLog.service_data is deliberately absent: it is+ // annotated OMIT, so SearchAuditLogs' own row does not re-transcribe the+ // before-images of every row the caller read.+ auditServiceDataField: {+ "bytebase.v1.Setting",+ "bytebase.v1.AuditData",+ },+ // Attached to a FAILED RPC by convertErrToStatus — exactly when a handler+ // adds context about what went wrong. PermissionDeniedDetail carries a+ // method, permissions and resource names; PlanCheckRun.Result carries+ // advisory text.+ auditStatusDetailsField: {+ "bytebase.v1.PermissionDeniedDetail",+ "bytebase.v1.PlanCheckRun.Result",+ },+}++// auditAnyPermits reports whether a packed type may be recorded in one Any+// field. An unregistered field permits nothing, so a new Any reaching the row+// before anyone classifies it drops its payload rather than writing it.+func auditAnyPermits(field, packed protoreflect.FullName) bool {+ for _, permitted := range auditAnyRegistry[field] {+ if permitted == packed {+ return true+ }+ }+ return false+}++// auditBehaviorFieldNumber is (bytebase.v1.audit_behavior)'s extension number,+// used to recognize the annotation when it survives only as an unknown field.+const auditBehaviorFieldNumber protowire.Number = 100010++type actionKind uint8++const (+ // actionDrop leaves the field out of the copy entirely.+ actionDrop actionKind = iota + 1+ // actionBlank copies the field with its zero value, keeping presence. Used+ // for SENSITIVE scalars that have explicit presence — a oneof arm or a+ // proto3 optional — so the row still records that a credential was+ // supplied. Clearing DataSourceExternalSecret.token would make token auth+ // indistinguishable from unconfigured while its AppRole sibling stayed+ // legible, and dropping LoginRequest.otp_code would make a failed login+ // with an OTP indistinguishable from one without.+ actionBlank+ // actionPacked resolves and redacts a google.protobuf.Any. The descriptor+ // walk sees only type_url and value, never the packed message.+ actionPacked+ // actionDescend rebuilds a subtree that contains something to redact.+ actionDescend+)++type fieldAction struct {+ kind actionKind+ // sub is the descriptor to descend into, set only for actionDescend. It is+ // a DESCRIPTOR rather than a plan so a recursive type resolves its own plan+ // at every depth: caching a plan value here would freeze a finite chain,+ // and an annotation below its end would be applied at the top and nowhere+ // else.+ sub protoreflect.MessageDescriptor+}++// redactPlan says what to do with one message type. A nil plan means the type+// holds nothing to redact anywhere beneath it, so a value of that type is+// shared by pointer and only read.+type redactPlan struct {+ fields map[protoreflect.FieldNumber]fieldAction+}++// planCache maps a message descriptor to its *redactPlan. Keyed on the+// descriptor itself, not on its full name: plans are applied by field NUMBER,+// so two distinct descriptors sharing a name would otherwise get each other's+// plan. A nil-valued entry is meaningful — it records that the type needs+// nothing.+var planCache sync.Map++// needsCache memoizes typeNeedsRedaction.+var needsCache sync.Map++// planFor returns the plan for a message type, building it on first use.+func planFor(descriptor protoreflect.MessageDescriptor) *redactPlan {+ if cached, ok := planCache.Load(descriptor); ok {+ if plan, isPlan := cached.(*redactPlan); isPlan {+ return plan+ }+ }++ var fields map[protoreflect.FieldNumber]fieldAction+ record := func(field protoreflect.FieldDescriptor, action fieldAction) {+ if fields == nil {+ fields = map[protoreflect.FieldNumber]fieldAction{}+ }+ fields[field.Number()] = action+ }++ declared := descriptor.Fields()+ for i := range declared.Len() {+ field := declared.Get(i)+ switch auditBehaviorOf(field) {+ case v1pb.AuditBehavior_SENSITIVE:+ // A blanked field needs a scalar zero value to stand in for the+ // removed one, and explicit presence for that value to survive+ // marshaling. Everything else is dropped.+ if isBlankable(field) && field.HasPresence() {+ record(field, fieldAction{kind: actionBlank})+ } else {+ record(field, fieldAction{kind: actionDrop})+ }+ continue+ case v1pb.AuditBehavior_OMIT:+ record(field, fieldAction{kind: actionDrop})+ continue+ default:+ }+ sub := submessageOf(field)+ if sub == nil {+ continue+ }+ if sub.FullName() == anyFullName {+ record(field, fieldAction{kind: actionPacked})+ continue+ }+ if typeNeedsRedaction(sub) {+ record(field, fieldAction{kind: actionDescend, sub: sub})+ }+ }++ var plan *redactPlan+ if fields != nil {+ plan = &redactPlan{fields: fields}+ }+ cached, _ := planCache.LoadOrStore(descriptor, plan)+ if stored, isPlan := cached.(*redactPlan); isPlan {+ return stored+ }+ return plan+}++// typeNeedsRedaction reports whether anything reachable from a message type has+// to be redacted. It is plain reachability with a visited set rather than a+// fixed point, which is what makes the four cyclic components in the descriptor+// graph terminate: ObjectSchema/StructKind/ArrayKind,+// google.protobuf.Value/Struct/ListValue, google.api.expr's Expr, and+// TablePartitionMetadata.subpartitions.+func typeNeedsRedaction(descriptor protoreflect.MessageDescriptor) bool {+ if cached, ok := needsCache.Load(descriptor); ok {+ if needs, isBool := cached.(bool); isBool {+ return needs+ }+ }+ needs := reachesRedaction(descriptor, map[protoreflect.MessageDescriptor]bool{})+ needsCache.Store(descriptor, needs)+ return needs+}++func reachesRedaction(descriptor protoreflect.MessageDescriptor, seen map[protoreflect.MessageDescriptor]bool) bool {+ if seen[descriptor] {+ return false+ }+ seen[descriptor] = true++ fields := descriptor.Fields()+ for i := range fields.Len() {+ field := fields.Get(i)+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {+ return true+ }+ sub := submessageOf(field)+ if sub == nil {+ continue+ }+ if sub.FullName() == anyFullName {+ return true+ }+ if reachesRedaction(sub, seen) {+ return true+ }+ }+ return false+}++// submessageOf returns the message descriptor a field leads to: the element+// type of a repeated field, the value type of a map, or the field's own type.+func submessageOf(field protoreflect.FieldDescriptor) protoreflect.MessageDescriptor {+ if field.IsMap() {+ if isMessageKind(field.MapValue().Kind()) {+ return field.MapValue().Message()+ }+ return nil+ }+ if isMessageKind(field.Kind()) {+ return field.Message()+ }+ return nil+}++func isMessageKind(kind protoreflect.Kind) bool {+ return kind == protoreflect.MessageKind || kind == protoreflect.GroupKind+}++func isBlankable(field protoreflect.FieldDescriptor) bool {+ return !field.IsList() && !field.IsMap() && !isMessageKind(field.Kind())+}++// auditBehaviorOf reads the annotation, failing CLOSED. A denylist that cannot+// read its own marker has to assume the field is a credential: the alternative+// is writing it out verbatim with no error and no log line.+func auditBehaviorOf(field protoreflect.FieldDescriptor) v1pb.AuditBehavior {+ options, ok := field.Options().(*descriptorpb.FieldOptions)+ if !ok {+ return v1pb.AuditBehavior_SENSITIVE+ }+ if options == nil {+ return v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED+ }+ if behavior, ok := proto.GetExtension(options, v1pb.E_AuditBehavior).(v1pb.AuditBehavior); ok &&+ behavior != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {+ return behavior+ }+ // The annotation survives as an UNKNOWN field when the descriptor was+ // unmarshaled by a resolver that does not know this extension — which is+ // reachable, since redactPackedAny resolves types out of a registry that+ // can hold runtime-registered files. Reading UNSPECIFIED there would log+ // the very field the annotation exists to protect.+ if unknownCarriesAuditBehavior(options) {+ return v1pb.AuditBehavior_SENSITIVE+ }+ return v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED+}++func unknownCarriesAuditBehavior(options *descriptorpb.FieldOptions) bool {+ unknown := options.ProtoReflect().GetUnknown()+ for len(unknown) > 0 {+ number, kind, tagLen := protowire.ConsumeTag(unknown)+ if tagLen < 0 {+ // Unparseable options: fail closed rather than guess.+ return true+ }+ if number == auditBehaviorFieldNumber {+ return true+ }+ valueLen := protowire.ConsumeFieldValue(number, kind, unknown[tagLen:])+ if valueLen < 0 {+ return true+ }+ unknown = unknown[tagLen+valueLen:]+ }+ return false+}++// redactForAudit returns the message to marshal into an audit payload. When+// nothing under the message needs redacting it returns the message itself —+// the common case, and what makes a 5 MB sheet free.+//+// Otherwise it returns a partial copy. Every message on the path to a redacted+// field is copied by Range — never by iterating Descriptor().Fields() and+// calling Set, which panics on an unset message, list or map field. Range+// yields only populated fields, which is also how a dropped field is never+// copied: it is skipped during the copy rather than cleared afterwards on a+// message that never held it. Every subtree with nothing to redact is shared by+// pointer and only read.+//+// Because subtrees are shared, the result is write-once: marshal it and discard+// it. Mutating it would write through to the caller's live message, which the+// handler is still about to return to the client.+func redactForAudit[M proto.Message](message M) M {+ reflected := message.ProtoReflect()+ if !reflected.IsValid() {+ return message+ }+ plan := planFor(reflected.Descriptor())+ if plan == nil {+ return message+ }+ redacted, ok := redactMessage(reflected, plan).Interface().(M)+ if !ok {+ // src.New() returns the same concrete type, so this cannot happen. If+ // a future runtime made it possible, an empty message of the right type+ // is the only safe answer: returning the input would log it unredacted,+ // and returning the zero value would hand a typed nil to callers that+ // dereference it.+ empty, isM := reflected.New().Interface().(M)+ if !isM {+ return message+ }+ return empty+ }+ return redacted+}++// redactMessage copies src, applying plan.+func redactMessage(src protoreflect.Message, plan *redactPlan) protoreflect.Message {+ dst := src.New()+ src.Range(func(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {+ action, ok := plan.fields[field.Number()]+ if !ok {+ dst.Set(field, value)+ return true+ }+ switch action.kind {+ case actionDrop:+ case actionBlank:+ dst.Set(field, field.Default())+ case actionPacked:+ copyAnyField(dst, field, value)+ case actionDescend:+ copyDescended(dst, field, value, action.sub)+ default:+ }+ return true+ })+ return dst+}++// copyDescended rebuilds the subtree under one field.+func copyDescended(dst protoreflect.Message, field protoreflect.FieldDescriptor, value protoreflect.Value, sub protoreflect.MessageDescriptor) {+ plan := planFor(sub)+ if plan == nil {+ dst.Set(field, value)+ return+ }+ switch {+ case field.IsMap():+ // dst.Set(field, value) on a map shares the Go map itself and its+ // message values, so redacting an entry would write through to the+ // caller's live message. Rebuild entry by entry.+ out := dst.Mutable(field).Map()+ value.Map().Range(func(key protoreflect.MapKey, entry protoreflect.Value) bool {+ out.Set(key, protoreflect.ValueOfMessage(redactMessage(entry.Message(), plan)))+ return true+ })+ case field.IsList():+ // Sharing the list cannot produce redacted elements without writing+ // through to the caller's, so it is rebuilt per element.+ // Instance.roles[].password and instance_resource.data_sources[] both+ // cross one of these — the first is the leak this design opens with.+ out := dst.Mutable(field).List()+ list := value.List()+ for i := range list.Len() {+ out.Append(protoreflect.ValueOfMessage(redactMessage(list.Get(i).Message(), plan)))+ }+ default:+ dst.Set(field, protoreflect.ValueOfMessage(redactMessage(value.Message(), plan)))+ }+}++// copyAnyField copies an Any-typed field, redacting each packed message and+// dropping any the registry does not permit for this field.+func copyAnyField(dst protoreflect.Message, field protoreflect.FieldDescriptor, value protoreflect.Value) {+ switch {+ case field.IsMap():+ out := dst.Mutable(field).Map()+ value.Map().Range(func(key protoreflect.MapKey, entry protoreflect.Value) bool {+ if redacted, ok := redactPackedAny(field.FullName(), entry.Message()); ok {+ out.Set(key, protoreflect.ValueOfMessage(redacted))+ }+ return true+ })+ case field.IsList():+ out := dst.Mutable(field).List()+ list := value.List()+ for i := range list.Len() {+ if redacted, ok := redactPackedAny(field.FullName(), list.Get(i).Message()); ok {+ out.Append(protoreflect.ValueOfMessage(redacted))+ }+ }+ default:+ if redacted, ok := redactPackedAny(field.FullName(), value.Message()); ok {+ dst.Set(field, protoreflect.ValueOfMessage(redacted))+ }+ }+}++// redactAuditServiceData redacts the Any a handler attached to the row through+// common.SetServiceData — an UpdateSetting before-image, whose Setting carries+// SMTP and AI credentials, or a SetIamPolicy delta. Returns nil when the packed+// type is not one the registry names for this field.+func redactAuditServiceData(serviceData *anypb.Any) *anypb.Any {+ if serviceData == nil {+ return nil+ }+ redacted, ok := redactPackedAny(auditServiceDataField, serviceData.ProtoReflect())+ if !ok {+ return nil+ }+ out, isAny := redacted.Interface().(*anypb.Any)+ if !isAny {+ return nil+ }+ return out+}++// redactAuditStatus rebuilds a failed RPC's status with only the details the+// registry permits.+//+// Not about secrecy: PermissionDeniedDetail and PlanCheckRun.Result carry+// nothing annotated. It is about keeping the row at all — protojson.Marshal+// fails the WHOLE AuditLog on an Any it cannot resolve, so one unrecognized+// detail from a wrapped upstream error would cost the entire record rather+// than just its own payload.+func redactAuditStatus(status *spb.Status) *spb.Status {+ if len(status.GetDetails()) == 0 {+ return status+ }+ details := make([]*anypb.Any, 0, len(status.GetDetails()))+ for _, detail := range status.GetDetails() {+ redacted, ok := redactPackedAny(auditStatusDetailsField, detail.ProtoReflect())+ if !ok {+ continue+ }+ if out, ok := redacted.Interface().(*anypb.Any); ok {+ details = append(details, out)+ }+ }+ if len(details) == len(status.GetDetails()) {+ return status+ }+ // Copied rather than edited: the caller's error is still in flight.+ return &spb.Status{Code: status.GetCode(), Message: status.GetMessage(), Details: details}+}++// redactPackedAny redacts one google.protobuf.Any. The bool reports whether it+// may be recorded at all: an Any this function cannot vouch for is DROPPED, not+// logged, because protojson.Marshal fails the ENTIRE audit row on an+// unresolvable one — keeping it would lose the record rather than the payload.+//+// Vouching means resolving the type through the SAME registry protojson will+// use and parsing the bytes, whether or not there is anything to redact.+// Checking only the types that happen to have a plan would leave the drop+// guarantee off exactly the types that do not — both Status.details types among+// them — so a corrupt payload there would take the whole row down.+//+// Losing a payload silently is its own defect, so a drop is logged.+func redactPackedAny(field protoreflect.FullName, packed protoreflect.Message) (protoreflect.Message, bool) {+ fields := packed.Descriptor().Fields()+ typeURLField := fields.ByName("type_url")+ valueField := fields.ByName("value")+ if typeURLField == nil || valueField == nil {+ return nil, false+ }+ typeURL := packed.Get(typeURLField).String()+ name := packedTypeName(typeURL)+ drop := func(reason string) (protoreflect.Message, bool) {+ slog.Warn("audit: dropped an Any from the audit row",+ slog.String("field", string(field)),+ slog.String("type", string(name)),+ slog.String("reason", reason))+ return nil, false+ }+ if !auditAnyPermits(field, name) {+ return drop("type is not in auditAnyRegistry for this field")+ }++ messageType, err := protoregistry.GlobalTypes.FindMessageByName(name)+ if err != nil {+ return drop("type is not in the global type registry")+ }+ inner := messageType.New().Interface()+ if err := proto.Unmarshal(packed.Get(valueField).Bytes(), inner); err != nil {+ return drop("packed bytes do not parse")+ }++ plan := planFor(messageType.Descriptor())+ if plan == nil {+ // Resolvable and parseable, with nothing to redact. Returned as found+ // so the type_url keeps whatever form the producer wrote: connect's+ // ErrorDetail.Type() returns a bare full name, and a round-trip through+ // anypb.New would rewrite it to type.googleapis.com/... and silently+ // change what SearchAuditLogs emits as @type.+ return packed, true+ }++ body, err := proto.Marshal(redactMessage(inner.ProtoReflect(), plan).Interface())+ if err != nil {+ slog.Warn("audit: dropped an Any from the audit row", log.BBError(err),+ slog.String("field", string(field)), slog.String("type", string(name)),+ slog.String("reason", "redacted message does not marshal"))+ return nil, false+ }+ out := packed.New()+ out.Set(typeURLField, protoreflect.ValueOfString(typeURL))+ out.Set(valueField, protoreflect.ValueOfBytes(body))+ return out, true+}++// packedTypeName strips the optional "type.googleapis.com/" prefix an Any's+// type_url may carry.+func packedTypeName(typeURL string) protoreflect.FullName {+ for i := len(typeURL) - 1; i >= 0; i-- {+ if typeURL[i] == '/' {+ return protoreflect.FullName(typeURL[i+1:])+ }+ }+ return protoreflect.FullName(typeURL)+}diff --git a/backend/api/v1/audit_redact_inventory_test.go b/backend/api/v1/audit_redact_inventory_test.gonew file mode 100644index 0000000000..c137870f4e--- /dev/null+++ b/backend/api/v1/audit_redact_inventory_test.go@@ -0,0 +1,1183 @@+package v1++import (+ "fmt"+ "slices"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/reflect/protoreflect"+ "google.golang.org/protobuf/reflect/protoregistry"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// The inventory is the oracle the coverage sweep cannot be. That sweep proves+// annotated fields get blanked; it cannot tell an unannotated credential from+// an ordinary field the row intentionally keeps, because both survive+// redaction identically.+//+// So every string and bytes field reachable from an audit payload is listed+// here, minus the annotated ones. A field missing from the list fails the+// build; clearing that failure means annotating it or adding a line saying the+// row may keep it. Annotated fields are absent by design, so the list reads as+// "what the audit row writes down" and a new credential shows up as one line of+// diff.+//+// The population is wider than the audited RPCs: WrapUnary writes a row on+// `needAudit(ctx) || mcpPolicyDenied`, so gate-refused methods carrying no+// audit annotation are in scope. Every registered Any type is in scope too,+// since those reach the row without passing marshalAuditPayload — which is why+// the registry is enforced at its call sites for this list to mean anything.+//+// The bytebase.store entries are not a layering slip: storepb.AuditLog IS the+// audit row, protojson-marshaled into audit_log.payload by store.CreateAuditLog+// and printed by logAuditToStdout. RequestMetadata and MCPDelegation are there+// because createAuditLog assigns them directly, so nothing else would look at+// them — and RequestMetadata.caller_supplied_user_agent is caller-controlled.++// TestLintAuditPayloadInventory fails when a string or bytes field joins or+// leaves the audited surface without anyone deciding what the audit row may+// record for it.+func TestLintAuditPayloadInventory(t *testing.T) {+ found := auditRecordedScalarFields(t)++ recorded := map[string]bool{}+ for _, field := range auditRecordedFields {+ require.False(t, recorded[field], "%s is listed twice", field)+ recorded[field] = true+ }++ var undecided, unresolvable, unreachable []string+ for _, field := range found {+ if !recorded[field] {+ undecided = append(undecided, field)+ }+ }+ // A listed name stops meaning anything in two different ways, and the fix+ // differs, so they are reported apart rather than as one "stale" bucket.+ for field := range recorded {+ switch {+ case slices.Contains(found, field):+ case inventoryFieldExists(field):+ unreachable = append(unreachable, field)+ default:+ unresolvable = append(unresolvable, field)+ }+ }+ slices.Sort(undecided)+ slices.Sort(unresolvable)+ slices.Sort(unreachable)++ require.Empty(t, undecided,+ "these reach an audit payload and nobody has decided about them: annotate the field "++ "(bytebase.v1.audit_behavior) = SENSITIVE or OMIT, or add it to auditRecordedFields to record that "++ "the audit row may keep it")+ require.Empty(t, unresolvable,+ "these name a field that does not exist, which reads as coverage while asserting nothing; drop them")+ require.Empty(t, unreachable,+ "these no longer reach an audit payload, or are now annotated; drop them so the list keeps meaning something")++ require.True(t, slices.IsSorted(auditRecordedFields),+ "auditRecordedFields is kept sorted so a new entry reads as one line of diff")+}++// auditRecordedScalarFields is every (message, string-or-bytes field) pair the+// audit row can record, derived from the descriptors rather than from reading.+// Reading is how the first sweep of this surface missed ListInstanceDatabase,+// whose request carries an entire Instance.+func auditRecordedScalarFields(t *testing.T) []string {+ t.Helper()+ types := map[protoreflect.FullName]bool{}+ var walk func(protoreflect.MessageDescriptor)+ walk = func(descriptor protoreflect.MessageDescriptor) {+ if types[descriptor.FullName()] {+ return+ }+ types[descriptor.FullName()] = true+ fields := descriptor.Fields()+ for i := range fields.Len() {+ field := fields.Get(i)+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {+ continue+ }+ if sub := submessageOf(field); sub != nil {+ walk(sub)+ }+ }+ }++ // Every v1 method, both directions.+ protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {+ if fd.Package() != "bytebase.v1" {+ return true+ }+ services := fd.Services()+ for i := range services.Len() {+ methods := services.Get(i).Methods()+ for j := range methods.Len() {+ walk(methods.Get(j).Input())+ walk(methods.Get(j).Output())+ }+ }+ return true+ })++ // The audit row itself: service_data, status.details, request_metadata and+ // mcp_delegation never pass an entry point.+ walk((&storepb.AuditLog{}).ProtoReflect().Descriptor())++ // Every registered Any type. These reach the row packed, so no descriptor+ // walk finds them: registering a type is what pulls its fields in here.+ for _, packed := range auditAnyRegistry {+ for _, name := range packed {+ descriptor, err := protoregistry.GlobalFiles.FindDescriptorByName(name)+ require.NoError(t, err, "registered Any type %s is not in the descriptor registry", name)+ md, ok := descriptor.(protoreflect.MessageDescriptor)+ require.True(t, ok, "registered Any type %s is not a message", name)+ walk(md)+ }+ }++ var found []string+ for name := range types {+ descriptor, err := protoregistry.GlobalFiles.FindDescriptorByName(name)+ if err != nil {+ continue+ }+ md, ok := descriptor.(protoreflect.MessageDescriptor)+ if !ok {+ continue+ }+ fields := md.Fields()+ for i := range fields.Len() {+ field := fields.Get(i)+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {+ continue+ }+ kind := field.Kind()+ if field.IsMap() {+ kind = field.MapValue().Kind()+ }+ if kind == protoreflect.StringKind || kind == protoreflect.BytesKind {+ found = append(found, fmt.Sprintf("%s.%s", name, field.Name()))+ }+ }+ }+ slices.Sort(found)+ return slices.Compact(found)+}++// inventoryFieldExists reports whether a "message.field" entry still resolves.+func inventoryFieldExists(entry string) bool {+ cut := strings.LastIndex(entry, ".")+ if cut <= 0 {+ return false+ }+ descriptor, err := protoregistry.GlobalFiles.FindDescriptorByName(protoreflect.FullName(entry[:cut]))+ if err != nil {+ return false+ }+ md, ok := descriptor.(protoreflect.MessageDescriptor)+ return ok && md.Fields().ByName(protoreflect.Name(entry[cut+1:])) != nil+}++// auditRecordedFields is every string and bytes field an audit payload may+// record, sorted, one per line.+//+// The rule for adding one is mechanical: if the value would let someone+// authenticate, authorize, or act as somebody — or is an unbounded body, a+// blob, or personal data the row has no business keeping — it belongs on the+// annotation instead of here.+//+// Names that look like credentials and are not, so the next reader does not+// re-litigate them: DataSourceExternalSecret's url, engine_name, secret_name,+// password_key_name and mount_path say WHERE the secret lives, never what it+// is; DataSource.AzureCredential's tenant_id and client_id name the principal+// rather than authenticate as it; google.protobuf.Any.type_url and value are+// the envelope of a packed message whose own fields come in through+// auditAnyRegistry. AuditLog.request and .response are the redacted payloads+// themselves, kept because the rule is every scalar on the row with no+// exceptions — a new column lands here and forces a decision.+var auditRecordedFields = []string{+ "bytebase.store.AuditLog.method",+ "bytebase.store.AuditLog.parent",+ "bytebase.store.AuditLog.request",+ "bytebase.store.AuditLog.resource",+ "bytebase.store.AuditLog.response",+ "bytebase.store.AuditLog.user",+ "bytebase.store.MCPDelegation.client_id",+ "bytebase.store.MCPDelegation.correlation_id",+ "bytebase.store.MCPDelegation.resource",+ "bytebase.store.MCPDelegation.scope",+ "bytebase.store.RequestMetadata.caller_ip",+ "bytebase.store.RequestMetadata.caller_supplied_user_agent",+ "bytebase.v1.AIChatResponse.content",+ "bytebase.v1.AIChatToolCall.arguments",+ "bytebase.v1.AIChatToolCall.id",+ "bytebase.v1.AIChatToolCall.metadata",+ "bytebase.v1.AIChatToolCall.name",+ "bytebase.v1.AISetting.endpoint",+ "bytebase.v1.AISetting.model",+ "bytebase.v1.AISetting.version",+ "bytebase.v1.AccessGrant.container",+ "bytebase.v1.AccessGrant.creator",+ "bytebase.v1.AccessGrant.issue",+ "bytebase.v1.AccessGrant.name",+ "bytebase.v1.AccessGrant.query",+ "bytebase.v1.AccessGrant.reason",+ "bytebase.v1.AccessGrant.schema",+ "bytebase.v1.AccessGrant.targets",+ "bytebase.v1.ActivateAccessGrantRequest.name",+ "bytebase.v1.ActuatorInfo.default_project",+ "bytebase.v1.ActuatorInfo.external_url",+ "bytebase.v1.ActuatorInfo.git_commit",+ "bytebase.v1.ActuatorInfo.unlicensed_features",+ "bytebase.v1.ActuatorInfo.version",+ "bytebase.v1.ActuatorInfo.workspace",+ "bytebase.v1.AddDataSourceRequest.name",+ "bytebase.v1.AddWebhookRequest.project",+ "bytebase.v1.AdminExecuteRequest.container",+ "bytebase.v1.AdminExecuteRequest.name",+ "bytebase.v1.AdminExecuteRequest.schema",+ "bytebase.v1.AdminExecuteRequest.statement",+ "bytebase.v1.Advice.content",+ "bytebase.v1.Advice.title",+ "bytebase.v1.Algorithm.FullMask.substitution",+ "bytebase.v1.Algorithm.InnerOuterMask.substitution",+ "bytebase.v1.Algorithm.MD5Mask.salt",+ "bytebase.v1.Algorithm.RangeMask.Slice.substitution",+ "bytebase.v1.Announcement.link",+ "bytebase.v1.Announcement.text",+ "bytebase.v1.AppIMSetting.DingTalk.client_id",+ "bytebase.v1.AppIMSetting.DingTalk.robot_code",+ "bytebase.v1.AppIMSetting.Feishu.app_id",+ "bytebase.v1.AppIMSetting.Lark.app_id",+ "bytebase.v1.AppIMSetting.Teams.client_id",+ "bytebase.v1.AppIMSetting.Teams.tenant_id",+ "bytebase.v1.AppIMSetting.Wecom.agent_id",+ "bytebase.v1.AppIMSetting.Wecom.corp_id",+ "bytebase.v1.ApprovalFlow.roles",+ "bytebase.v1.ApprovalTemplate.description",+ "bytebase.v1.ApprovalTemplate.title",+ "bytebase.v1.ApproveIssueRequest.comment",+ "bytebase.v1.ApproveIssueRequest.name",+ "bytebase.v1.AuditLog.method",+ "bytebase.v1.AuditLog.name",+ "bytebase.v1.AuditLog.request",+ "bytebase.v1.AuditLog.resource",+ "bytebase.v1.AuditLog.response",+ "bytebase.v1.AuditLog.user",+ "bytebase.v1.AuthenticationInfo.workspace",+ "bytebase.v1.BatchCancelTaskRunsRequest.parent",+ "bytebase.v1.BatchCancelTaskRunsRequest.task_runs",+ "bytebase.v1.BatchCreateRevisionsRequest.parent",+ "bytebase.v1.BatchCreateSheetsRequest.parent",+ "bytebase.v1.BatchDeleteProjectsRequest.names",+ "bytebase.v1.BatchDeparseResponse.expressions",+ "bytebase.v1.BatchGetDatabasesRequest.names",+ "bytebase.v1.BatchGetDatabasesRequest.parent",+ "bytebase.v1.BatchGetGroupsRequest.names",+ "bytebase.v1.BatchGetProjectsRequest.names",+ "bytebase.v1.BatchGetUsersRequest.names",+ "bytebase.v1.BatchParseRequest.expressions",+ "bytebase.v1.BatchRunTasksRequest.parent",+ "bytebase.v1.BatchRunTasksRequest.tasks",+ "bytebase.v1.BatchSkipTasksRequest.parent",+ "bytebase.v1.BatchSkipTasksRequest.reason",+ "bytebase.v1.BatchSkipTasksRequest.tasks",+ "bytebase.v1.BatchSyncDatabasesRequest.names",+ "bytebase.v1.BatchSyncDatabasesRequest.parent",+ "bytebase.v1.BatchSyncInstancesRequest.parent",+ "bytebase.v1.BatchUpdateDatabasesRequest.parent",+ "bytebase.v1.BatchUpdateInstancesRequest.parent",+ "bytebase.v1.BatchUpdateIssuesStatusRequest.issues",+ "bytebase.v1.BatchUpdateIssuesStatusRequest.parent",+ "bytebase.v1.BatchUpdateIssuesStatusRequest.reason",+ "bytebase.v1.Binding.members",+ "bytebase.v1.Binding.role",+ "bytebase.v1.BindingDelta.member",+ "bytebase.v1.BindingDelta.role",+ "bytebase.v1.CancelPlanCheckRunRequest.name",+ "bytebase.v1.CancelPurchaseRequest.comment",+ "bytebase.v1.CancelPurchaseRequest.feedback",+ "bytebase.v1.Changelog.name",+ "bytebase.v1.Changelog.plan_title",+ "bytebase.v1.Changelog.schema",+ "bytebase.v1.Changelog.task_run",+ "bytebase.v1.CheckConstraintMetadata.expression",+ "bytebase.v1.CheckConstraintMetadata.name",+ "bytebase.v1.CheckReleaseRequest.custom_rules",+ "bytebase.v1.CheckReleaseRequest.parent",+ "bytebase.v1.CheckReleaseRequest.targets",+ "bytebase.v1.CheckReleaseResponse.CheckResult.file",+ "bytebase.v1.CheckReleaseResponse.CheckResult.target",+ "bytebase.v1.ColumnCatalog.classification",+ "bytebase.v1.ColumnCatalog.labels",+ "bytebase.v1.ColumnCatalog.name",+ "bytebase.v1.ColumnCatalog.semantic_type",+ "bytebase.v1.ColumnMetadata.character_set",+ "bytebase.v1.ColumnMetadata.collation",+ "bytebase.v1.ColumnMetadata.comment",+ "bytebase.v1.ColumnMetadata.default",+ "bytebase.v1.ColumnMetadata.default_constraint_name",+ "bytebase.v1.ColumnMetadata.name",+ "bytebase.v1.ColumnMetadata.on_update",+ "bytebase.v1.ColumnMetadata.type",+ "bytebase.v1.CompositeTypeAttribute.collation",+ "bytebase.v1.CompositeTypeAttribute.comment",+ "bytebase.v1.CompositeTypeAttribute.name",+ "bytebase.v1.CompositeTypeAttribute.type",+ "bytebase.v1.CompositeTypeMetadata.comment",+ "bytebase.v1.CompositeTypeMetadata.name",+ "bytebase.v1.CreateAccessGrantRequest.parent",+ "bytebase.v1.CreateDatabaseGroupRequest.database_group_id",+ "bytebase.v1.CreateDatabaseGroupRequest.parent",+ "bytebase.v1.CreateGroupRequest.group_email",+ "bytebase.v1.CreateIdentityProviderRequest.identity_provider_id",+ "bytebase.v1.CreateInstanceRequest.instance_id",+ "bytebase.v1.CreateInstanceRequest.parent",+ "bytebase.v1.CreateIssueCommentRequest.parent",+ "bytebase.v1.CreateIssueRequest.parent",+ "bytebase.v1.CreatePlanRequest.parent",+ "bytebase.v1.CreatePolicyRequest.parent",+ "bytebase.v1.CreateProjectRequest.project_id",+ "bytebase.v1.CreateReleaseRequest.parent",+ "bytebase.v1.CreateReleaseRequest.release_id_template",+ "bytebase.v1.CreateReleaseRequest.release_id_timezone",+ "bytebase.v1.CreateRevisionRequest.parent",+ "bytebase.v1.CreateRoleRequest.role_id",+ "bytebase.v1.CreateRolloutRequest.parent",+ "bytebase.v1.CreateRolloutRequest.target",+ "bytebase.v1.CreateSavedQueryRequest.parent",+ "bytebase.v1.CreateServiceAccountRequest.parent",+ "bytebase.v1.CreateServiceAccountRequest.service_account_id",+ "bytebase.v1.CreateSheetRequest.parent",+ "bytebase.v1.CreateWorkloadIdentityRequest.parent",+ "bytebase.v1.CreateWorkloadIdentityRequest.workload_identity_id",+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.DataClassification.id",+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.DataClassification.title",+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.Level.title",+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.id",+ "bytebase.v1.DataClassificationSetting.DataClassificationConfig.title",+ "bytebase.v1.DataSource.Address.host",+ "bytebase.v1.DataSource.Address.port",+ "bytebase.v1.DataSource.AzureCredential.client_id",+ "bytebase.v1.DataSource.AzureCredential.tenant_id",+ "bytebase.v1.DataSource.authentication_database",+ "bytebase.v1.DataSource.database",+ "bytebase.v1.DataSource.extra_connection_parameters",+ "bytebase.v1.DataSource.host",+ "bytebase.v1.DataSource.id",+ "bytebase.v1.DataSource.instance_id",+ "bytebase.v1.DataSource.master_name",+ "bytebase.v1.DataSource.master_username",+ "bytebase.v1.DataSource.port",+ "bytebase.v1.DataSource.project_id",+ "bytebase.v1.DataSource.region",+ "bytebase.v1.DataSource.replica_set",+ "bytebase.v1.DataSource.service_name",+ "bytebase.v1.DataSource.sid",+ "bytebase.v1.DataSource.ssh_host",+ "bytebase.v1.DataSource.ssh_port",+ "bytebase.v1.DataSource.ssh_user",+ "bytebase.v1.DataSource.username",+ "bytebase.v1.DataSource.warehouse_id",+ "bytebase.v1.DataSourceExternalSecret.AppRoleAuthOption.mount_path",+ "bytebase.v1.DataSourceExternalSecret.engine_name",+ "bytebase.v1.DataSourceExternalSecret.password_key_name",+ "bytebase.v1.DataSourceExternalSecret.secret_name",+ "bytebase.v1.DataSourceExternalSecret.url",+ "bytebase.v1.Database.effective_environment",+ "bytebase.v1.Database.environment",+ "bytebase.v1.Database.labels",+ "bytebase.v1.Database.name",+ "bytebase.v1.Database.project",+ "bytebase.v1.Database.release",+ "bytebase.v1.Database.sync_error",+ "bytebase.v1.DatabaseCatalog.name",+ "bytebase.v1.DatabaseGroup.Database.name",+ "bytebase.v1.DatabaseGroup.name",+ "bytebase.v1.DatabaseGroup.title",+ "bytebase.v1.DatabaseMetadata.character_set",+ "bytebase.v1.DatabaseMetadata.collation",+ "bytebase.v1.DatabaseMetadata.name",+ "bytebase.v1.DatabaseMetadata.owner",+ "bytebase.v1.DatabaseMetadata.search_path",+ "bytebase.v1.DatabaseSDLSchema.content_type",+ "bytebase.v1.DatabaseSDLSchema.schema",+ "bytebase.v1.DatabaseSchema.schema",+ "bytebase.v1.DeleteDatabaseGroupRequest.name",+ "bytebase.v1.DeleteGroupRequest.name",+ "bytebase.v1.DeleteIdentityProviderRequest.name",+ "bytebase.v1.DeleteInstanceRequest.name",+ "bytebase.v1.DeletePolicyRequest.name",+ "bytebase.v1.DeleteProjectRequest.name",+ "bytebase.v1.DeleteReleaseRequest.name",+ "bytebase.v1.DeleteReviewConfigRequest.name",+ "bytebase.v1.DeleteRevisionRequest.name",+ "bytebase.v1.DeleteRoleRequest.name",+ "bytebase.v1.DeleteSavedQueryRequest.name",+ "bytebase.v1.DeleteServiceAccountRequest.name",+ "bytebase.v1.DeleteUserRequest.name",+ "bytebase.v1.DeleteWorkloadIdentityRequest.name",+ "bytebase.v1.DeleteWorkspaceRequest.name",+ "bytebase.v1.DependencyColumn.column",+ "bytebase.v1.DependencyColumn.schema",+ "bytebase.v1.DependencyColumn.table",+ "bytebase.v1.DependencyTable.schema",+ "bytebase.v1.DependencyTable.table",+ "bytebase.v1.DiffMetadataRequest.name",+ "bytebase.v1.DiffMetadataResponse.diff",+ "bytebase.v1.DiffSchemaRequest.changelog",+ "bytebase.v1.DiffSchemaRequest.name",+ "bytebase.v1.DiffSchemaRequest.schema",+ "bytebase.v1.DiffSchemaResponse.diff",+ "bytebase.v1.DimensionConstraint.dimension",+ "bytebase.v1.DimensionalConfig.data_type",+ "bytebase.v1.EmailSetting.SMTPConfig.host",+ "bytebase.v1.EmailSetting.SMTPConfig.username",+ "bytebase.v1.EmailSetting.from",+ "bytebase.v1.EmailSetting.from_name",+ "bytebase.v1.EnumTypeMetadata.comment",+ "bytebase.v1.EnumTypeMetadata.name",+ "bytebase.v1.EnumTypeMetadata.values",+ "bytebase.v1.EnvironmentSetting.Environment.id",+ "bytebase.v1.EnvironmentSetting.Environment.name",+ "bytebase.v1.EnvironmentSetting.Environment.tags",+ "bytebase.v1.EnvironmentSetting.Environment.title",+ "bytebase.v1.EventMetadata.character_set_client",+ "bytebase.v1.EventMetadata.collation_connection",+ "bytebase.v1.EventMetadata.comment",+ "bytebase.v1.EventMetadata.definition",+ "bytebase.v1.EventMetadata.name",+ "bytebase.v1.EventMetadata.sql_mode",+ "bytebase.v1.EventMetadata.time_zone",+ "bytebase.v1.ExchangeTokenRequest.email",+ "bytebase.v1.ExportAuditLogsRequest.filter",+ "bytebase.v1.ExportAuditLogsRequest.order_by",+ "bytebase.v1.ExportAuditLogsRequest.page_token",+ "bytebase.v1.ExportAuditLogsRequest.parent",+ "bytebase.v1.ExportAuditLogsResponse.next_page_token",+ "bytebase.v1.ExportRequest.container",+ "bytebase.v1.ExportRequest.data_source_id",+ "bytebase.v1.ExportRequest.name",+ "bytebase.v1.ExportRequest.schema",+ "bytebase.v1.ExportRequest.statement",+ "bytebase.v1.ExportResponse.applied_access_grant",+ "bytebase.v1.ExtensionMetadata.description",+ "bytebase.v1.ExtensionMetadata.name",+ "bytebase.v1.ExtensionMetadata.schema",+ "bytebase.v1.ExtensionMetadata.version",+ "bytebase.v1.ExternalTableMetadata.external_database_name",+ "bytebase.v1.ExternalTableMetadata.external_server_name",+ "bytebase.v1.ExternalTableMetadata.name",+ "bytebase.v1.FieldMapping.display_name",+ "bytebase.v1.FieldMapping.groups",+ "bytebase.v1.FieldMapping.identifier",+ "bytebase.v1.FieldMapping.phone",+ "bytebase.v1.ForeignKeyMetadata.columns",+ "bytebase.v1.ForeignKeyMetadata.match_type",+ "bytebase.v1.ForeignKeyMetadata.name",+ "bytebase.v1.ForeignKeyMetadata.on_delete",+ "bytebase.v1.ForeignKeyMetadata.on_update",+ "bytebase.v1.ForeignKeyMetadata.referenced_columns",+ "bytebase.v1.ForeignKeyMetadata.referenced_schema",+ "bytebase.v1.ForeignKeyMetadata.referenced_table",+ "bytebase.v1.FunctionMetadata.character_set_client",+ "bytebase.v1.FunctionMetadata.collation_connection",+ "bytebase.v1.FunctionMetadata.comment",+ "bytebase.v1.FunctionMetadata.database_collation",+ "bytebase.v1.FunctionMetadata.definition",+ "bytebase.v1.FunctionMetadata.name",+ "bytebase.v1.FunctionMetadata.signature",+ "bytebase.v1.FunctionMetadata.sql_mode",+ "bytebase.v1.GenerationMetadata.expression",+ "bytebase.v1.GetAccessGrantRequest.name",+ "bytebase.v1.GetAuthenticationRestrictionRequest.workspace",+ "bytebase.v1.GetChangelogRequest.name",+ "bytebase.v1.GetDatabaseCatalogRequest.name",+ "bytebase.v1.GetDatabaseGroupRequest.name",+ "bytebase.v1.GetDatabaseMetadataRequest.filter",+ "bytebase.v1.GetDatabaseMetadataRequest.name",+ "bytebase.v1.GetDatabaseRequest.name",+ "bytebase.v1.GetDatabaseSDLSchemaRequest.name",+ "bytebase.v1.GetDatabaseSchemaRequest.name",+ "bytebase.v1.GetGroupRequest.name",+ "bytebase.v1.GetIamPolicyRequest.resource",+ "bytebase.v1.GetIdentityProviderRequest.name",+ "bytebase.v1.GetInstanceRequest.name",+ "bytebase.v1.GetIssueRequest.name",+ "bytebase.v1.GetPlanCheckRunRequest.name",+ "bytebase.v1.GetPlanRequest.name",+ "bytebase.v1.GetPolicyRequest.name",+ "bytebase.v1.GetProjectRequest.name",+ "bytebase.v1.GetQueryHistoryRequest.name",+ "bytebase.v1.GetReleaseRequest.name",+ "bytebase.v1.GetReviewConfigRequest.name",+ "bytebase.v1.GetRevisionRequest.name",+ "bytebase.v1.GetRoleRequest.name",+ "bytebase.v1.GetRolloutRequest.name",+ "bytebase.v1.GetSavedQueryPolicyRequest.resource",+ "bytebase.v1.GetSavedQueryRequest.name",+ "bytebase.v1.GetSchemaStringRequest.name",+ "bytebase.v1.GetSchemaStringRequest.object",+ "bytebase.v1.GetSchemaStringRequest.schema",+ "bytebase.v1.GetSchemaStringResponse.schema_string",+ "bytebase.v1.GetServiceAccountRequest.name",+ "bytebase.v1.GetSettingRequest.name",+ "bytebase.v1.GetSheetRequest.name",+ "bytebase.v1.GetTaskRunLogRequest.parent",+ "bytebase.v1.GetTaskRunRequest.name",+ "bytebase.v1.GetTaskRunSessionRequest.parent",+ "bytebase.v1.GetUserRequest.name",+ "bytebase.v1.GetWorkloadIdentityRequest.name",+ "bytebase.v1.GetWorkspaceRequest.name",+ "bytebase.v1.GridLevel.density",+ "bytebase.v1.Group.description",+ "bytebase.v1.Group.email",+ "bytebase.v1.Group.name",+ "bytebase.v1.Group.source",+ "bytebase.v1.Group.title",+ "bytebase.v1.GroupMember.member",+ "bytebase.v1.IamPolicy.etag",+ "bytebase.v1.IdentityProvider.domain",+ "bytebase.v1.IdentityProvider.name",+ "bytebase.v1.IdentityProvider.title",+ "bytebase.v1.IndexMetadata.comment",+ "bytebase.v1.IndexMetadata.definition",+ "bytebase.v1.IndexMetadata.expressions",+ "bytebase.v1.IndexMetadata.name",+ "bytebase.v1.IndexMetadata.opclass_names",+ "bytebase.v1.IndexMetadata.parent_index_name",+ "bytebase.v1.IndexMetadata.parent_index_schema",+ "bytebase.v1.IndexMetadata.type",+ "bytebase.v1.Instance.engine_version",+ "bytebase.v1.Instance.environment",+ "bytebase.v1.Instance.external_link",+ "bytebase.v1.Instance.labels",+ "bytebase.v1.Instance.name",+ "bytebase.v1.Instance.title",+ "bytebase.v1.InstanceResource.engine_version",+ "bytebase.v1.InstanceResource.environment",+ "bytebase.v1.InstanceResource.name",+ "bytebase.v1.InstanceResource.title",+ "bytebase.v1.InstanceRole.attribute",+ "bytebase.v1.InstanceRole.name",+ "bytebase.v1.InstanceRole.role_name",+ "bytebase.v1.InstanceRole.valid_until",+ "bytebase.v1.Issue.Approver.principal",+ "bytebase.v1.Issue.access_grant",+ "bytebase.v1.Issue.creator",+ "bytebase.v1.Issue.description",+ "bytebase.v1.Issue.labels",+ "bytebase.v1.Issue.name",+ "bytebase.v1.Issue.plan",+ "bytebase.v1.Issue.title",+ "bytebase.v1.IssueComment.IssueUpdate.from_description",+ "bytebase.v1.IssueComment.IssueUpdate.from_labels",+ "bytebase.v1.IssueComment.IssueUpdate.from_title",+ "bytebase.v1.IssueComment.IssueUpdate.to_description",+ "bytebase.v1.IssueComment.IssueUpdate.to_labels",+ "bytebase.v1.IssueComment.IssueUpdate.to_title",+ "bytebase.v1.IssueComment.comment",+ "bytebase.v1.IssueComment.creator",+ "bytebase.v1.IssueComment.name",+ "bytebase.v1.IssueComment.payload",+ "bytebase.v1.KerberosConfig.instance",+ "bytebase.v1.KerberosConfig.kdc_host",+ "bytebase.v1.KerberosConfig.kdc_port",+ "bytebase.v1.KerberosConfig.kdc_transport_protocol",+ "bytebase.v1.KerberosConfig.primary",+ "bytebase.v1.KerberosConfig.realm",+ "bytebase.v1.LDAPIdentityProviderConfig.base_dn",+ "bytebase.v1.LDAPIdentityProviderConfig.bind_dn",+ "bytebase.v1.LDAPIdentityProviderConfig.host",+ "bytebase.v1.LDAPIdentityProviderConfig.user_filter",+ "bytebase.v1.LDAPIdentityProviderTestRequestContext.username",+ "bytebase.v1.Label.group",+ "bytebase.v1.Label.value",+ "bytebase.v1.LeaveWorkspaceRequest.name",+ "bytebase.v1.ListAccessGrantsRequest.filter",+ "bytebase.v1.ListAccessGrantsRequest.order_by",+ "bytebase.v1.ListAccessGrantsRequest.page_token",+ "bytebase.v1.ListAccessGrantsRequest.parent",+ "bytebase.v1.ListAccessGrantsResponse.next_page_token",+ "bytebase.v1.ListChangelogsRequest.filter",+ "bytebase.v1.ListChangelogsRequest.page_token",+ "bytebase.v1.ListChangelogsRequest.parent",+ "bytebase.v1.ListChangelogsResponse.next_page_token",+ "bytebase.v1.ListDatabaseGroupsRequest.parent",+ "bytebase.v1.ListDatabasesRequest.filter",+ "bytebase.v1.ListDatabasesRequest.order_by",+ "bytebase.v1.ListDatabasesRequest.page_token",+ "bytebase.v1.ListDatabasesRequest.parent",+ "bytebase.v1.ListDatabasesResponse.next_page_token",+ "bytebase.v1.ListGroupsRequest.filter",+ "bytebase.v1.ListGroupsRequest.page_token",+ "bytebase.v1.ListGroupsResponse.next_page_token",+ "bytebase.v1.ListIdentityProvidersRequest.parent",+ "bytebase.v1.ListInstanceDatabaseRequest.name",+ "bytebase.v1.ListInstanceDatabaseResponse.databases",+ "bytebase.v1.ListInstanceRolesRequest.page_token",+ "bytebase.v1.ListInstanceRolesRequest.parent",+ "bytebase.v1.ListInstanceRolesResponse.next_page_token",+ "bytebase.v1.ListInstancesRequest.filter",+ "bytebase.v1.ListInstancesRequest.order_by",+ "bytebase.v1.ListInstancesRequest.page_token",+ "bytebase.v1.ListInstancesRequest.parent",+ "bytebase.v1.ListInstancesResponse.next_page_token",+ "bytebase.v1.ListIssueCommentsRequest.page_token",+ "bytebase.v1.ListIssueCommentsRequest.parent",+ "bytebase.v1.ListIssueCommentsResponse.next_page_token",+ "bytebase.v1.ListIssuesRequest.filter",+ "bytebase.v1.ListIssuesRequest.order_by",+ "bytebase.v1.ListIssuesRequest.page_token",+ "bytebase.v1.ListIssuesRequest.parent",+ "bytebase.v1.ListIssuesRequest.query",+ "bytebase.v1.ListIssuesResponse.next_page_token",+ "bytebase.v1.ListPlansRequest.filter",+ "bytebase.v1.ListPlansRequest.page_token",+ "bytebase.v1.ListPlansRequest.parent",+ "bytebase.v1.ListPlansResponse.next_page_token",+ "bytebase.v1.ListPoliciesRequest.parent",+ "bytebase.v1.ListProjectsRequest.filter",+ "bytebase.v1.ListProjectsRequest.order_by",+ "bytebase.v1.ListProjectsRequest.page_token",+ "bytebase.v1.ListProjectsResponse.next_page_token",+ "bytebase.v1.ListQueryHistoriesRequest.filter",+ "bytebase.v1.ListQueryHistoriesRequest.page_token",+ "bytebase.v1.ListQueryHistoriesRequest.parent",+ "bytebase.v1.ListQueryHistoriesResponse.next_page_token",+ "bytebase.v1.ListReleaseCategoriesRequest.parent",+ "bytebase.v1.ListReleaseCategoriesResponse.categories",+ "bytebase.v1.ListReleasesRequest.filter",+ "bytebase.v1.ListReleasesRequest.page_token",+ "bytebase.v1.ListReleasesRequest.parent",+ "bytebase.v1.ListReleasesResponse.next_page_token",+ "bytebase.v1.ListRevisionsRequest.page_token",+ "bytebase.v1.ListRevisionsRequest.parent",+ "bytebase.v1.ListRevisionsResponse.next_page_token",+ "bytebase.v1.ListRolloutsRequest.filter",+ "bytebase.v1.ListRolloutsRequest.page_token",+ "bytebase.v1.ListRolloutsRequest.parent",+ "bytebase.v1.ListRolloutsResponse.next_page_token",+ "bytebase.v1.ListSavedQueriesRequest.filter",+ "bytebase.v1.ListSavedQueriesRequest.order_by",+ "bytebase.v1.ListSavedQueriesRequest.page_token",+ "bytebase.v1.ListSavedQueriesRequest.parent",+ "bytebase.v1.ListSavedQueriesResponse.next_page_token",+ "bytebase.v1.ListServiceAccountsRequest.filter",+ "bytebase.v1.ListServiceAccountsRequest.page_token",+ "bytebase.v1.ListServiceAccountsRequest.parent",+ "bytebase.v1.ListServiceAccountsResponse.next_page_token",+ "bytebase.v1.ListTaskRunsRequest.parent",+ "bytebase.v1.ListUsersRequest.filter",+ "bytebase.v1.ListUsersRequest.page_token",+ "bytebase.v1.ListUsersResponse.next_page_token",+ "bytebase.v1.ListWorkloadIdentitiesRequest.filter",+ "bytebase.v1.ListWorkloadIdentitiesRequest.page_token",+ "bytebase.v1.ListWorkloadIdentitiesRequest.parent",+ "bytebase.v1.ListWorkloadIdentitiesResponse.next_page_token",+ "bytebase.v1.LoginRequest.email",+ "bytebase.v1.LoginRequest.idp_name",+ "bytebase.v1.LoginRequest.workspace",+ "bytebase.v1.MCPDelegation.client_id",+ "bytebase.v1.MCPDelegation.correlation_id",+ "bytebase.v1.MCPDelegation.resource",+ "bytebase.v1.MCPDelegation.scope",+ "bytebase.v1.MaskingExemptionPolicy.Exemption.members",+ "bytebase.v1.MaskingReason.algorithm",+ "bytebase.v1.MaskingReason.context",+ "bytebase.v1.MaskingReason.masking_rule_id",+ "bytebase.v1.MaskingReason.semantic_type_id",+ "bytebase.v1.MaskingReason.semantic_type_title",+ "bytebase.v1.MaskingRulePolicy.MaskingRule.id",+ "bytebase.v1.MaskingRulePolicy.MaskingRule.semantic_type",+ "bytebase.v1.MaterializedViewMetadata.comment",+ "bytebase.v1.MaterializedViewMetadata.definition",+ "bytebase.v1.MaterializedViewMetadata.name",+ "bytebase.v1.MoveMySavedQueriesRequest.parent",+ "bytebase.v1.MoveMySavedQueriesRequest.source_folder",+ "bytebase.v1.MoveMySavedQueriesRequest.target_folder",+ "bytebase.v1.OAuth2IdentityProviderConfig.auth_url",+ "bytebase.v1.OAuth2IdentityProviderConfig.client_id",+ "bytebase.v1.OAuth2IdentityProviderConfig.scopes",+ "bytebase.v1.OAuth2IdentityProviderConfig.token_url",+ "bytebase.v1.OAuth2IdentityProviderConfig.user_info_url",+ "bytebase.v1.OIDCIdentityProviderConfig.auth_endpoint",+ "bytebase.v1.OIDCIdentityProviderConfig.client_id",+ "bytebase.v1.OIDCIdentityProviderConfig.issuer",+ "bytebase.v1.OIDCIdentityProviderConfig.scopes",+ "bytebase.v1.ObjectSchema.semantic_type",+ "bytebase.v1.PackageMetadata.definition",+ "bytebase.v1.PackageMetadata.name",+ "bytebase.v1.PaymentInfo.currency",+ "bytebase.v1.PaymentInfo.invoice_url",+ "bytebase.v1.PaymentInfo.next_period_price",+ "bytebase.v1.PaymentInfo.period_end",+ "bytebase.v1.PaymentInfo.period_start",+ "bytebase.v1.PaymentInfo.total_price",+ "bytebase.v1.PermissionDeniedDetail.method",+ "bytebase.v1.PermissionDeniedDetail.required_permissions",+ "bytebase.v1.PermissionDeniedDetail.resources",+ "bytebase.v1.Plan.ChangeDatabaseConfig.release",+ "bytebase.v1.Plan.ChangeDatabaseConfig.sheet",+ "bytebase.v1.Plan.ChangeDatabaseConfig.targets",+ "bytebase.v1.Plan.CreateDatabaseConfig.character_set",+ "bytebase.v1.Plan.CreateDatabaseConfig.cluster",+ "bytebase.v1.Plan.CreateDatabaseConfig.collation",+ "bytebase.v1.Plan.CreateDatabaseConfig.database",+ "bytebase.v1.Plan.CreateDatabaseConfig.environment",+ "bytebase.v1.Plan.CreateDatabaseConfig.owner",+ "bytebase.v1.Plan.CreateDatabaseConfig.table",+ "bytebase.v1.Plan.CreateDatabaseConfig.target",+ "bytebase.v1.Plan.RolloutStageSummary.stage",+ "bytebase.v1.Plan.Spec.id",+ "bytebase.v1.Plan.creator",+ "bytebase.v1.Plan.description",+ "bytebase.v1.Plan.issue",+ "bytebase.v1.Plan.name",+ "bytebase.v1.Plan.title",+ "bytebase.v1.PlanCheckRun.Result.content",+ "bytebase.v1.PlanCheckRun.Result.target",+ "bytebase.v1.PlanCheckRun.Result.title",+ "bytebase.v1.PlanCheckRun.error",+ "bytebase.v1.PlanCheckRun.name",+ "bytebase.v1.Policy.name",+ "bytebase.v1.PrepareSampleProjectInstanceRequest.parent",+ "bytebase.v1.PreviewTaskRunRollbackRequest.name",+ "bytebase.v1.PreviewTaskRunRollbackResponse.statement",+ "bytebase.v1.ProcedureMetadata.character_set_client",+ "bytebase.v1.ProcedureMetadata.collation_connection",+ "bytebase.v1.ProcedureMetadata.comment",+ "bytebase.v1.ProcedureMetadata.database_collation",+ "bytebase.v1.ProcedureMetadata.definition",+ "bytebase.v1.ProcedureMetadata.name",+ "bytebase.v1.ProcedureMetadata.signature",+ "bytebase.v1.ProcedureMetadata.sql_mode",+ "bytebase.v1.Project.data_classification_config_id",+ "bytebase.v1.Project.labels",+ "bytebase.v1.Project.name",+ "bytebase.v1.Project.title",+ "bytebase.v1.QueryHistory.creator",+ "bytebase.v1.QueryHistory.database",+ "bytebase.v1.QueryHistory.error",+ "bytebase.v1.QueryHistory.name",+ "bytebase.v1.QueryHistory.statement",+ "bytebase.v1.QueryRequest.container",+ "bytebase.v1.QueryRequest.data_source_id",+ "bytebase.v1.QueryRequest.name",+ "bytebase.v1.QueryRequest.schema",+ "bytebase.v1.QueryRequest.statement",+ "bytebase.v1.QueryResponse.applied_access_grant",+ "bytebase.v1.QueryResult.PostgresError.code",+ "bytebase.v1.QueryResult.PostgresError.column_name",+ "bytebase.v1.QueryResult.PostgresError.constraint_name",+ "bytebase.v1.QueryResult.PostgresError.data_type_name",+ "bytebase.v1.QueryResult.PostgresError.detail",+ "bytebase.v1.QueryResult.PostgresError.file",+ "bytebase.v1.QueryResult.PostgresError.hint",+ "bytebase.v1.QueryResult.PostgresError.internal_query",+ "bytebase.v1.QueryResult.PostgresError.message",+ "bytebase.v1.QueryResult.PostgresError.routine",+ "bytebase.v1.QueryResult.PostgresError.schema_name",+ "bytebase.v1.QueryResult.PostgresError.severity",+ "bytebase.v1.QueryResult.PostgresError.table_name",+ "bytebase.v1.QueryResult.PostgresError.where",+ "bytebase.v1.QueryResult.column_names",+ "bytebase.v1.QueryResult.column_type_names",+ "bytebase.v1.QueryResult.error",+ "bytebase.v1.QueryResult.statement",+ "bytebase.v1.RejectIssueRequest.comment",+ "bytebase.v1.RejectIssueRequest.name",+ "bytebase.v1.Release.File.path",+ "bytebase.v1.Release.File.sheet",+ "bytebase.v1.Release.File.sheet_sha256",+ "bytebase.v1.Release.File.version",+ "bytebase.v1.Release.VCSSource.url",+ "bytebase.v1.Release.category",+ "bytebase.v1.Release.creator",+ "bytebase.v1.Release.name",+ "bytebase.v1.RemoveDataSourceRequest.name",+ "bytebase.v1.RequestIssueRequest.comment",+ "bytebase.v1.RequestIssueRequest.name",+ "bytebase.v1.RequestMetadata.caller_ip",+ "bytebase.v1.RequestMetadata.caller_supplied_user_agent",+ "bytebase.v1.RequestPasswordResetRequest.email",+ "bytebase.v1.RequestPasswordResetRequest.workspace",+ "bytebase.v1.ResetPasswordRequest.email",+ "bytebase.v1.RetryIssueApprovalRequest.name",+ "bytebase.v1.ReviewConfig.name",+ "bytebase.v1.ReviewConfig.resources",+ "bytebase.v1.ReviewConfig.title",+ "bytebase.v1.Revision.deleter",+ "bytebase.v1.Revision.file",+ "bytebase.v1.Revision.name",+ "bytebase.v1.Revision.release",+ "bytebase.v1.Revision.sheet",+ "bytebase.v1.Revision.sheet_sha256",+ "bytebase.v1.Revision.task_run",+ "bytebase.v1.Revision.version",+ "bytebase.v1.RevokeAccessGrantRequest.name",+ "bytebase.v1.Role.description",+ "bytebase.v1.Role.name",+ "bytebase.v1.Role.permissions",+ "bytebase.v1.Role.title",+ "bytebase.v1.RoleGrant.role",+ "bytebase.v1.RoleGrant.user",+ "bytebase.v1.Rollout.name",+ "bytebase.v1.Rollout.title",+ "bytebase.v1.RolloutPolicy.roles",+ "bytebase.v1.RotateDirectorySyncTokenRequest.name",+ "bytebase.v1.RunPlanChecksRequest.name",+ "bytebase.v1.RunPlanChecksRequest.spec_id",+ "bytebase.v1.SQLEditorThemeSetting.id",+ "bytebase.v1.SQLEditorThemeSetting.monaco_base",+ "bytebase.v1.SQLEditorThemeSetting.name",+ "bytebase.v1.SQLReviewRule.NamingRulePayload.format",+ "bytebase.v1.SQLReviewRule.StringArrayRulePayload.list",+ "bytebase.v1.SQLReviewRule.StringRulePayload.value",+ "bytebase.v1.SavedQuery.creator",+ "bytebase.v1.SavedQuery.database",+ "bytebase.v1.SavedQuery.folder",+ "bytebase.v1.SavedQuery.name",+ "bytebase.v1.SavedQuery.project",+ "bytebase.v1.SavedQuery.title",+ "bytebase.v1.SavedQueryBinding.members",+ "bytebase.v1.SavedQueryPolicy.etag",+ "bytebase.v1.SchemaCatalog.name",+ "bytebase.v1.SchemaMetadata.comment",+ "bytebase.v1.SchemaMetadata.name",+ "bytebase.v1.SchemaMetadata.owner",+ "bytebase.v1.SearchAuditLogsRequest.filter",+ "bytebase.v1.SearchAuditLogsRequest.order_by",+ "bytebase.v1.SearchAuditLogsRequest.page_token",+ "bytebase.v1.SearchAuditLogsRequest.parent",+ "bytebase.v1.SearchAuditLogsResponse.next_page_token",+ "bytebase.v1.SearchIssuesRequest.filter",+ "bytebase.v1.SearchIssuesRequest.order_by",+ "bytebase.v1.SearchIssuesRequest.page_token",+ "bytebase.v1.SearchIssuesRequest.parent",+ "bytebase.v1.SearchIssuesRequest.query",+ "bytebase.v1.SearchIssuesResponse.next_page_token",+ "bytebase.v1.SearchMyAccessGrantsRequest.filter",+ "bytebase.v1.SearchMyAccessGrantsRequest.order_by",+ "bytebase.v1.SearchMyAccessGrantsRequest.page_token",+ "bytebase.v1.SearchMyAccessGrantsRequest.parent",+ "bytebase.v1.SearchMyAccessGrantsResponse.next_page_token",+ "bytebase.v1.SearchProjectsRequest.filter",+ "bytebase.v1.SearchProjectsRequest.order_by",+ "bytebase.v1.SearchProjectsRequest.page_token",+ "bytebase.v1.SearchProjectsResponse.next_page_token",+ "bytebase.v1.SearchQueryHistoriesRequest.filter",+ "bytebase.v1.SearchQueryHistoriesRequest.page_token",+ "bytebase.v1.SearchQueryHistoriesRequest.parent",+ "bytebase.v1.SearchQueryHistoriesResponse.next_page_token",+ "bytebase.v1.SearchSavedQueriesRequest.filter",+ "bytebase.v1.SearchSavedQueriesRequest.page_token",+ "bytebase.v1.SearchSavedQueriesRequest.parent",+ "bytebase.v1.SearchSavedQueriesResponse.next_page_token",+ "bytebase.v1.SearchSavedQueryFoldersRequest.filter",+ "bytebase.v1.SearchSavedQueryFoldersRequest.parent",+ "bytebase.v1.SearchSavedQueryFoldersResponse.folders",+ "bytebase.v1.SemanticTypeSetting.SemanticType.description",+ "bytebase.v1.SemanticTypeSetting.SemanticType.icon",+ "bytebase.v1.SemanticTypeSetting.SemanticType.id",+ "bytebase.v1.SemanticTypeSetting.SemanticType.title",+ "bytebase.v1.SendEmailLoginCodeRequest.email",+ "bytebase.v1.SendEmailLoginCodeRequest.workspace",+ "bytebase.v1.SequenceMetadata.cache_size",+ "bytebase.v1.SequenceMetadata.comment",+ "bytebase.v1.SequenceMetadata.data_type",+ "bytebase.v1.SequenceMetadata.increment",+ "bytebase.v1.SequenceMetadata.last_value",+ "bytebase.v1.SequenceMetadata.max_value",+ "bytebase.v1.SequenceMetadata.min_value",+ "bytebase.v1.SequenceMetadata.name",+ "bytebase.v1.SequenceMetadata.owner_column",+ "bytebase.v1.SequenceMetadata.owner_table",+ "bytebase.v1.SequenceMetadata.start",+ "bytebase.v1.ServiceAccount.email",+ "bytebase.v1.ServiceAccount.name",+ "bytebase.v1.ServiceAccount.title",+ "bytebase.v1.SetIamPolicyRequest.etag",+ "bytebase.v1.SetIamPolicyRequest.resource",+ "bytebase.v1.SetSavedQueryPolicyRequest.resource",+ "bytebase.v1.Setting.name",+ "bytebase.v1.Sheet.name",+ "bytebase.v1.SignupRequest.email",+ "bytebase.v1.SignupRequest.title",+ "bytebase.v1.SpatialIndexConfig.method",+ "bytebase.v1.Stage.environment",+ "bytebase.v1.Stage.id",+ "bytebase.v1.Stage.name",+ "bytebase.v1.StorageConfig.buffering",+ "bytebase.v1.StorageConfig.data_compression",+ "bytebase.v1.StorageConfig.sort_in_tempdb",+ "bytebase.v1.StorageConfig.tablespace",+ "bytebase.v1.StorageConfig.work_tablespace",+ "bytebase.v1.StreamMetadata.comment",+ "bytebase.v1.StreamMetadata.definition",+ "bytebase.v1.StreamMetadata.name",+ "bytebase.v1.StreamMetadata.owner",+ "bytebase.v1.StreamMetadata.table_name",+ "bytebase.v1.Subscription.etag",+ "bytebase.v1.Subscription.org_name",+ "bytebase.v1.SwitchWorkspaceRequest.workspace",+ "bytebase.v1.SyncDatabaseRequest.name",+ "bytebase.v1.SyncDatabases.databases",+ "bytebase.v1.SyncInstanceRequest.name",+ "bytebase.v1.SyncInstanceResponse.databases",+ "bytebase.v1.TableCatalog.classification",+ "bytebase.v1.TableCatalog.name",+ "bytebase.v1.TableMetadata.charset",+ "bytebase.v1.TableMetadata.collation",+ "bytebase.v1.TableMetadata.comment",+ "bytebase.v1.TableMetadata.create_options",+ "bytebase.v1.TableMetadata.engine",+ "bytebase.v1.TableMetadata.name",+ "bytebase.v1.TableMetadata.owner",+ "bytebase.v1.TableMetadata.primary_key_type",+ "bytebase.v1.TableMetadata.sharding_info",+ "bytebase.v1.TableMetadata.sorting_keys",+ "bytebase.v1.TablePartitionMetadata.expression",+ "bytebase.v1.TablePartitionMetadata.name",+ "bytebase.v1.TablePartitionMetadata.use_default",+ "bytebase.v1.TablePartitionMetadata.value",+ "bytebase.v1.TagPolicy.tags",+ "bytebase.v1.Task.DatabaseCreate.sheet",+ "bytebase.v1.Task.DatabaseUpdate.release",+ "bytebase.v1.Task.DatabaseUpdate.sheet",+ "bytebase.v1.Task.name",+ "bytebase.v1.Task.skipped_reason",+ "bytebase.v1.Task.spec_id",+ "bytebase.v1.Task.target",+ "bytebase.v1.TaskMetadata.comment",+ "bytebase.v1.TaskMetadata.condition",+ "bytebase.v1.TaskMetadata.definition",+ "bytebase.v1.TaskMetadata.id",+ "bytebase.v1.TaskMetadata.name",+ "bytebase.v1.TaskMetadata.owner",+ "bytebase.v1.TaskMetadata.predecessors",+ "bytebase.v1.TaskMetadata.schedule",+ "bytebase.v1.TaskMetadata.warehouse",+ "bytebase.v1.TaskRun.creator",+ "bytebase.v1.TaskRun.detail",+ "bytebase.v1.TaskRun.name",+ "bytebase.v1.TaskRunLog.name",+ "bytebase.v1.TaskRunLogEntry.CommandExecute.CommandResponse.error",+ "bytebase.v1.TaskRunLogEntry.CommandExecute.statement",+ "bytebase.v1.TaskRunLogEntry.ComputeDiff.error",+ "bytebase.v1.TaskRunLogEntry.DatabaseSync.error",+ "bytebase.v1.TaskRunLogEntry.GhostMigration.error",+ "bytebase.v1.TaskRunLogEntry.PriorBackup.PriorBackupDetail.Item.Table.database",+ "bytebase.v1.TaskRunLogEntry.PriorBackup.PriorBackupDetail.Item.Table.schema",+ "bytebase.v1.TaskRunLogEntry.PriorBackup.PriorBackupDetail.Item.Table.table",+ "bytebase.v1.TaskRunLogEntry.PriorBackup.error",+ "bytebase.v1.TaskRunLogEntry.ReleaseFileExecute.file_path",+ "bytebase.v1.TaskRunLogEntry.ReleaseFileExecute.version",+ "bytebase.v1.TaskRunLogEntry.RetryInfo.error",+ "bytebase.v1.TaskRunLogEntry.SchemaDump.error",+ "bytebase.v1.TaskRunLogEntry.TransactionControl.error",+ "bytebase.v1.TaskRunLogEntry.replica_id",+ "bytebase.v1.TaskRunSession.Postgres.Session.application_name",+ "bytebase.v1.TaskRunSession.Postgres.Session.blocked_by_pids",+ "bytebase.v1.TaskRunSession.Postgres.Session.client_addr",+ "bytebase.v1.TaskRunSession.Postgres.Session.client_port",+ "bytebase.v1.TaskRunSession.Postgres.Session.datname",+ "bytebase.v1.TaskRunSession.Postgres.Session.pid",+ "bytebase.v1.TaskRunSession.Postgres.Session.query",+ "bytebase.v1.TaskRunSession.Postgres.Session.state",+ "bytebase.v1.TaskRunSession.Postgres.Session.usename",+ "bytebase.v1.TaskRunSession.Postgres.Session.wait_event",+ "bytebase.v1.TaskRunSession.Postgres.Session.wait_event_type",+ "bytebase.v1.TaskRunSession.name",+ "bytebase.v1.TessellationConfig.scheme",+ "bytebase.v1.TestEmailSettingRequest.parent",+ "bytebase.v1.TestEmailSettingRequest.to",+ "bytebase.v1.TestEmailSettingResponse.error",+ "bytebase.v1.TestIdentityProviderResponse.claims",+ "bytebase.v1.TestIdentityProviderResponse.user_info",+ "bytebase.v1.TestWebhookRequest.project",+ "bytebase.v1.TestWebhookResponse.error",+ "bytebase.v1.TriggerMetadata.body",+ "bytebase.v1.TriggerMetadata.character_set_client",+ "bytebase.v1.TriggerMetadata.collation_connection",+ "bytebase.v1.TriggerMetadata.comment",+ "bytebase.v1.TriggerMetadata.event",+ "bytebase.v1.TriggerMetadata.name",+ "bytebase.v1.TriggerMetadata.sql_mode",+ "bytebase.v1.TriggerMetadata.timing",+ "bytebase.v1.UndeleteInstanceRequest.name",+ "bytebase.v1.UndeleteProjectRequest.name",+ "bytebase.v1.UndeleteReleaseRequest.name",+ "bytebase.v1.UndeleteServiceAccountRequest.name",+ "bytebase.v1.UndeleteUserRequest.name",+ "bytebase.v1.UndeleteWorkloadIdentityRequest.name",+ "bytebase.v1.UpdateDataSourceRequest.name",+ "bytebase.v1.UpdateEmailRequest.email",+ "bytebase.v1.UpdateEmailRequest.name",+ "bytebase.v1.UpdateIssueCommentRequest.parent",+ "bytebase.v1.UpdatePurchaseRequest.etag",+ "bytebase.v1.UpdateSavedQueryStarRequest.name",+ "bytebase.v1.User.email",+ "bytebase.v1.User.name",+ "bytebase.v1.User.title",+ "bytebase.v1.User.workspace",+ "bytebase.v1.VCSUser.display_name",+ "bytebase.v1.VCSUser.user_id",+ "bytebase.v1.VCSUser.user_name",+ "bytebase.v1.VerifyCheckoutSessionResponse.status",+ "bytebase.v1.ViewMetadata.comment",+ "bytebase.v1.ViewMetadata.definition",+ "bytebase.v1.ViewMetadata.name",+ "bytebase.v1.Webhook.name",+ "bytebase.v1.Webhook.title",+ "bytebase.v1.WorkloadIdentity.email",+ "bytebase.v1.WorkloadIdentity.name",+ "bytebase.v1.WorkloadIdentity.title",+ "bytebase.v1.WorkloadIdentityConfig.allowed_audiences",+ "bytebase.v1.WorkloadIdentityConfig.issuer_url",+ "bytebase.v1.WorkloadIdentityConfig.subject_pattern",+ "bytebase.v1.Workspace.logo",+ "bytebase.v1.Workspace.name",+ "bytebase.v1.Workspace.title",+ "bytebase.v1.WorkspaceProfileSetting.domains",+ "bytebase.v1.WorkspaceProfileSetting.external_url",+ "bytebase.v1.WorkspaceProfileSetting.sql_editor_theme_id",+ "google.api.expr.v1alpha1.Constant.bytes_value",+ "google.api.expr.v1alpha1.Constant.string_value",+ "google.api.expr.v1alpha1.Expr.Call.function",+ "google.api.expr.v1alpha1.Expr.Comprehension.accu_var",+ "google.api.expr.v1alpha1.Expr.Comprehension.iter_var",+ "google.api.expr.v1alpha1.Expr.Comprehension.iter_var2",+ "google.api.expr.v1alpha1.Expr.CreateStruct.Entry.field_key",+ "google.api.expr.v1alpha1.Expr.CreateStruct.message_name",+ "google.api.expr.v1alpha1.Expr.Ident.name",+ "google.api.expr.v1alpha1.Expr.Select.field",+ "google.protobuf.Any.type_url",+ "google.protobuf.Any.value",+ "google.protobuf.FieldMask.paths",+ "google.rpc.Status.message",+ "google.type.Expr.description",+ "google.type.Expr.expression",+ "google.type.Expr.location",+ "google.type.Expr.title",+}++// TestAuditRowNeedsNoRedactionBeyondTheAnyPayloads pins the narrowing in+// createAuditLog. The row is not walked as a whole; service_data and+// status.details are redacted where they are assigned, and everything else is+// left alone because it holds nothing annotated.+//+// That is a fact about today's protos, not a guarantee, so it is asserted+// rather than assumed: annotate a field on RequestMetadata or MCPDelegation —+// caller_supplied_user_agent is caller-controlled and the likeliest candidate —+// and this fails, because nothing would redact it.+func TestAuditRowNeedsNoRedactionBeyondTheAnyPayloads(t *testing.T) {+ for _, message := range []proto.Message{&storepb.RequestMetadata{}, &storepb.MCPDelegation{}} {+ descriptor := message.ProtoReflect().Descriptor()+ require.Nil(t, planFor(descriptor),+ "%s is assigned onto the audit row unredacted; annotating a field under it needs redaction wired up "++ "in createAuditLog first", descriptor.FullName())+ }++ // The row's own plan may cover the two Any fields and nothing else.+ plan := planFor((&storepb.AuditLog{}).ProtoReflect().Descriptor())+ require.NotNil(t, plan)+ fields := (&storepb.AuditLog{}).ProtoReflect().Descriptor().Fields()+ var covered []string+ for number := range plan.fields {+ covered = append(covered, string(fields.ByNumber(number).Name()))+ }+ slices.Sort(covered)+ require.Equal(t, []string{"service_data", "status"}, covered,+ "a field on the audit row started needing redaction; redact it where it is assigned in createAuditLog")+}++// TestLintAuditAnyFieldsAreRegistered is the descriptor half. It catches a new+// google.protobuf.Any FIELD reaching an audit row — a field the walk can see,+// even though the type packed inside it is chosen in Go and cannot be seen.+// Without it a new Any field would be governed by nothing: the runtime drops+// what it cannot place, so the payload would vanish silently.+func TestLintAuditAnyFieldsAreRegistered(t *testing.T) {+ found := auditReachableAnyFields(t)++ var unregistered []string+ for _, field := range found {+ if _, ok := auditAnyRegistry[protoreflect.FullName(field)]; !ok {+ unregistered = append(unregistered, field)+ }+ }+ slices.Sort(unregistered)+ require.Empty(t, unregistered,+ "these carry a google.protobuf.Any onto an audit row and auditAnyRegistry does not name what may be "++ "packed in them, so the runtime would drop whatever they carry: register the permitted types")++ var stale []protoreflect.FullName+ for field := range auditAnyRegistry {+ if !slices.Contains(found, string(field)) {+ stale = append(stale, field)+ }+ }+ slices.SortFunc(stale, func(a, b protoreflect.FullName) int { return strings.Compare(string(a), string(b)) })+ require.Empty(t, stale, "these no longer reach an audit row; drop them so the registry keeps meaning something")+}++// auditReachableAnyFields is every google.protobuf.Any-typed field reachable+// from a message that can reach an audit payload, including the audit row's own+// directly assigned fields.+func auditReachableAnyFields(t *testing.T) []string {+ t.Helper()+ seen := map[protoreflect.FullName]bool{}+ var found []string+ var walk func(protoreflect.MessageDescriptor)+ walk = func(descriptor protoreflect.MessageDescriptor) {+ if seen[descriptor.FullName()] {+ return+ }+ seen[descriptor.FullName()] = true+ fields := descriptor.Fields()+ for i := range fields.Len() {+ field := fields.Get(i)+ // An annotated field never reaches the row, so neither does an Any+ // at or under it. Skipping matches what the redactor does and what+ // the inventory walk counts as reachable.+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {+ continue+ }+ sub := submessageOf(field)+ if sub == nil {+ continue+ }+ if sub.FullName() == anyFullName {+ found = append(found, string(field.FullName()))+ continue+ }+ walk(sub)+ }+ }++ protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {+ if fd.Package() != "bytebase.v1" {+ return true+ }+ services := fd.Services()+ for i := range services.Len() {+ methods := services.Get(i).Methods()+ for j := range methods.Len() {+ walk(methods.Get(j).Input())+ walk(methods.Get(j).Output())+ }+ }+ return true+ })+ walk((&storepb.AuditLog{}).ProtoReflect().Descriptor())++ slices.Sort(found)+ return slices.Compact(found)+}diff --git a/backend/api/v1/audit_redact_test.go b/backend/api/v1/audit_redact_test.gonew file mode 100644index 0000000000..39c5399f1d--- /dev/null+++ b/backend/api/v1/audit_redact_test.go@@ -0,0 +1,1076 @@+package v1++import (+ "context"+ "encoding/base64"+ "errors"+ "fmt"+ "net/http"+ "slices"+ "strings"+ "testing"+ "time"++ "connectrpc.com/connect"+ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/encoding/protojson"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/reflect/protodesc"+ "google.golang.org/protobuf/reflect/protoreflect"+ "google.golang.org/protobuf/reflect/protoregistry"+ "google.golang.org/protobuf/types/descriptorpb"+ "google.golang.org/protobuf/types/dynamicpb"+ "google.golang.org/protobuf/types/known/anypb"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/component/config"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/generated-go/v1/v1connect"+ "github.com/bytebase/bytebase/backend/store"+)++// Audited RPCs write their request and response payloads to audit_log, and to+// stdout when RuntimeEnableAuditLogStdout is set. Anything with+// bb.auditLogs.search/export, or read access to the log pipeline, can read+// them. Redaction therefore owes the row two things, and the first group below+// is exactly those two:+//+// - the secret leaves. Proven exhaustively rather than by example:+// TestAuditRedactionCoversEveryAnnotatedField seeds a sentinel into all 567+// annotated field paths across 321 request and response types and asserts+// none survives. Hand-written per-credential tables used to sit here and+// were removed — every field they covered is in that sweep, on the same+// generated Go types, and a table is a list someone has to remember to+// extend.+//+// - the substance stays. TestAuditRowKeepsItsSubstance is the only guard for+// this, and has to be written by hand: the sweep asserts absence, so a+// redactor that blanked the entire row passes it. Over-redaction is the+// failure mode nothing else in this package can see.+//+// The groups after that are the walker's own invariants — it terminates on the+// cyclic components, reaches an annotation at any depth, rebuilds rather than+// aliases, and never touches the caller's message — then the two payloads a+// descriptor walk cannot see into on its own, a packed Any and the streaming+// path. Fixtures and machinery are last, so the file reads as claims first.++const secretSentinel = "s3cr3t-sentinel-value"++var encodedSecretSentinel = base64.StdEncoding.EncodeToString([]byte(secretSentinel))++// ---- The contract: the secret leaves, the substance stays ----------------++// TestAuditRedactionCoversEveryAnnotatedField is the sweep. A redactor that+// stops covering a field fails here rather than silently starting to log it.+func TestAuditRedactionCoversEveryAnnotatedField(t *testing.T) {+ population := auditPopulation(t)+ names := make([]protoreflect.FullName, 0, len(population))+ for name := range population {+ names = append(names, name)+ }+ slices.SortFunc(names, func(a, b protoreflect.FullName) int { return strings.Compare(string(a), string(b)) })++ covered := 0+ for _, name := range names {+ descriptor := messageDescriptorByName(t, name)+ targets := annotatedTargets(descriptor, string(descriptor.Name()), nil, map[protoreflect.FullName]bool{})+ if len(targets) == 0 {+ continue+ }+ t.Run(string(name), func(t *testing.T) {+ for _, target := range targets {+ covered+++ filled := dynamicMessage(t, descriptor)+ require.True(t, seedTarget(filled.ProtoReflect(), target.chain),+ "%s could not be populated, so every assertion below would pass vacuously", target.path)++ redacted := redactForAudit(filled)+ assertNoAnnotatedContent(t, redacted.ProtoReflect(), string(descriptor.Name()))++ got, err := protojson.Marshal(redacted)+ require.NoError(t, err, target.path)+ require.NotContains(t, string(got), secretSentinel,+ "%s reached the audit payload of %v", target.path, population[name])+ require.NotContains(t, string(got), encodedSecretSentinel,+ "%s reached the audit payload of %v, base64-encoded", target.path, population[name])+ }+ })+ }+ require.NotZero(t, covered, "the sweep found nothing to assert on")+ t.Logf("swept %d annotated field paths across %d request/response types", covered, len(names))+}++// The over-redaction guard, and the half TestAuditRedactionCoversEveryAnnotatedField+// structurally cannot be: that sweep proves annotated fields are gone, and has+// no notion of what the row must still say. Every value below carries a+// sentinel, so the loop asserts both halves — the secret leaves, the substance+// stays.+//+// The rows naming a role, a user and a result count are the leaks this design+// was written for. The sweep already covers their credentials by field path;+// what only lives here is that redacting them did not take the surrounding+// record with it.+func TestAuditRowKeepsItsSubstance(t *testing.T) {+ batchParent := "projects/project-a"+ otpCode := secretSentinel+ mfaTempToken := secretSentinel+ queryResult := func() *v1pb.QueryResult {+ return &v1pb.QueryResult{+ RowsCount: 7,+ ColumnNames: []string{"card_number"},+ Rows: []*v1pb.QueryRow{{Values: []*v1pb.RowValue{{Kind: &v1pb.RowValue_StringValue{StringValue: secretSentinel}}}}},+ }+ }+ for _, tt := range []struct {+ name string+ value any+ response bool+ want []string+ }{+ {+ name: "webhook request retains identifiers",+ value: &v1pb.AddWebhookRequest{Project: "projects/project-a", Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel}},+ want: []string{"projects/project-a", "projects/project-a/webhooks/webhook-a", "Webhook A"},+ },+ {+ // The row has to stay useful: which instances were retargeted, and+ // which kind of IAM credential was supplied.+ name: "batch instance update retains targets and credential type",+ value: &v1pb.BatchUpdateInstancesRequest{+ Parent: &batchParent,+ Requests: []*v1pb.UpdateInstanceRequest{{Instance: &v1pb.Instance{+ Name: "instances/instance-a",+ DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},+ }}},+ },+ want: []string{"projects/project-a", "instances/instance-a", "db.example.com", "gcpCredential"},+ },+ {+ name: "azure iam credential retains the principal it names",+ value: &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{+ IamExtension: &v1pb.DataSource_AzureCredential_{AzureCredential: &v1pb.DataSource_AzureCredential{+ TenantId: "tenant-a", ClientId: "client-a", ClientSecret: secretSentinel,+ }},+ }},+ want: []string{"azureCredential", "tenant-a", "client-a"},+ },+ {+ name: "aws iam credential records that one was supplied",+ value: &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{+ IamExtension: &v1pb.DataSource_AwsCredential{AwsCredential: &v1pb.DataSource_AWSCredential{+ AccessKeyId: secretSentinel, SecretAccessKey: secretSentinel,+ }},+ }},+ want: []string{"awsCredential"},+ },+ {+ name: "audit export retains page token",+ value: &v1pb.ExportAuditLogsResponse{Content: []byte(secretSentinel), NextPageToken: "next-page-token"},+ response: true,+ want: []string{"next-page-token"},+ },+ {+ // INPUT_ONLY on the password, OUTPUT_ONLY only on its container, so+ // clients are meant to send it. Which role was written is the point+ // of the row; redactInstance used to pass the password with it.+ name: "instance role update retains the role it names",+ value: &v1pb.UpdateInstanceRequest{Instance: instanceWithRolePassword()},+ want: []string{"instances/instance-a", "role-a"},+ },+ {+ // A failed validation neither consumes the code nor suppresses the+ // row, so this used to write a still-live OTP.+ name: "user update retains who was updated",+ value: &v1pb.UpdateUserRequest{User: &v1pb.User{Name: "users/[email protected]"}, OtpCode: &otpCode},+ want: []string{"users/[email protected]"},+ },+ {+ // The two hand-written redactors disagreed here: Query kept+ // rows_count and AdminExecute dropped it. One annotation on `rows`+ // settles it, and the count is what makes the row worth reading.+ name: "admin execute retains the shape of the result",+ value: &v1pb.AdminExecuteResponse{Results: []*v1pb.QueryResult{queryResult()}},+ response: true,+ want: []string{`"rowsCount":"7"`, "card_number"},+ },+ {+ name: "query retains the shape of the result",+ value: &v1pb.QueryResponse{Results: []*v1pb.QueryResult{queryResult()}},+ response: true,+ want: []string{`"rowsCount":"7"`, "card_number"},+ },+ {+ // Who tried to log in is the whole point of a login row, and the+ // password, OTP and MFA token beside it are all annotated.+ name: "login retains who attempted it",+ value: &v1pb.LoginRequest{+ Email: "[email protected]",+ Password: secretSentinel,+ OtpCode: &otpCode,+ MfaTempToken: &mfaTempToken,+ },+ want: []string{"[email protected]"},+ },+ {+ name: "login response retains the user it authenticated",+ value: &v1pb.LoginResponse{Token: secretSentinel, User: &v1pb.User{Name: "users/[email protected]"}},+ response: true,+ want: []string{"users/[email protected]"},+ },+ {+ // The Signup password leak Codex flagged on #20024, which surfaced+ // only once SetAuditWorkspaceID re-enabled this audit path.+ name: "signup retains the account being created",+ value: &v1pb.SignupRequest{Email: "[email protected]", Password: secretSentinel, Title: "bob"},+ want: []string{"[email protected]", "bob"},+ },+ {+ // The other half of #20024. An external OIDC token can be replayed+ // against the issuing IdP, so it must not reach a row that+ // bb.auditLogs.search can read — but which workload presented it is+ // exactly what the row is for. ExchangeTokenResponse.access_token+ // leaves nothing behind to assert on, so the sweep covers it alone.+ name: "token exchange retains the workload it correlates",+ value: &v1pb.ExchangeTokenRequest{Token: secretSentinel, Email: "[email protected]"},+ want: []string{"[email protected]"},+ },+ } {+ t.Run(tt.name, func(t *testing.T) {+ got := marshalAuditPayload(tt.value)+ for _, want := range tt.want {+ require.Contains(t, got, want)+ }+ require.NotContains(t, got, secretSentinel, "a credential reached the audit log")+ require.NotContains(t, got, encodedSecretSentinel, "an encoded credential reached the audit log")+ })+ }+}++// ---- The walker's invariants ---------------------------------------------++// Redaction must not mutate the caller's message: it runs on the live request+// and response objects, so masking in place would corrupt what the handler+// returns to the client or what a later interceptor reads.+//+// Asserted as "the message is unchanged afterwards" rather than by reading back+// the one field a case cares about. A per-field check passes while redaction+// quietly clears the neighbour nobody thought to read, and the earlier version+// of this spent forty lines of type switch picking that field out.+//+// The last three rows are the repeated-message half, and the leak this design+// opens with: Instance.roles[].password and instance_resource.data_sources[]+// both cross a repeated message field, where copy-and-share is easiest to get+// wrong.+func TestAuditRedactionDoesNotMutateTheCallersMessage(t *testing.T) {+ for name, message := range map[string]proto.Message{+ "service account response": &v1pb.ServiceAccount{ServiceKey: secretSentinel},+ "project webhook response": &v1pb.Project{Webhooks: []*v1pb.Webhook{{Url: secretSentinel}}},+ "release response": &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}},+ "saved query response": &v1pb.SavedQuery{Content: []byte(secretSentinel)},+ "audit export response": &v1pb.ExportAuditLogsResponse{Content: []byte(secretSentinel), NextPageToken: "next-page-token"},+ "ai setting request": &v1pb.UpdateSettingRequest{Setting: &v1pb.Setting{Value: &v1pb.SettingValue{Value: &v1pb.SettingValue_Ai{Ai: &v1pb.AISetting{ApiKey: secretSentinel}}}}},+ "add data source request": &v1pb.AddDataSourceRequest{DataSource: dataSourceWithEverySecret()},+ "update data source request": &v1pb.UpdateDataSourceRequest{DataSource: dataSourceWithEverySecret()},+ "remove data source request": &v1pb.RemoveDataSourceRequest{DataSource: dataSourceWithEverySecret()},+ "create instance request": &v1pb.CreateInstanceRequest{Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}}},+ "update instance request": &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}}},+ "create release request": &v1pb.CreateReleaseRequest{Release: &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}},+ "update release request": &v1pb.UpdateReleaseRequest{Release: &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}},+ "create saved query request": &v1pb.CreateSavedQueryRequest{SavedQuery: &v1pb.SavedQuery{Content: []byte(secretSentinel)}},+ "reset password request": &v1pb.ResetPasswordRequest{Email: "[email protected]", Code: secretSentinel, NewPassword: secretSentinel},+ "instance roles": &v1pb.Instance{Roles: []*v1pb.InstanceRole{{Name: "r", Password: proto.String(secretSentinel)}}},+ "database instance resource": &v1pb.Database{InstanceResource: &v1pb.InstanceResource{DataSources: []*v1pb.DataSource{{Id: "admin", Password: secretSentinel}}}},+ "batch update instances": &v1pb.BatchUpdateInstancesRequest{Requests: []*v1pb.UpdateInstanceRequest{{+ Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}},+ }}},+ } {+ t.Run(name, func(t *testing.T) {+ before := proto.Clone(message)+ got := marshalAuditPayload(message)++ require.NotContains(t, got, secretSentinel, "a credential reached the audit log")+ require.NotContains(t, got, encodedSecretSentinel, "an encoded credential reached the audit log")+ require.True(t, proto.Equal(before, message),+ "redaction mutated the caller's message\nbefore: %v\nafter: %v", before, message)+ })+ }+}++// TestAuditPlanHandlesCyclicDescriptors pins the fixed-point builder against+// the four cyclic components in the descriptor graph. An unguarded descend runs+// until the stack does, inside the interceptor, taking the RPC with it — and a+// guard keyed on "the field's type equals the parent's" catches only the+// self-recursive one.+//+// Termination is the easy half. The harder one is that breaking a cycle by+// provisionally answering "no plan" leaves the resulting plan with no descend+// arm for the fields that resolved that way, so an annotation inside the cycle+// would be applied at depth 0 and nowhere else — with the coverage sweep still+// passing, because the top-level copy is clean.+func TestAuditPlanHandlesCyclicDescriptors(t *testing.T) {+ for name, root := range map[string]proto.Message{+ // TablePartitionMetadata.subpartitions, the only self-recursive field,+ // and the one component not on an audited path.+ "self-recursive partitions": &v1pb.DiffMetadataRequest{},+ // ObjectSchema <-> StructKind <-> ArrayKind, whose cycle runs through a+ // map. Reached from UpdateDatabaseCatalog, which is audited.+ "object schema through a map": &v1pb.UpdateDatabaseCatalogRequest{},+ // google.protobuf.Value <-> Struct <-> ListValue, reached from+ // QueryResponse.results[].rows[].values[].value_value on every audited+ // Query and AdminExecute.+ "protobuf Value": &v1pb.QueryResponse{},+ // google.api.expr Expr and its nested types, reached from+ // SetIamPolicyRequest.policy.bindings[].parsed_expr.+ "CEL expression": &v1pb.SetIamPolicyRequest{},+ } {+ t.Run(name, func(t *testing.T) {+ done := make(chan *redactPlan, 1)+ go func() { done <- planFor(root.ProtoReflect().Descriptor()) }()+ select {+ case plan := <-done:+ // The plan may legitimately be nil; what matters is that+ // building it terminated rather than recursing forever.+ _ = plan+ case <-time.After(30 * time.Second):+ t.Fatal("plan construction did not terminate on a cyclic descriptor")+ }+ // A second call has to answer from the cache, not rebuild.+ require.NotPanics(t, func() { planFor(root.ProtoReflect().Descriptor()) })+ })+ }+}++// TestAuditRedactionFollowsACycleToAnyDepth is the half termination alone does+// not give: that an annotation INSIDE a cycle is still reached. Breaking a+// cycle by provisionally answering "no plan" leaves the plan with no descend+// arm for the fields that resolved that way, so the annotation is applied at+// depth 0 and nowhere else — with the coverage sweep still green, because the+// top-level copy is clean.+//+// An earlier build did something subtler: it cached plan VALUES in the descend+// arm, freezing a finite chain, so everything below its end was shared+// unredacted. A 10-deep chain leaked 8 of its 11 secrets while a one-level+// fixture stayed green — which is why the depth here is far past three rather+// than the single hop that would read as sufficient. Descend arms hold a+// DESCRIPTOR now and resolve their plan at each depth.+//+// The cycle is built rather than found in the surface, because no annotation+// sits inside one today — exactly why either bug would go unnoticed until one+// did.+func TestAuditRedactionFollowsACycleToAnyDepth(t *testing.T) {+ descriptor := cyclicSensitiveDescriptor(t)+ secret := descriptor.Fields().ByName("secret")+ child := descriptor.Fields().ByName("child")++ plan := planFor(descriptor)+ require.NotNil(t, plan, "a message with an annotated field needs a plan")+ require.Equal(t, actionDescend, plan.fields[child.Number()].kind,+ "the recursive field has to carry a descend arm, or the annotation is applied at depth 0 and nowhere else")++ // Far deeper than any chain a value-caching builder could freeze, which was+ // three links.+ const depth = 12+ root := dynamicpb.NewMessage(descriptor)+ level := root+ for range depth {+ level.Set(secret, protoreflect.ValueOfString(secretSentinel))+ next := dynamicpb.NewMessage(descriptor)+ level.Set(child, protoreflect.ValueOfMessage(next))+ level = next+ }+ level.Set(secret, protoreflect.ValueOfString(secretSentinel))++ got, err := protojson.Marshal(redactForAudit(proto.Message(root)))+ require.NoError(t, err)+ require.NotContains(t, string(got), secretSentinel,+ "an annotated field deep inside a cycle survived redaction")+ require.Equal(t, depth, strings.Count(string(got), `"child"`),+ "the fixture has to actually reach that depth for the assertion above to mean anything")+ require.Equal(t, secretSentinel, root.Get(secret).String(), "redaction mutated the caller's message")+}++// TestAuditRedactionRebuildsMapsRatherThanSharing pins the map rule. Setting a+// map field from the value Range yields shares the Go map itself and its+// message values, so clearing a field in an entry writes through to the+// caller's live message — measured, the source loses the field.+//+// The fixture is built rather than found: no map in the v1 surface has a value+// type with a plan today, so this violation of "redaction does not mutate the+// caller's message" would otherwise go uncaught until one appeared.+func TestAuditRedactionRebuildsMapsRatherThanSharing(t *testing.T) {+ descriptor := mapOfSensitiveDescriptor(t)+ entryValue := dynamicpb.NewMessage(descriptor.Fields().ByName("entries").MapValue().Message())+ entryValue.Set(entryValue.Descriptor().Fields().ByName("secret"), protoreflect.ValueOfString(secretSentinel))++ message := dynamicpb.NewMessage(descriptor)+ entries := message.Mutable(descriptor.Fields().ByName("entries")).Map()+ entries.Set(protoreflect.ValueOfString("k").MapKey(), protoreflect.ValueOfMessage(entryValue))++ plan := planFor(descriptor)+ require.NotNil(t, plan)+ redacted := redactMessage(message, plan)++ got := redacted.Get(descriptor.Fields().ByName("entries")).Map().+ Get(protoreflect.ValueOfString("k").MapKey()).Message()+ require.Empty(t, got.Get(got.Descriptor().Fields().ByName("secret")).String(),+ "a credential inside a map value survived redaction")+ require.Equal(t, secretSentinel, entryValue.Get(entryValue.Descriptor().Fields().ByName("secret")).String(),+ "redaction wrote through the shared map and cleared the caller's own message")+}++// TestAuditRedactionKeepsSensitiveOneofArmsPresent pins the blank-versus-clear+// rule. Clearing a scalar arm unsets the oneof and erases which arm was+// supplied: DataSourceExternalSecret.token is a string arm whose sibling+// app_role is a message that would survive as {}, so clearing would make token+// auth indistinguishable from unconfigured while AppRole stayed legible.+func TestAuditRedactionKeepsSensitiveOneofArmsPresent(t *testing.T) {+ request := &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{+ ExternalSecret: &v1pb.DataSourceExternalSecret{+ Url: "https://vault.example.com",+ AuthOption: &v1pb.DataSourceExternalSecret_Token{Token: secretSentinel},+ },+ }}+ got := marshalAuditPayload(request)+ require.NotContains(t, got, secretSentinel)+ require.Contains(t, got, `"token":""`,+ "the arm must stay present with no value, so the row still records that token auth was configured")+ require.Contains(t, got, "vault.example.com", "the secret store the caller named is the point of the row")++ // The message arm keeps its own shape, minus the credentials inside it.+ appRole := &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{+ ExternalSecret: &v1pb.DataSourceExternalSecret{+ AuthOption: &v1pb.DataSourceExternalSecret_AppRole{+ AppRole: &v1pb.DataSourceExternalSecret_AppRoleAuthOption{+ RoleId: secretSentinel, SecretId: secretSentinel, MountPath: "approle",+ },+ },+ },+ }}+ got = marshalAuditPayload(appRole)+ require.NotContains(t, got, secretSentinel)+ require.Contains(t, got, "approle", "which auth method was configured stays readable")+}++// TestAuditRedactionSharesUnannotatedSubtrees pins the performance property the+// whole design rests on: a subtree holding nothing annotated is shared by+// pointer, never copied. A 5 MB sheet inside an audited batch used to be cloned+// only to have its content nulled.+func TestAuditRedactionSharesUnannotatedSubtrees(t *testing.T) {+ labels := map[string]string{"env": "prod"}+ instance := &v1pb.Instance{+ Name: "instances/instance-a",+ Labels: labels,+ Roles: []*v1pb.InstanceRole{{Name: "instances/instance-a/roles/r", Password: proto.String(secretSentinel)}},+ DataSources: []*v1pb.DataSource{{Id: "admin", Password: secretSentinel}},+ }+ redacted := redactForAudit(instance)+ require.NotSame(t, instance, redacted, "a message with a plan is copied, not returned as itself")+ require.Equal(t, labels, redacted.GetLabels(), "an unannotated map is shared, not rebuilt")++ // A message with nothing annotated anywhere beneath it is returned as+ // itself, which is what makes the sheet case free.+ sheet := &v1pb.CreateSheetRequest{Parent: "projects/project-a", Sheet: &v1pb.Sheet{Content: []byte(secretSentinel)}}+ require.NotSame(t, sheet, redactForAudit(sheet), "the sheet content is annotated, so the request is copied")+ plain := &v1pb.GetSheetRequest{Name: "projects/project-a/sheets/sheet-a"}+ require.Same(t, plain, redactForAudit(plain), "nothing under this message is annotated, so it is not copied")+}++// A rejected request is audited too, so redaction runs on messages the handler+// was about to refuse. A hand-written redactor used to read through the nil+// data source in one of these and panic the interceptor; the descriptor walk+// visits only populated fields, so there is nothing to read through — these+// pin that.+func TestAuditRedactionSurvivesAnIncompleteRequest(t *testing.T) {+ for _, tt := range []struct {+ name string+ request any+ }{+ {"add data source without one", &v1pb.AddDataSourceRequest{Name: "instances/instance-a"}},+ {"update data source without one", &v1pb.UpdateDataSourceRequest{Name: "instances/instance-a"}},+ {"remove data source without one", &v1pb.RemoveDataSourceRequest{Name: "instances/instance-a"}},+ {"create instance without one", &v1pb.CreateInstanceRequest{InstanceId: "instance-a"}},+ {"update instance without one", &v1pb.UpdateInstanceRequest{}},+ {"batch update instances without any", &v1pb.BatchUpdateInstancesRequest{+ Requests: []*v1pb.UpdateInstanceRequest{{}},+ }},+ {"instance carrying an empty data source", &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{+ DataSources: []*v1pb.DataSource{{}},+ }}},+ } {+ t.Run(tt.name, func(t *testing.T) {+ got := marshalAuditPayload(tt.request)+ require.NotEmpty(t, got, "a rejected request still gets a row")+ })+ }+}++// ---- The two payloads a descriptor walk cannot see on its own ------------++// TestAuditRedactsPackedAny pins the Any half. The descriptor walk sees only+// type_url and value, so a credential inside a packed message is invisible to+// it: the packed type is resolved and redacted at runtime, against the+// registry, or the Any is dropped.+func TestAuditRedactsPackedAny(t *testing.T) {+ setting, err := anypb.New(&v1pb.Setting{Value: &v1pb.SettingValue{Value: &v1pb.SettingValue_Ai{+ Ai: &v1pb.AISetting{Endpoint: "https://ai.example.com", ApiKey: secretSentinel},+ }}})+ require.NoError(t, err)++ row := &storepb.AuditLog{+ Method: "/bytebase.v1.SettingService/UpdateSetting",+ ServiceData: redactAuditServiceData(setting),+ }+ got, err := protojson.Marshal(row)+ require.NoError(t, err)+ require.NotContains(t, string(got), secretSentinel, "a credential inside the service_data before-image was recorded")+ require.Contains(t, string(got), "ai.example.com", "the rest of the before-image is the point of the row")+ require.Contains(t, string(setting.GetValue()), secretSentinel, "redaction mutated the caller's Any")++ t.Run("an unregistered packed type is dropped, not logged", func(t *testing.T) {+ // protojson.Marshal fails the ENTIRE row on an unresolvable Any, so+ // dropping is what keeps the record rather than losing it.+ stranger, err := anypb.New(&v1pb.Sheet{Content: []byte(secretSentinel)})+ require.NoError(t, err)+ row := &storepb.AuditLog{+ Method: "/bytebase.v1.SheetService/CreateSheet",+ ServiceData: redactAuditServiceData(stranger),+ }+ require.Nil(t, row.GetServiceData())+ got, err := protojson.Marshal(row)+ require.NoError(t, err)+ require.NotContains(t, string(got), encodedSecretSentinel)+ })++ t.Run("a registered type with nothing to redact keeps its type_url verbatim", func(t *testing.T) {+ // connect's ErrorDetail.Type() returns a bare full name, so details are+ // stored without the type.googleapis.com/ prefix. A round-trip through+ // anypb.New would rewrite it and silently change what SearchAuditLogs+ // emits as @type.+ detail, err := connect.NewErrorDetail(&v1pb.PermissionDeniedDetail{Method: "/bytebase.v1.UserService/GetUser"})+ require.NoError(t, err)+ connectErr := connect.NewError(connect.CodePermissionDenied, errors.New("denied"))+ connectErr.AddDetail(detail)++ row := &storepb.AuditLog{Status: redactAuditStatus(convertErrToStatus(connectErr))}+ require.Len(t, row.GetStatus().GetDetails(), 1)+ require.Equal(t, "bytebase.v1.PermissionDeniedDetail", row.GetStatus().GetDetails()[0].GetTypeUrl())+ })+}++// TestStreamingAuditRedactsRows is the end-to-end assertion the streaming path+// needs. AdminExecute is the only streaming RPC, it is audited, and its+// response carries every row of an admin-mode query — and Send builds its own+// auditEntry and calls createAuditLog directly, so a redaction walk that lived+// in WrapUnary would silently skip it. Everything else about streaming+// persistence is exercised through the createAuditLogFunc stub, which bypasses+// the real path; this one writes a real row and reads it back.+func TestStreamingAuditRedactsRows(t *testing.T) {+ st := newAuditLiveStore(t)+ t.Cleanup(func() { require.NoError(t, st.Close()) })+ interceptor := NewAuditInterceptor(st, "test-secret", &config.Profile{})++ handler := interceptor.WrapStreamingHandler(func(_ context.Context, conn connect.StreamingHandlerConn) error {+ if err := conn.Receive(&v1pb.AdminExecuteRequest{+ Name: "instances/instance-a/databases/db",+ Statement: "SELECT card_number FROM payments",+ }); err != nil {+ return err+ }+ return conn.Send(&v1pb.AdminExecuteResponse{Results: []*v1pb.QueryResult{{+ ColumnNames: []string{"card_number"},+ RowsCount: 1,+ Statement: "SELECT card_number FROM payments",+ Rows: []*v1pb.QueryRow{{Values: []*v1pb.RowValue{+ {Kind: &v1pb.RowValue_StringValue{StringValue: secretSentinel}},+ }}},+ }}})+ })++ ctx := context.WithValue(context.Background(), common.AuthContextKey, &common.AuthContext{+ Audit: true,+ Resources: []*common.Resource{{Type: common.ResourceTypeWorkspace, ID: auditTestWorkspace}},+ })+ require.NoError(t, handler(ctx, &auditStreamingConn{}))++ rows, err := st.SearchAuditLogs(ctx, &store.AuditLogFind{})+ require.NoError(t, err)+ require.Len(t, rows, 1)+ require.NotContains(t, rows[0].Payload.Response, secretSentinel, "an admin-mode result row reached the audit log")+ require.Contains(t, rows[0].Payload.Response, "card_number", "the column names and the statement are the point of the row")+ require.Contains(t, rows[0].Payload.Response, `"rowsCount":"1"`,+ "how many rows the query returned survives; only the rows themselves are dropped")+}++// ---- Guards on the fixtures the sweep builds for itself ------------------++// TestNoAnnotationOutsideBytebaseProtos is what makes the fill's well-known-type+// skip safe. The skip exists because google.protobuf.Value rejects an empty+// instance at marshal time; it would be a hole if an annotation could ever sit+// inside one.+func TestNoAnnotationOutsideBytebaseProtos(t *testing.T) {+ var annotated []string+ protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {+ if strings.HasPrefix(string(fd.Package()), "bytebase.") {+ return true+ }+ var walk func(protoreflect.MessageDescriptors)+ walk = func(messages protoreflect.MessageDescriptors) {+ for i := range messages.Len() {+ md := messages.Get(i)+ fields := md.Fields()+ for j := range fields.Len() {+ if field := fields.Get(j); auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {+ annotated = append(annotated, string(field.FullName()))+ }+ }+ walk(md.Messages())+ }+ }+ walk(fd.Messages())+ return true+ })+ require.Empty(t, annotated,+ "the coverage sweep skips non-bytebase messages when building fixtures; an annotation inside one would go untested")+}++// ---- Fixtures and machinery ----------------------------------------------++func instanceWithRolePassword() *v1pb.Instance {+ password := secretSentinel+ return &v1pb.Instance{+ Name: "instances/instance-a",+ Roles: []*v1pb.InstanceRole{{Name: "instances/instance-a/roles/role-a", RoleName: "role-a", Password: &password}},+ }+}++// dataSourceWithEverySecret is the readable fixture: every credential a data+// source can carry, on one message, so a case reads as the request a caller+// sends. It is a hand-kept list and cannot see a field added later —+// TestAuditRedactionCoversEveryAnnotatedField is the net that can.+func dataSourceWithEverySecret() *v1pb.DataSource {+ return &v1pb.DataSource{+ Id: "admin-ds",+ Host: "db.example.com",+ Username: "admin",+ Password: secretSentinel,+ SslCa: secretSentinel,+ SslCaPath: secretSentinel,+ SslCert: secretSentinel,+ SslCertPath: secretSentinel,+ SslKey: secretSentinel,+ SslKeyPath: secretSentinel,+ SshPassword: secretSentinel,+ SshPrivateKey: secretSentinel,+ AuthenticationPrivateKey: secretSentinel,+ AuthenticationPrivateKeyPassphrase: secretSentinel,+ MasterPassword: secretSentinel,+ ExternalSecret: &v1pb.DataSourceExternalSecret{+ AuthOption: &v1pb.DataSourceExternalSecret_Token{Token: secretSentinel},+ },+ SaslConfig: &v1pb.SASLConfig{Mechanism: &v1pb.SASLConfig_KrbConfig{+ KrbConfig: &v1pb.KerberosConfig{Keytab: []byte(secretSentinel)},+ }},+ IamExtension: &v1pb.DataSource_GcpCredential{+ GcpCredential: &v1pb.DataSource_GCPCredential{Content: secretSentinel},+ },+ }+}++// auditStreamingConn is a minimal AdminExecute stream.+type auditStreamingConn struct {+ connect.StreamingHandlerConn+}++func (*auditStreamingConn) Spec() connect.Spec {+ return connect.Spec{Procedure: v1connect.SQLServiceAdminExecuteProcedure}+}+func (*auditStreamingConn) Peer() connect.Peer { return connect.Peer{} }+func (*auditStreamingConn) RequestHeader() http.Header { return http.Header{} }+func (*auditStreamingConn) Receive(any) error { return nil }+func (*auditStreamingConn) Send(any) error { return nil }++// cyclicSensitiveDescriptor builds `message Node { string secret = 1+// [SENSITIVE]; Node child = 2; }` — a self-recursive message with an+// annotation inside the cycle.+func cyclicSensitiveDescriptor(t *testing.T) protoreflect.MessageDescriptor {+ t.Helper()+ return buildTestMessage(t, "audit_cycle.proto", "audittest.cycle", &descriptorpb.DescriptorProto{+ Name: proto.String("Node"),+ Field: []*descriptorpb.FieldDescriptorProto{+ {+ Name: proto.String("secret"),+ Number: proto.Int32(1),+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),+ Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(),+ Options: sensitiveFieldOptions(),+ },+ {+ Name: proto.String("child"),+ Number: proto.Int32(2),+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),+ Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(),+ TypeName: proto.String(".audittest.cycle.Node"),+ },+ },+ })+}++// mapOfSensitiveDescriptor builds `message Holder { map<string, Entry> entries+// = 1; }` where Entry carries a SENSITIVE field.+func mapOfSensitiveDescriptor(t *testing.T) protoreflect.MessageDescriptor {+ t.Helper()+ return buildTestMessage(t, "audit_map.proto", "audittest.maps", &descriptorpb.DescriptorProto{+ Name: proto.String("Holder"),+ NestedType: []*descriptorpb.DescriptorProto{+ {+ Name: proto.String("EntriesEntry"),+ Field: []*descriptorpb.FieldDescriptorProto{+ {+ Name: proto.String("key"),+ Number: proto.Int32(1),+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),+ Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(),+ },+ {+ Name: proto.String("value"),+ Number: proto.Int32(2),+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),+ Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(),+ TypeName: proto.String(".audittest.maps.Entry"),+ },+ },+ Options: &descriptorpb.MessageOptions{MapEntry: proto.Bool(true)},+ },+ },+ Field: []*descriptorpb.FieldDescriptorProto{+ {+ Name: proto.String("entries"),+ Number: proto.Int32(1),+ Label: descriptorpb.FieldDescriptorProto_LABEL_REPEATED.Enum(),+ Type: descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum(),+ TypeName: proto.String(".audittest.maps.Holder.EntriesEntry"),+ },+ },+ }, &descriptorpb.DescriptorProto{+ Name: proto.String("Entry"),+ Field: []*descriptorpb.FieldDescriptorProto{+ {+ Name: proto.String("secret"),+ Number: proto.Int32(1),+ Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(),+ Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(),+ Options: sensitiveFieldOptions(),+ },+ },+ })+}++func sensitiveFieldOptions() *descriptorpb.FieldOptions {+ options := &descriptorpb.FieldOptions{}+ proto.SetExtension(options, v1pb.E_AuditBehavior, v1pb.AuditBehavior_SENSITIVE)+ return options+}++// buildTestMessage compiles a throwaway file descriptor and returns its first+// message. Building the fixture rather than borrowing one from the surface is+// the point: the shapes it covers — an annotation inside a cycle, and one+// inside a map value — do not exist in the v1 protos today, which is precisely+// why nothing would notice if the redactor stopped handling them.+func buildTestMessage(t *testing.T, path, pkg string, messages ...*descriptorpb.DescriptorProto) protoreflect.MessageDescriptor {+ t.Helper()+ file, err := protodesc.NewFile(&descriptorpb.FileDescriptorProto{+ Name: proto.String(path),+ Package: proto.String(pkg),+ Syntax: proto.String("proto3"),+ MessageType: messages,+ }, nil)+ require.NoError(t, err)+ return file.Messages().Get(0)+}++// auditPopulation is every (procedure, direction) pair whose message can reach+// marshalAuditPayload, keyed by message type: 210 methods resolve to far fewer+// distinct types, and the redactor is per type.+func auditPopulation(t *testing.T) map[protoreflect.FullName][]string {+ t.Helper()+ byType := map[protoreflect.FullName][]string{}+ protoregistry.GlobalFiles.RangeFiles(func(fd protoreflect.FileDescriptor) bool {+ if fd.Package() != "bytebase.v1" {+ return true+ }+ services := fd.Services()+ for i := range services.Len() {+ sd := services.Get(i)+ methods := sd.Methods()+ for j := range methods.Len() {+ md := methods.Get(j)+ procedure := fmt.Sprintf("/%s/%s", sd.FullName(), md.Name())+ byType[md.Input().FullName()] = append(byType[md.Input().FullName()], procedure+" request")+ byType[md.Output().FullName()] = append(byType[md.Output().FullName()], procedure+" response")+ }+ }+ return true+ })+ require.NotEmpty(t, byType)+ return byType+}++// messageDescriptorByName resolves a name the population produced.+func messageDescriptorByName(t *testing.T, name protoreflect.FullName) protoreflect.MessageDescriptor {+ t.Helper()+ descriptor, err := protoregistry.GlobalFiles.FindDescriptorByName(name)+ require.NoError(t, err)+ md, ok := descriptor.(protoreflect.MessageDescriptor)+ require.True(t, ok, "%s is not a message", name)+ return md+}++// isWellKnown reports whether a message comes from outside our own protos.+// The fill skips these: google.protobuf.Value rejects an empty instance at+// marshal time, and building one just to prove it holds no credential is+// wasted work. TestNoAnnotationOutsideBytebaseProtos is what makes the skip+// safe rather than assumed.+func isWellKnown(descriptor protoreflect.MessageDescriptor) bool {+ return !strings.HasPrefix(string(descriptor.FullName()), "bytebase.")+}++// auditTarget is one annotated field and the chain of fields that reaches it+// from a top-level request or response message.+//+// One message per FIELD rather than one message with every field set: setting a+// second arm of a oneof clears the first, so a single-message sweep would+// exercise only the last-numbered arm while every other assertion passed+// vacuously. Driving off the target path also reaches a oneof nested inside+// another oneof's arm — every AppIMSetting payload arm sits under SettingValue's+// app_im arm.+type auditTarget struct {+ path string+ chain []protoreflect.FieldDescriptor+}++// annotatedTargets enumerates every annotated field reachable from a+// descriptor, through repeated fields, map values and every arm of every oneof+// at once. Recursion into a type already on the path stops, which bounds the+// four cyclic components in the descriptor graph.+func annotatedTargets(descriptor protoreflect.MessageDescriptor, prefix string, chain []protoreflect.FieldDescriptor, onPath map[protoreflect.FullName]bool) []auditTarget {+ if onPath[descriptor.FullName()] || isWellKnown(descriptor) {+ return nil+ }+ onPath[descriptor.FullName()] = true+ defer delete(onPath, descriptor.FullName())++ var targets []auditTarget+ for i := range descriptor.Fields().Len() {+ field := descriptor.Fields().Get(i)+ path := fmt.Sprintf("%s.%s", prefix, field.Name())+ next := append(append([]protoreflect.FieldDescriptor{}, chain...), field)+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {+ targets = append(targets, auditTarget{path: path, chain: next})+ continue+ }+ if sub := submessageOf(field); sub != nil {+ targets = append(targets, annotatedTargets(sub, path, next, onPath)...)+ }+ }+ return targets+}++// seedTarget populates exactly one annotated field, walking the chain and+// creating the containers on the way. It reports whether a sentinel value+// actually landed: a target it cannot seed is a target the assertions below+// would pass vacuously, so the caller fails instead of skipping.+func seedTarget(m protoreflect.Message, chain []protoreflect.FieldDescriptor) bool {+ field := chain[0]+ if len(chain) == 1 {+ return seedAnnotatedField(m, field)+ }+ switch {+ case field.IsMap():+ entry := m.Mutable(field).Map().NewValue()+ if !seedTarget(entry.Message(), chain[1:]) {+ return false+ }+ m.Mutable(field).Map().Set(protoreflect.ValueOfString("key").MapKey(), entry)+ return true+ case field.IsList():+ list := m.Mutable(field).List()+ element := list.NewElement()+ if !seedTarget(element.Message(), chain[1:]) {+ return false+ }+ list.Append(element)+ return true+ default:+ return seedTarget(m.Mutable(field).Message(), chain[1:])+ }+}++// seedAnnotatedField puts the sentinel into the annotated field itself. A+// scalar takes it directly; an annotated message or container takes it on the+// first scalar reachable inside, so an OMIT subtree like User.profile or+// QueryResult.rows is populated rather than merely present.+func seedAnnotatedField(m protoreflect.Message, field protoreflect.FieldDescriptor) bool {+ if sub := submessageOf(field); sub != nil {+ seeded := false+ switch {+ case field.IsMap():+ entry := m.Mutable(field).Map().NewValue()+ seeded = seedFirstScalar(entry.Message(), map[protoreflect.FullName]bool{})+ m.Mutable(field).Map().Set(protoreflect.ValueOfString("key").MapKey(), entry)+ case field.IsList():+ list := m.Mutable(field).List()+ element := list.NewElement()+ seeded = seedFirstScalar(element.Message(), map[protoreflect.FullName]bool{})+ list.Append(element)+ default:+ seeded = seedFirstScalar(m.Mutable(field).Message(), map[protoreflect.FullName]bool{})+ }+ return seeded+ }+ value, ok := sentinelValue(field)+ if !ok {+ return false+ }+ if field.IsList() {+ m.Mutable(field).List().Append(value)+ return true+ }+ m.Set(field, value)+ return true+}++// seedFirstScalar puts the sentinel on the first string or bytes field+// reachable inside a message, descending until it finds one.+func seedFirstScalar(m protoreflect.Message, onPath map[protoreflect.FullName]bool) bool {+ descriptor := m.Descriptor()+ if descriptor.FullName() == anyFullName {+ // An Any carries no scalar of its own worth seeding, but its type_url+ // is a string and is what an annotated Any field has to be shown+ // dropping. It never reaches protojson, because the annotation that+ // made it a target is what removes it.+ m.Set(descriptor.Fields().ByName("type_url"), protoreflect.ValueOfString(secretSentinel))+ return true+ }+ if onPath[descriptor.FullName()] || isWellKnown(descriptor) {+ return false+ }+ onPath[descriptor.FullName()] = true+ defer delete(onPath, descriptor.FullName())++ for i := range descriptor.Fields().Len() {+ field := descriptor.Fields().Get(i)+ if submessageOf(field) != nil {+ continue+ }+ if value, ok := sentinelValue(field); ok {+ if field.IsList() {+ m.Mutable(field).List().Append(value)+ } else {+ m.Set(field, value)+ }+ return true+ }+ }+ for i := range descriptor.Fields().Len() {+ field := descriptor.Fields().Get(i)+ sub := submessageOf(field)+ if sub == nil {+ continue+ }+ switch {+ case field.IsMap():+ entry := m.Mutable(field).Map().NewValue()+ if seedFirstScalar(entry.Message(), onPath) {+ m.Mutable(field).Map().Set(protoreflect.ValueOfString("key").MapKey(), entry)+ return true+ }+ case field.IsList():+ list := m.Mutable(field).List()+ element := list.NewElement()+ if seedFirstScalar(element.Message(), onPath) {+ list.Append(element)+ return true+ }+ default:+ if seedFirstScalar(m.Mutable(field).Message(), onPath) {+ return true+ }+ m.Clear(field)+ }+ }+ return false+}++func sentinelValue(field protoreflect.FieldDescriptor) (protoreflect.Value, bool) {+ switch field.Kind() {+ case protoreflect.StringKind:+ return protoreflect.ValueOfString(secretSentinel), true+ case protoreflect.BytesKind:+ return protoreflect.ValueOfBytes([]byte(secretSentinel)), true+ default:+ return protoreflect.Value{}, false+ }+}++// assertNoAnnotatedContent walks every populated field of a redacted message and+// requires that annotated ones carry no content. It is the whole-annotation+// assertion the string check below cannot make: a field the fixture never+// populated passes a "does not contain" check for free.+//+// Blank rather than absent, because a SENSITIVE oneof arm stays present with an+// empty value — that is how the row still records which credential type the+// caller supplied.+func assertNoAnnotatedContent(t *testing.T, m protoreflect.Message, path string) {+ t.Helper()+ m.Range(func(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {+ fieldPath := fmt.Sprintf("%s.%s", path, field.Name())+ if auditBehaviorOf(field) != v1pb.AuditBehavior_AUDIT_BEHAVIOR_UNSPECIFIED {+ require.True(t, isBlankValue(field, value), "annotated field %s survived redaction", fieldPath)+ return true+ }+ switch {+ case field.IsMap():+ if isMessageKind(field.MapValue().Kind()) {+ value.Map().Range(func(key protoreflect.MapKey, entry protoreflect.Value) bool {+ assertNoAnnotatedContent(t, entry.Message(), fmt.Sprintf("%s[%v]", fieldPath, key))+ return true+ })+ }+ case isMessageKind(field.Kind()) && field.IsList():+ for i := range value.List().Len() {+ assertNoAnnotatedContent(t, value.List().Get(i).Message(), fmt.Sprintf("%s[%d]", fieldPath, i))+ }+ case isMessageKind(field.Kind()):+ assertNoAnnotatedContent(t, value.Message(), fieldPath)+ default:+ }+ return true+ })+}++func isBlankValue(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {+ switch {+ case field.IsMap():+ return value.Map().Len() == 0+ case field.IsList():+ return value.List().Len() == 0+ case field.Kind() == protoreflect.BytesKind:+ return len(value.Bytes()) == 0+ case field.Kind() == protoreflect.StringKind:+ return value.String() == ""+ case isMessageKind(field.Kind()):+ return !value.Message().IsValid() || proto.Size(value.Message().Interface()) == 0+ default:+ return false+ }+}++// dynamicMessage builds an empty instance of a descriptor through the global+// type registry, so the sweep needs no per-type Go literal.+func dynamicMessage(t *testing.T, descriptor protoreflect.MessageDescriptor) proto.Message {+ t.Helper()+ messageType, err := protoregistry.GlobalTypes.FindMessageByName(descriptor.FullName())+ require.NoError(t, err)+ return messageType.New().Interface()+}diff --git a/backend/api/v1/audit_redaction_test.go b/backend/api/v1/audit_redaction_test.godeleted file mode 100644index d8ccf3cecd..0000000000--- a/backend/api/v1/audit_redaction_test.go+++ /dev/null@@ -1,754 +0,0 @@-package v1--import (- "encoding/base64"- "fmt"- "slices"- "testing"-- "github.com/stretchr/testify/require"- "google.golang.org/genproto/googleapis/api/annotations"- "google.golang.org/protobuf/proto"- "google.golang.org/protobuf/reflect/protoreflect"- "google.golang.org/protobuf/types/descriptorpb"-- v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"-)--// Audited RPCs write their request and response payloads to audit_log, and to-// stdout when RuntimeEnableAuditLogStdout is set. Anything with-// bb.auditLogs.search/export, or read access to the log pipeline, can read them.-// So no live credential may survive into either payload.-//-// These are behavioral: each case populates a secret and asserts the marshaled-// audit string does not contain it. A redactor that stops covering a field fails-// here rather than silently starting to log it.--const secretSentinel = "s3cr3t-sentinel-value"--var encodedSecretSentinel = base64.StdEncoding.EncodeToString([]byte(secretSentinel))--func TestAuditResponseRedactsCredentials(t *testing.T) {- for _, tt := range []struct {- name string- response any- }{- {- // Only create and key rotation populate this; the read path never- // does. It is a live API key.- name: "service account key",- response: &v1pb.ServiceAccount{Name: "serviceAccounts/[email protected]", ServiceKey: secretSentinel},- },- {- name: "smtp password in a returned setting",- response: &v1pb.Setting{Value: &v1pb.SettingValue{Value: &v1pb.SettingValue_Email{- Email: &v1pb.EmailSetting{Config: &v1pb.EmailSetting_Smtp{- Smtp: &v1pb.EmailSetting_SMTPConfig{Password: secretSentinel},- }},- }}},- },- {- name: "idp client secret in a returned provider",- response: &v1pb.IdentityProvider{Config: &v1pb.IdentityProviderConfig{- Config: &v1pb.IdentityProviderConfig_Oauth2Config{- Oauth2Config: &v1pb.OAuth2IdentityProviderConfig{ClientSecret: secretSentinel},- },- }},- },- {- name: "project webhook URL",- response: &v1pb.Project{Webhooks: []*v1pb.Webhook{{- Name: "projects/project-a/webhooks/webhook-a",- Url: secretSentinel,- }}},- },- {- name: "release response SQL",- response: &v1pb.Release{Files: []*v1pb.Release_File{nil, {- Statement: []byte(secretSentinel),- }}},- },- {- name: "saved query response SQL",- response: &v1pb.SavedQuery{Content: []byte(secretSentinel)},- },- {- name: "purchase checkout credentials",- response: &v1pb.PurchaseResponse{- PaymentUrl: secretSentinel,- SessionId: secretSentinel,- },- },- {- name: "VCS provider user export",- response: &v1pb.ExportVCSProviderUsersResponse{- Content: []byte(secretSentinel),- },- },- {- name: "exported audit log content",- response: &v1pb.ExportAuditLogsResponse{- Content: []byte(secretSentinel),- NextPageToken: "next-page-token",- },- },- {- // The single-instance response is redacted as defense in depth even- // though the converter blanks these on reads; its batch twin gets- // the same treatment rather than the default arm.- name: "batch instance update response",- response: &v1pb.BatchUpdateInstancesResponse{Instances: []*v1pb.Instance{{- Name: "instances/instance-a",- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},- }}},- },- } {- t.Run(tt.name, func(t *testing.T) {- got, err := getResponseString(tt.response)- require.NoError(t, err)- require.NotContains(t, got, secretSentinel, "credential written to the audit log")- require.NotContains(t, got, encodedSecretSentinel, "encoded content written to the audit log")- })- }-}--// dataSourceWithEverySecret is the readable fixture: every secret redactDataSource-// masks today, on one data source, so a case reads as the request a caller sends.-// It is a hand-kept list and cannot see a field added later —-// TestAuditRedactsEveryInputOnlyDataSourceField is the net that can.-func dataSourceWithEverySecret() *v1pb.DataSource {- return &v1pb.DataSource{- Id: "admin",- Type: v1pb.DataSourceType_ADMIN,- Username: "bytebase",- Host: "db.example.com",- Password: secretSentinel,- SslCa: secretSentinel,- SslCaPath: secretSentinel,- SslCert: secretSentinel,- SslCertPath: secretSentinel,- SslKey: secretSentinel,- SslKeyPath: secretSentinel,- SshPassword: secretSentinel,- SshPrivateKey: secretSentinel,- AuthenticationPrivateKey: secretSentinel,- AuthenticationPrivateKeyPassphrase: secretSentinel,- MasterPassword: secretSentinel,- ExternalSecret: &v1pb.DataSourceExternalSecret{- AuthOption: &v1pb.DataSourceExternalSecret_Token{Token: secretSentinel},- },- SaslConfig: &v1pb.SASLConfig{Mechanism: &v1pb.SASLConfig_KrbConfig{- KrbConfig: &v1pb.KerberosConfig{Keytab: []byte(secretSentinel)},- }},- IamExtension: &v1pb.DataSource_GcpCredential{- GcpCredential: &v1pb.DataSource_GCPCredential{Content: secretSentinel},- },- }-}--// assertDataSourceIntact checks the fields a redactor rewrites rather than-// clears, then returns the password for the caller's own comparison.-func assertDataSourceIntact(t *testing.T, d *v1pb.DataSource) string {- t.Helper()- require.Equal(t, secretSentinel, d.GetAuthenticationPrivateKeyPassphrase(), "redaction mutated the passphrase")- require.Equal(t, secretSentinel, d.GetGcpCredential().GetContent(), "redaction mutated the IAM credential")- require.Equal(t, secretSentinel, string(d.GetSaslConfig().GetKrbConfig().GetKeytab()), "redaction mutated the keytab")- return d.GetPassword()-}--// fillInputOnlyFields sets every INPUT_ONLY string and bytes field reachable-// from m to the sentinel, singular or repeated. Each oneof takes its arm-th-// variant, so calling this for every arm up to maxOneofArms covers every branch-// of every oneof declared alongside its siblings.-//-// This reads the same annotation the read path is pinned against-// (assertNoInputOnlyValues), so a credential added to the proto lands in the-// fixture without anyone remembering to add it here. Two shapes it cannot build:-// a map, and a oneof nested inside another oneof's arm, which one global arm-// counter only ever constructs on one value. Neither is silent — inputOnlyLeaves-// counts both, so a fill that cannot reach one fails the test rather than-// quietly asking less of the redactor.-func fillInputOnlyFields(m protoreflect.Message, arm int, onPath map[protoreflect.FullName]bool) {- descriptor := m.Descriptor()- if onPath[descriptor.FullName()] {- return- }- onPath[descriptor.FullName()] = true- defer delete(onPath, descriptor.FullName())-- skip := map[protoreflect.FieldNumber]bool{}- for i := 0; i < descriptor.Oneofs().Len(); i++ {- oneof := descriptor.Oneofs().Get(i)- if oneof.IsSynthetic() {- continue- }- for j := 0; j < oneof.Fields().Len(); j++ {- skip[oneof.Fields().Get(j).Number()] = true- }- fillOneField(m, oneof.Fields().Get(arm%oneof.Fields().Len()), arm, onPath)- }- for i := 0; i < descriptor.Fields().Len(); i++ {- if field := descriptor.Fields().Get(i); !skip[field.Number()] {- fillOneField(m, field, arm, onPath)- }- }-}--func fillOneField(m protoreflect.Message, field protoreflect.FieldDescriptor, arm int, onPath map[protoreflect.FullName]bool) {- if field.IsMap() {- return- }- if field.Kind() == protoreflect.MessageKind {- if field.IsList() {- list := m.Mutable(field).List()- element := list.NewElement()- fillInputOnlyFields(element.Message(), arm, onPath)- list.Append(element)- return- }- fillInputOnlyFields(m.Mutable(field).Message(), arm, onPath)- return- }- if !isInputOnly(field) {- return- }- var value protoreflect.Value- switch field.Kind() {- case protoreflect.StringKind:- value = protoreflect.ValueOfString(secretSentinel)- case protoreflect.BytesKind:- value = protoreflect.ValueOfBytes([]byte(secretSentinel))- default:- return- }- if field.IsList() {- m.Mutable(field).List().Append(value)- return- }- m.Set(field, value)-}--// maxOneofArms returns the widest non-synthetic oneof reachable from the-// descriptor. Driving the arm loop off this rather than off a literal is the-// point: a fourth iam_extension variant raises the bound by declaring itself.-func maxOneofArms(descriptor protoreflect.MessageDescriptor, onPath map[protoreflect.FullName]bool) int {- if onPath[descriptor.FullName()] {- return 0- }- onPath[descriptor.FullName()] = true- defer delete(onPath, descriptor.FullName())-- widest := 1- for i := 0; i < descriptor.Oneofs().Len(); i++ {- if oneof := descriptor.Oneofs().Get(i); !oneof.IsSynthetic() {- widest = max(widest, oneof.Fields().Len())- }- }- for i := 0; i < descriptor.Fields().Len(); i++ {- if field := descriptor.Fields().Get(i); field.Kind() == protoreflect.MessageKind && !field.IsMap() {- widest = max(widest, maxOneofArms(field.Message(), onPath))- }- }- return widest-}--// inputOnlyLeaves is the yardstick: every INPUT_ONLY scalar reachable from the-// descriptor, down every arm of every oneof at once and through map values. The-// fill can only take one arm per pass and cannot build a map at all, so the-// union over passes has to equal this — which is what makes a fill that quietly-// reaches less than it claims fail instead of pass. Every other assertion in the-// test is negative, and a field nobody populated passes them all.-//-// Counting what the fill cannot yet build is deliberate. DataSource already-// carries a map (extra_connection_parameters); the day one like it is declared-// INPUT_ONLY, this fails and names it, rather than the gap staying a comment.-func inputOnlyLeaves(descriptor protoreflect.MessageDescriptor, prefix string, onPath map[protoreflect.FullName]bool) []string {- if onPath[descriptor.FullName()] {- return nil- }- onPath[descriptor.FullName()] = true- defer delete(onPath, descriptor.FullName())-- var paths []string- for i := 0; i < descriptor.Fields().Len(); i++ {- field := descriptor.Fields().Get(i)- path := fmt.Sprintf("%s.%s", prefix, field.Name())- switch {- case field.IsMap():- if isInputOnly(field) {- paths = append(paths, path)- }- if field.MapValue().Kind() == protoreflect.MessageKind {- paths = append(paths, inputOnlyLeaves(field.MapValue().Message(), path, onPath)...)- }- case field.Kind() == protoreflect.MessageKind:- paths = append(paths, inputOnlyLeaves(field.Message(), path, onPath)...)- case isInputOnly(field):- paths = append(paths, path)- default:- }- }- return paths-}--// sentinelPaths reports which fields the fill actually reached.-func sentinelPaths(m protoreflect.Message, prefix string) []string {- var paths []string- m.Range(func(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {- path := fmt.Sprintf("%s.%s", prefix, field.Name())- switch {- case field.IsMap():- case field.Kind() == protoreflect.MessageKind && field.IsList():- for i := range value.List().Len() {- paths = append(paths, sentinelPaths(value.List().Get(i).Message(), path)...)- }- case field.Kind() == protoreflect.MessageKind:- paths = append(paths, sentinelPaths(value.Message(), path)...)- case field.IsList():- for i := range value.List().Len() {- if isSentinel(field, value.List().Get(i)) {- paths = append(paths, path)- break- }- }- default:- if isSentinel(field, value) {- paths = append(paths, path)- }- }- return true- })- return paths-}--func isSentinel(field protoreflect.FieldDescriptor, value protoreflect.Value) bool {- switch field.Kind() {- case protoreflect.StringKind:- return value.String() == secretSentinel- case protoreflect.BytesKind:- return string(value.Bytes()) == secretSentinel- default:- return false- }-}--func isInputOnly(field protoreflect.FieldDescriptor) bool {- options, ok := field.Options().(*descriptorpb.FieldOptions)- if !ok || options == nil {- return false- }- behaviors, ok := proto.GetExtension(options, annotations.E_FieldBehavior).([]annotations.FieldBehavior)- return ok && slices.Contains(behaviors, annotations.FieldBehavior_INPUT_ONLY)-}--// A hand-kept fixture cannot fail for a credential nobody added to it. This one-// is derived from the proto instead: every INPUT_ONLY field on a DataSource,-// every arm of every oneof under it, on each request that carries one. A new-// credential field, or a fourth iam_extension variant, is covered on the day it-// is declared rather than on the day someone notices the audit row.-func TestAuditRedactsEveryInputOnlyDataSourceField(t *testing.T) {- dataSource := (&v1pb.DataSource{}).ProtoReflect().Descriptor()- reached := map[string]bool{}- for arm := range maxOneofArms(dataSource, map[protoreflect.FullName]bool{}) {- filled := &v1pb.DataSource{}- fillInputOnlyFields(filled.ProtoReflect(), arm, map[protoreflect.FullName]bool{})- for _, path := range sentinelPaths(filled.ProtoReflect(), "dataSource") {- reached[path] = true- }-- for _, tt := range []struct {- name string- request any- }{- {"create instance", &v1pb.CreateInstanceRequest{Instance: &v1pb.Instance{- DataSources: []*v1pb.DataSource{filled},- }}},- {"update instance", &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{- DataSources: []*v1pb.DataSource{filled},- }}},- {"batch update instances", &v1pb.BatchUpdateInstancesRequest{- Requests: []*v1pb.UpdateInstanceRequest{{Instance: &v1pb.Instance{- DataSources: []*v1pb.DataSource{filled},- }}},- }},- {"add data source", &v1pb.AddDataSourceRequest{DataSource: filled}},- {"update data source", &v1pb.UpdateDataSourceRequest{DataSource: filled}},- {"remove data source", &v1pb.RemoveDataSourceRequest{DataSource: filled}},- // The seventh carrier, and the one that reaches the audit log only- // because MCP denials are recorded: ListInstanceDatabase is- // EXCLUDED and unaudited, so its request was never written down- // until the ceiling gate started recording its refusals.- {"list instance database", &v1pb.ListInstanceDatabaseRequest{Instance: &v1pb.Instance{- DataSources: []*v1pb.DataSource{filled},- }}},- } {- t.Run(fmt.Sprintf("%s/oneof arm %d", tt.name, arm), func(t *testing.T) {- got, err := getRequestString(tt.request)- require.NoError(t, err)- require.NotContains(t, got, secretSentinel, "credential written to the audit log")- require.NotContains(t, got, encodedSecretSentinel, "encoded credential written to the audit log")- })- }-- t.Run(fmt.Sprintf("redacted message/oneof arm %d", arm), func(t *testing.T) {- // The string check above only catches what the fixture happened to- // populate. This one holds the whole annotation: nothing declared- // INPUT_ONLY may carry a value after redaction.- assertNoInputOnlyValues(t, redactDataSource(filled).ProtoReflect(), "dataSource")- })- }-- // The positive half. Everything above is an assertion that something is- // absent, which a field the fill never populated passes for free — so under-- // coverage would read as success. This one fails instead.- t.Run("the fill reaches every INPUT_ONLY field", func(t *testing.T) {- var missed []string- for _, path := range inputOnlyLeaves(dataSource, "dataSource", map[protoreflect.FullName]bool{}) {- if !reached[path] {- missed = append(missed, path)- }- }- require.Empty(t, missed, "never populated, so the redaction assertions above say nothing about them")- })-}--// A rejected request is audited too, so every redactor runs on messages the-// handler was about to refuse. redactDataSource used to read through the nil-// data source in one of these and panic the interceptor.-func TestAuditRedactionSurvivesAnIncompleteRequest(t *testing.T) {- for _, tt := range []struct {- name string- request any- }{- {"add data source without one", &v1pb.AddDataSourceRequest{Name: "instances/instance-a"}},- {"update data source without one", &v1pb.UpdateDataSourceRequest{Name: "instances/instance-a"}},- {"remove data source without one", &v1pb.RemoveDataSourceRequest{Name: "instances/instance-a"}},- {"create instance without one", &v1pb.CreateInstanceRequest{InstanceId: "instance-a"}},- {"update instance without one", &v1pb.UpdateInstanceRequest{}},- {"batch update instances without any", &v1pb.BatchUpdateInstancesRequest{- Requests: []*v1pb.UpdateInstanceRequest{{}},- }},- {"instance carrying an empty data source", &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{- DataSources: []*v1pb.DataSource{{}},- }}},- } {- t.Run(tt.name, func(t *testing.T) {- got, err := getRequestString(tt.request)- require.NoError(t, err)- require.NotEmpty(t, got, "a rejected request still gets a row")- })- }-}--func TestPrepareSampleProjectInstanceAuditRequestContainsOnlyParent(t *testing.T) {- request := &v1pb.PrepareSampleProjectInstanceRequest{Parent: "projects/project-a"}-- got, err := getRequestString(request)- require.NoError(t, err)- require.Contains(t, got, "projects/project-a")- require.NotContains(t, got, secretSentinel)- require.Equal(- t,- "projects/project-a",- getRequestResource(request, "/bytebase.v1.InstanceService/PrepareSampleProjectInstance"),- )-}--func TestAuditRequestRedactsCredentials(t *testing.T) {- batchParent := "projects/project-a"- settingRequest := func(v *v1pb.SettingValue) *v1pb.UpdateSettingRequest {- return &v1pb.UpdateSettingRequest{Setting: &v1pb.Setting{Value: v}}- }- imSetting := func(s *v1pb.AppIMSetting_IMSetting) *v1pb.UpdateSettingRequest {- return settingRequest(&v1pb.SettingValue{Value: &v1pb.SettingValue_AppIm{- AppIm: &v1pb.AppIMSetting{Settings: []*v1pb.AppIMSetting_IMSetting{s}},- }})- }-- for _, tt := range []struct {- name string- request any- }{- {"smtp password", settingRequest(&v1pb.SettingValue{Value: &v1pb.SettingValue_Email{- Email: &v1pb.EmailSetting{Config: &v1pb.EmailSetting_Smtp{- Smtp: &v1pb.EmailSetting_SMTPConfig{Password: secretSentinel},- }},- }})},- {"ai api key", settingRequest(&v1pb.SettingValue{Value: &v1pb.SettingValue_Ai{- Ai: &v1pb.AISetting{ApiKey: secretSentinel},- }})},- {"slack token", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Slack{Slack: &v1pb.AppIMSetting_Slack{Token: secretSentinel}}})},- {"feishu app secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Feishu{Feishu: &v1pb.AppIMSetting_Feishu{AppSecret: secretSentinel}}})},- {"wecom secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Wecom{Wecom: &v1pb.AppIMSetting_Wecom{Secret: secretSentinel}}})},- {"lark app secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Lark{Lark: &v1pb.AppIMSetting_Lark{AppSecret: secretSentinel}}})},- {"dingtalk client secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Dingtalk{Dingtalk: &v1pb.AppIMSetting_DingTalk{ClientSecret: secretSentinel}}})},- {"teams client secret", imSetting(&v1pb.AppIMSetting_IMSetting{Payload: &v1pb.AppIMSetting_IMSetting_Teams{Teams: &v1pb.AppIMSetting_Teams{ClientSecret: secretSentinel}}})},- {"oauth2 client secret", &v1pb.CreateIdentityProviderRequest{IdentityProvider: &v1pb.IdentityProvider{- Config: &v1pb.IdentityProviderConfig{Config: &v1pb.IdentityProviderConfig_Oauth2Config{- Oauth2Config: &v1pb.OAuth2IdentityProviderConfig{ClientSecret: secretSentinel},- }},- }}},- {"oidc client secret", &v1pb.CreateIdentityProviderRequest{IdentityProvider: &v1pb.IdentityProvider{- Config: &v1pb.IdentityProviderConfig{Config: &v1pb.IdentityProviderConfig_OidcConfig{- OidcConfig: &v1pb.OIDCIdentityProviderConfig{ClientSecret: secretSentinel},- }},- }}},- {"ldap bind password", &v1pb.UpdateIdentityProviderRequest{IdentityProvider: &v1pb.IdentityProvider{- Config: &v1pb.IdentityProviderConfig{Config: &v1pb.IdentityProviderConfig_LdapConfig{- LdapConfig: &v1pb.LDAPIdentityProviderConfig{BindPassword: secretSentinel},- }},- }}},- {"create project webhook URL", &v1pb.CreateProjectRequest{Project: &v1pb.Project{- Webhooks: []*v1pb.Webhook{{Name: "projects/project-a/webhooks/webhook-a", Url: secretSentinel}},- }}},- {"update project webhook URL", &v1pb.UpdateProjectRequest{Project: &v1pb.Project{- Name: "projects/project-a",- Webhooks: []*v1pb.Webhook{{Name: "projects/project-a/webhooks/webhook-a", Url: secretSentinel}},- }}},- {"release SQL", &v1pb.CreateReleaseRequest{Release: &v1pb.Release{- Files: []*v1pb.Release_File{nil, {Statement: []byte(secretSentinel)}},- }}},- {"updated release SQL", &v1pb.UpdateReleaseRequest{Release: &v1pb.Release{- Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}},- }}},- {"saved query SQL", &v1pb.CreateSavedQueryRequest{SavedQuery: &v1pb.SavedQuery{- Content: []byte(secretSentinel),- }}},- // The data-source secrets are INPUT_ONLY, so a write request is the only- // place they appear — which is exactly what an audit row records.- {"instance data source secrets", &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{- Name: "instances/instance-a",- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},- }}},- {"batch instance data source secrets", &v1pb.BatchUpdateInstancesRequest{- Parent: &batchParent,- Requests: []*v1pb.UpdateInstanceRequest{nil, {Instance: &v1pb.Instance{- Name: "instances/instance-a",- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},- }}},- }},- {"created instance data source secrets", &v1pb.CreateInstanceRequest{- InstanceId: "instance-a",- Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}},- }},- {"added data source secrets", &v1pb.AddDataSourceRequest{- Name: "instances/instance-a",- DataSource: dataSourceWithEverySecret(),- }},- {"updated data source secrets", &v1pb.UpdateDataSourceRequest{- Name: "instances/instance-a",- DataSource: dataSourceWithEverySecret(),- }},- {"removed data source secrets", &v1pb.RemoveDataSourceRequest{- Name: "instances/instance-a",- DataSource: dataSourceWithEverySecret(),- }},- {"azure iam client secret", &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{- IamExtension: &v1pb.DataSource_AzureCredential_{AzureCredential: &v1pb.DataSource_AzureCredential{- TenantId: "tenant-a", ClientId: "client-a", ClientSecret: secretSentinel,- }},- }}},- {"aws iam access keys", &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{- IamExtension: &v1pb.DataSource_AwsCredential{AwsCredential: &v1pb.DataSource_AWSCredential{- AccessKeyId: secretSentinel, SecretAccessKey: secretSentinel, SessionToken: secretSentinel,- RoleArn: secretSentinel, ExternalId: secretSentinel,- }},- }}},- {"password reset code and password", &v1pb.ResetPasswordRequest{- Email: "[email protected]",- Code: secretSentinel,- NewPassword: secretSentinel,- }},- {"enterprise license", &v1pb.UploadLicenseRequest{License: secretSentinel}},- {"add webhook URL", &v1pb.AddWebhookRequest{- Project: "projects/project-a",- Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel},- }},- {"update webhook URL", &v1pb.UpdateWebhookRequest{- Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel},- }},- {"remove webhook URL", &v1pb.RemoveWebhookRequest{- Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel},- }},- } {- t.Run(tt.name, func(t *testing.T) {- got, err := getRequestString(tt.request)- require.NoError(t, err)- require.NotContains(t, got, secretSentinel, "credential written to the audit log")- require.NotContains(t, got, encodedSecretSentinel, "encoded content written to the audit log")- if _, ok := tt.request.(*v1pb.ResetPasswordRequest); ok {- require.Contains(t, got, "[email protected]", "email is required for audit attribution")- }- })- }-}--func TestSensitiveOutboundAuditMetadata(t *testing.T) {- batchParent := "projects/project-a"- for _, tt := range []struct {- name string- value any- response bool- want []string- }{- {- name: "webhook request retains identifiers",- value: &v1pb.AddWebhookRequest{Project: "projects/project-a", Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a", Title: "Webhook A", Url: secretSentinel}},- want: []string{"projects/project-a", "projects/project-a/webhooks/webhook-a", "Webhook A"},- },- {- // The row has to stay useful: which instances were retargeted, and- // which kind of IAM credential was supplied.- name: "batch instance update retains targets and credential type",- value: &v1pb.BatchUpdateInstancesRequest{- Parent: &batchParent,- Requests: []*v1pb.UpdateInstanceRequest{{Instance: &v1pb.Instance{- Name: "instances/instance-a",- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},- }}},- },- want: []string{"projects/project-a", "instances/instance-a", "db.example.com", "gcpCredential"},- },- {- name: "azure iam credential retains the principal it names",- value: &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{- IamExtension: &v1pb.DataSource_AzureCredential_{AzureCredential: &v1pb.DataSource_AzureCredential{- TenantId: "tenant-a", ClientId: "client-a", ClientSecret: secretSentinel,- }},- }},- want: []string{"azureCredential", "tenant-a", "client-a"},- },- {- name: "aws iam credential records that one was supplied",- value: &v1pb.AddDataSourceRequest{DataSource: &v1pb.DataSource{- IamExtension: &v1pb.DataSource_AwsCredential{AwsCredential: &v1pb.DataSource_AWSCredential{- AccessKeyId: secretSentinel, SecretAccessKey: secretSentinel,- }},- }},- want: []string{"awsCredential"},- },- {- name: "audit export retains page token",- value: &v1pb.ExportAuditLogsResponse{Content: []byte(secretSentinel), NextPageToken: "next-page-token"},- response: true,- want: []string{"next-page-token"},- },- } {- t.Run(tt.name, func(t *testing.T) {- var (- got string- err error- )- if tt.response {- got, err = getResponseString(tt.value)- } else {- got, err = getRequestString(tt.value)- }- require.NoError(t, err)- for _, want := range tt.want {- require.Contains(t, got, want)- }- })- }-}--// Redaction must not mutate the caller's message: these run on the live request-// and response objects, so masking in place would corrupt what the handler-// returns to the client or what a later interceptor reads.-func TestAuditRedactionDoesNotMutateInput(t *testing.T) {- account := &v1pb.ServiceAccount{ServiceKey: secretSentinel}- _, err := getResponseString(account)- require.NoError(t, err)- require.Equal(t, secretSentinel, account.GetServiceKey(), "redaction mutated the response")-- request := &v1pb.UpdateSettingRequest{Setting: &v1pb.Setting{Value: &v1pb.SettingValue{- Value: &v1pb.SettingValue_Ai{Ai: &v1pb.AISetting{ApiKey: secretSentinel}},- }}}- _, err = getRequestString(request)- require.NoError(t, err)- require.Equal(t, secretSentinel, request.GetSetting().GetValue().GetAi().GetApiKey(),- "redaction mutated the request")-- project := &v1pb.Project{Webhooks: []*v1pb.Webhook{{Url: secretSentinel}}}- _, err = getResponseString(project)- require.NoError(t, err)- require.Equal(t, secretSentinel, project.GetWebhooks()[0].GetUrl(), "redaction mutated the project response")-- release := &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}- _, err = getResponseString(release)- require.NoError(t, err)- require.Equal(t, secretSentinel, string(release.GetFiles()[0].GetStatement()), "redaction mutated the release response")-- savedQuery := &v1pb.SavedQuery{Content: []byte(secretSentinel)}- _, err = getResponseString(savedQuery)- require.NoError(t, err)- require.Equal(t, secretSentinel, string(savedQuery.GetContent()), "redaction mutated the saved query response")-- auditExport := &v1pb.ExportAuditLogsResponse{Content: []byte(secretSentinel), NextPageToken: "next-page-token"}- _, err = getResponseString(auditExport)- require.NoError(t, err)- require.Equal(t, secretSentinel, string(auditExport.GetContent()), "redaction mutated the audit export response")-- for _, tt := range []struct {- name string- request any- }{- {- name: "create instance request",- request: &v1pb.CreateInstanceRequest{Instance: &v1pb.Instance{- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},- }},- },- {- name: "update instance request",- request: &v1pb.UpdateInstanceRequest{Instance: &v1pb.Instance{- DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()},- }},- },- {- name: "batch update instances request",- request: &v1pb.BatchUpdateInstancesRequest{Requests: []*v1pb.UpdateInstanceRequest{{- Instance: &v1pb.Instance{DataSources: []*v1pb.DataSource{dataSourceWithEverySecret()}},- }}},- },- {name: "add data source request", request: &v1pb.AddDataSourceRequest{DataSource: dataSourceWithEverySecret()}},- {name: "update data source request", request: &v1pb.UpdateDataSourceRequest{DataSource: dataSourceWithEverySecret()}},- {name: "remove data source request", request: &v1pb.RemoveDataSourceRequest{DataSource: dataSourceWithEverySecret()}},- {name: "create release request", request: &v1pb.CreateReleaseRequest{Release: &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}}},- {name: "update release request", request: &v1pb.UpdateReleaseRequest{Release: &v1pb.Release{Files: []*v1pb.Release_File{{Statement: []byte(secretSentinel)}}}}},- {name: "create saved query request", request: &v1pb.CreateSavedQueryRequest{SavedQuery: &v1pb.SavedQuery{Content: []byte(secretSentinel)}}},- {name: "reset password request", request: &v1pb.ResetPasswordRequest{Email: "[email protected]", Code: secretSentinel, NewPassword: secretSentinel}},- } {- t.Run(tt.name, func(t *testing.T) {- _, err := getRequestString(tt.request)- require.NoError(t, err)- var sensitiveValue string- switch request := tt.request.(type) {- case *v1pb.CreateInstanceRequest:- sensitiveValue = assertDataSourceIntact(t, request.GetInstance().GetDataSources()[0])- case *v1pb.UpdateInstanceRequest:- sensitiveValue = assertDataSourceIntact(t, request.GetInstance().GetDataSources()[0])- case *v1pb.BatchUpdateInstancesRequest:- sensitiveValue = assertDataSourceIntact(t, request.GetRequests()[0].GetInstance().GetDataSources()[0])- case *v1pb.AddDataSourceRequest:- sensitiveValue = assertDataSourceIntact(t, request.GetDataSource())- case *v1pb.UpdateDataSourceRequest:- sensitiveValue = assertDataSourceIntact(t, request.GetDataSource())- case *v1pb.RemoveDataSourceRequest:- sensitiveValue = assertDataSourceIntact(t, request.GetDataSource())- case *v1pb.CreateReleaseRequest:- sensitiveValue = string(request.GetRelease().GetFiles()[0].GetStatement())- case *v1pb.UpdateReleaseRequest:- sensitiveValue = string(request.GetRelease().GetFiles()[0].GetStatement())- case *v1pb.CreateSavedQueryRequest:- sensitiveValue = string(request.GetSavedQuery().GetContent())- case *v1pb.ResetPasswordRequest:- sensitiveValue = request.GetCode()- require.Equal(t, secretSentinel, request.GetNewPassword())- default:- t.Fatalf("unexpected request type %T", request)- }- require.Equal(t, secretSentinel, sensitiveValue, "redaction mutated the request")- })- }-}diff --git a/backend/api/v1/audit_test.go b/backend/api/v1/audit_test.goindex eeedd09a8b..ef3dc9df33 100644--- a/backend/api/v1/audit_test.go+++ b/backend/api/v1/audit_test.go@@ -135,77 +135,6 @@ func TestLogAuditToStdoutFormat(t *testing.T) {a.Equal("client-A", delegated["mcp_client_id"])}-// TestAuditRedactsCredentials covers the request/response redactors that-// strip secrets before the audit pipeline serializes payloads. Specifically-// guards against regressions like the Signup password leak and the-// ExchangeToken OIDC/access-token leak that Codex flagged on #20024 — both-// only surfaced once the corresponding audit path was re-enabled by the-// SetAuditWorkspaceID callback.-func TestAuditRedactsCredentials(t *testing.T) {- a := require.New(t)-- t.Run("LoginRequest redacts password and MFA secrets", func(_ *testing.T) {- otp := "123456"- mfa := "mfa-temp-jwt"- reqStr, err := getRequestString(&v1pb.LoginRequest{- Email: "[email protected]",- Password: "hunter2",- OtpCode: &otp,- MfaTempToken: &mfa,- })- a.NoError(err)- a.Contains(reqStr, "[email protected]", "non-sensitive email stays")- a.NotContains(reqStr, "hunter2", "plaintext password must not appear")- a.NotContains(reqStr, "123456", "OTP must not appear")- a.NotContains(reqStr, "mfa-temp-jwt", "MFA temp token must not appear")- })-- t.Run("LoginResponse drops token", func(_ *testing.T) {- respStr, err := getResponseString(&v1pb.LoginResponse{- Token: "secret-access-token",- User: &v1pb.User{Name: "users/[email protected]"},- })- a.NoError(err)- a.Contains(respStr, "users/[email protected]", "user info is retained")- a.NotContains(respStr, "secret-access-token", "access token must not appear")- })-- t.Run("SignupRequest redacts password", func(_ *testing.T) {- reqStr, err := getRequestString(&v1pb.SignupRequest{- Email: "[email protected]",- Password: "signup-password",- Title: "bob",- })- a.NoError(err)- a.Contains(reqStr, "[email protected]")- a.NotContains(reqStr, "signup-password",- "plaintext password must not appear in Signup audit")- })-- t.Run("ExchangeTokenRequest redacts OIDC token", func(_ *testing.T) {- reqStr, err := getRequestString(&v1pb.ExchangeTokenRequest{- Token: "oidc.jwt.payload",- Email: "[email protected]",- })- a.NoError(err)- a.Contains(reqStr, "[email protected]",- "workload email retained for audit correlation")- a.NotContains(reqStr, "oidc.jwt.payload",- "external OIDC token must not appear in audit — it can be replayed "+- "against the original IdP or reveal workload claims")- })-- t.Run("ExchangeTokenResponse drops issued access token", func(_ *testing.T) {- respStr, err := getResponseString(&v1pb.ExchangeTokenResponse{- AccessToken: "issued-bytebase-api-token",- })- a.NoError(err)- a.NotContains(respStr, "issued-bytebase-api-token",- "issued Bytebase access token must never be logged — anyone with "+- "audit read access could use it as a valid API token")- })-}-func TestFailedLoginWithoutWorkspaceIsSkipped(t *testing.T) {ctx := context.WithValue(context.Background(), common.AuthContextKey, &common.AuthContext{Audit: true})st := newAuditLiveStore(t)@@ -398,6 +327,7 @@ func TestLifecycleAuditResource(t *testing.T) {{name: "update webhook", request: &v1pb.UpdateWebhookRequest{Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a"}}, method: v1connect.ProjectServiceUpdateWebhookProcedure, want: "projects/project-a/webhooks/webhook-a"},{name: "remove webhook", request: &v1pb.RemoveWebhookRequest{Webhook: &v1pb.Webhook{Name: "projects/project-a/webhooks/webhook-a"}}, method: v1connect.ProjectServiceRemoveWebhookProcedure, want: "projects/project-a/webhooks/webhook-a"},{name: "export audit logs", request: &v1pb.ExportAuditLogsRequest{Parent: "projects/project-a"}, method: v1connect.AuditLogServiceExportAuditLogsProcedure, want: "projects/project-a"},+ {name: "prepare sample project instance", request: &v1pb.PrepareSampleProjectInstanceRequest{Parent: "projects/project-a"}, method: "/bytebase.v1.InstanceService/PrepareSampleProjectInstance", want: "projects/project-a"},} {t.Run(test.name, func(t *testing.T) {require.Equal(t, test.want, getRequestResource(test.request, test.method))diff --git a/backend/api/v1/audit_validate_only_test.go b/backend/api/v1/audit_validate_only_test.goindex 5c4403615b..53f482fb7d 100644--- a/backend/api/v1/audit_validate_only_test.go+++ b/backend/api/v1/audit_validate_only_test.go@@ -91,13 +91,12 @@ func TestValidateOnlyAuditSkipAppliesOnlyToSuccess(t *testing.T) {require.Equal(t, int32(connect.CodePermissionDenied), rows[0].Payload.GetStatus().GetCode(),"the row must carry the denial, not a blank status")- // The newly-logged row goes through the same getRequestString+ // The newly-logged row goes through the same marshalAuditPayload// redaction as every other UpdateDataSource row — that is why// narrowing the skip exposes no field the plain path did not already- // write. It pins that the path runs, not that redactDataSource covers- // every secret: the iam_extension oneof and- // authentication_private_key_passphrase are unmasked there today, on- // this row and on every UpdateDataSource row that preceded it.+ // write. It pins that the path runs on a row the skip used to swallow;+ // which fields the redaction covers is the annotation's job, pinned by+ // TestAuditRedactionCoversEveryAnnotatedField.request := rows[0].Payload.GetRequest()require.NotEmpty(t, request)require.NotContains(t, request, storedPassword,diff --git a/backend/api/v1/instance_service_converter_test.go b/backend/api/v1/instance_service_converter_test.goindex a0a0689776..3cec259f4b 100644--- a/backend/api/v1/instance_service_converter_test.go+++ b/backend/api/v1/instance_service_converter_test.go@@ -442,57 +442,88 @@ func TestRetainStoredKeytabs(t *testing.T) {require.Empty(t, moved[1].GetSaslConfig().GetKrbConfig().Keytab)}-// assertNoInputOnlyValues walks every populated message field and requires-// that fields annotated INPUT_ONLY carry no value. It pins the read-path-// contract: whatever the proto declares write-only must be blanked by the-// store->v1 converters.-func assertNoInputOnlyValues(t *testing.T, m protoreflect.Message, path string) {+// assertNoSensitiveValues walks every populated message field and requires that+// fields the proto declares write-only or credential-bearing carry no value. It+// pins the read-path contract: whatever is INPUT_ONLY or SENSITIVE must be+// blanked by the store->v1 converters. A converter that legitimately mints one+// — CreateServiceAccount returning the key it just made — is exercised by its+// own test rather than exempted here, so this walk stays unconditional.+//+// SENSITIVE as well as INPUT_ONLY, because the two answer different questions+// and neither subsumes the other: ServiceAccount.service_key is OUTPUT_ONLY and+// is a live credential, while IdP client_secret and Login password carry no+// field_behavior at all.+func assertNoSensitiveValues(t *testing.T, m protoreflect.Message, path string) {fields := m.Descriptor().Fields()for i := 0; i < fields.Len(); i++ {fd := fields.Get(i)fieldPath := fmt.Sprintf("%s.%s", path, fd.Name())- if opts, ok := fd.Options().(*descriptorpb.FieldOptions); ok && opts != nil {- behaviors, ok := proto.GetExtension(opts, annotations.E_FieldBehavior).([]annotations.FieldBehavior)- if ok && slices.Contains(behaviors, annotations.FieldBehavior_INPUT_ONLY) {- // Oneof members may stay present as an is-configured signal- // (e.g. the Vault token), so require blank content, not- // absence.- if m.Has(fd) {- blank := false- switch {- case fd.IsList():- blank = m.Get(fd).List().Len() == 0- case fd.Kind() == protoreflect.BytesKind:- blank = len(m.Get(fd).Bytes()) == 0- case fd.Kind() == protoreflect.StringKind:- blank = m.Get(fd).String() == ""- default:- }- require.True(t, blank, "INPUT_ONLY field %s must be blank on reads", fieldPath)- }- continue+ if isInputOnly(fd) || auditBehaviorOf(fd) == v1pb.AuditBehavior_SENSITIVE {+ // Oneof members may stay present as an is-configured signal+ // (e.g. the Vault token), so require blank content, not+ // absence.+ if m.Has(fd) {+ require.True(t, isBlankValue(fd, m.Get(fd)), "write-only field %s must be blank on reads", fieldPath)}+ continue}if fd.IsMap() {+ // Four converter-reachable maps carry message values, and a+ // credential inside one would reach a read response with CI green+ // if the walk stopped here.+ if isMessageKind(fd.MapValue().Kind()) && m.Has(fd) {+ m.Get(fd).Map().Range(func(key protoreflect.MapKey, entry protoreflect.Value) bool {+ assertNoSensitiveValues(t, entry.Message(), fmt.Sprintf("%s[%v]", fieldPath, key))+ return true+ })+ }continue}- if fd.Kind() != protoreflect.MessageKind {+ if !isMessageKind(fd.Kind()) {continue}if fd.IsList() {list := m.Get(fd).List()for j := 0; j < list.Len(); j++ {- assertNoInputOnlyValues(t, list.Get(j).Message(), fmt.Sprintf("%s[%d]", fieldPath, j))+ assertNoSensitiveValues(t, list.Get(j).Message(), fmt.Sprintf("%s[%d]", fieldPath, j))}continue}if m.Has(fd) {- assertNoInputOnlyValues(t, m.Get(fd).Message(), fieldPath)+ assertNoSensitiveValues(t, m.Get(fd).Message(), fieldPath)}}}-func TestConvertDataSourcesBlanksEveryInputOnlyField(t *testing.T) {+// isInputOnly reports whether the proto declares a field write-only.+func isInputOnly(field protoreflect.FieldDescriptor) bool {+ options, ok := field.Options().(*descriptorpb.FieldOptions)+ if !ok || options == nil {+ return false+ }+ behaviors, ok := proto.GetExtension(options, annotations.E_FieldBehavior).([]annotations.FieldBehavior)+ return ok && slices.Contains(behaviors, annotations.FieldBehavior_INPUT_ONLY)+}++// InstanceRole.password is INPUT_ONLY and SENSITIVE, and the store message has+// no password field at all — so reads cannot populate it. This pins that: the+// audit-side leak this design opens with was the request path, and the read+// path staying clean is what lets the annotation be SENSITIVE rather than OMIT.+func TestConvertInstanceRolesBlanksEveryWriteOnlyField(t *testing.T) {+ instance := &store.InstanceMessage{ResourceID: "instance-a"}+ roles := []*storepb.InstanceRole{+ {Name: "admin", Attribute: proto.String("SUPERUSER"), ConnectionLimit: proto.Int32(10)},+ {Name: "readonly"},+ }+ got := convertInstanceRoles(instance, roles)+ require.Len(t, got, len(roles))+ for _, role := range got {+ assertNoSensitiveValues(t, role.ProtoReflect(), fmt.Sprintf("instance_role(%s)", role.GetRoleName()))+ }+ require.Equal(t, "instances/instance-a/roles/admin", got[0].GetName())+}++func TestConvertDataSourcesBlanksEveryWriteOnlyField(t *testing.T) {// One store data source per authentication/secret shape, each with every// secret populated, so the INPUT_ONLY walk covers all converter branches.dataSources := []*storepb.DataSource{@@ -593,7 +624,7 @@ func TestConvertDataSourcesBlanksEveryInputOnlyField(t *testing.T) {got := convertDataSources(dataSources)require.Len(t, got, len(dataSources))for _, ds := range got {- assertNoInputOnlyValues(t, ds.ProtoReflect(), fmt.Sprintf("data_source(%s)", ds.GetId()))+ assertNoSensitiveValues(t, ds.ProtoReflect(), fmt.Sprintf("data_source(%s)", ds.GetId()))}// The AWS credential keeps signaling presence without content.diff --git a/backend/api/v1/mcp_gate.go b/backend/api/v1/mcp_gate.goindex 4226c15974..294c3ab9ca 100644--- a/backend/api/v1/mcp_gate.go+++ b/backend/api/v1/mcp_gate.go@@ -313,7 +313,8 @@ func mcpSettingsFromContext(ctx context.Context) (store.MCPSettings, bool) {// EXCLUDED ones — and TestEmailSetting and TestIdentityProvider are the rows an// operator would most want, since each would have carried a stored secret to an// address the agent chose. Recording requests that were never recorded is why-// getRequestString grew redactors in the same change (audit.go).+// redaction has to cover more than the audited RPCs (audit.go): a denial must+// not transcribe the secret it refused.//// A ceiling the gate cannot act on splits in two, and the split is the same one// the /mcp connection gate makes. A stored value this build cannot interpret —diff --git a/backend/api/v1/mcp_gate_test.go b/backend/api/v1/mcp_gate_test.goindex fd4dd8dbc2..739cdb6210 100644--- a/backend/api/v1/mcp_gate_test.go+++ b/backend/api/v1/mcp_gate_test.go@@ -590,8 +590,8 @@ func TestForbiddenClassLeavesReadsAlone(t *testing.T) {// (project_service_converter.go), which copied the incoming-webhook URL out// verbatim. That URL is a bearer credential — whoever holds it posts into// the customer's Slack, Feishu, WeCom or Teams as Bytebase — and this repo-// already treated it as one: redactWebhook masks it in audit rows-// (audit.go) beside the OIDC client secret and the LDAP bind password.+// already treated it as one: it is annotated SENSITIVE, so audit rows drop+// it beside the OIDC client secret and the LDAP bind password.// ListProjects pages the whole workspace, so one call took the lot. The url// is write-only now, the way a data source's password is: reads leave it// empty. TestWebhook takes the stored url when the request names a webhook@@ -1348,27 +1348,27 @@ var mcpDenialRequestsUnderReview = map[string]mcpDenialRequestReview{// masks, and the gate refuses before dispatch, so it was never even used.v1connect.AIServiceChatProcedure: {[]string{"messages", "tool_definitions"},- "redactAIChatRequest drops the conversation body, which is unbounded and stored untruncated",+ "messages and tool_definitions are annotated OMIT, so the unbounded conversation body is dropped",},v1connect.AuthServiceSwitchWorkspaceProcedure: {[]string{"mfa_temp_token", "otp_code", "recovery_code", "web", "workspace"},- "redactSwitchWorkspaceRequest masks the three MFA proofs; the workspace it targeted and the cookie flag stay",+ "the three MFA proofs are annotated SENSITIVE; the workspace it targeted and the cookie flag stay",},v1connect.IdentityProviderServiceTestIdentityProviderProcedure: {[]string{"identity_provider", "ldap_context", "oauth2_context", "oidc_context"},- "redactTestIdentityProviderRequest masks the provider secret and the test credential",+ "the provider secret and the test credential are annotated SENSITIVE, wherever they appear",},v1connect.InstanceServiceListInstanceDatabaseProcedure: {[]string{"instance"},- "redactInstance masks every data-source credential; pinned by TestAuditRedactsEveryInputOnlyDataSourceField",+ "every data-source credential is annotated SENSITIVE; pinned by TestAuditRedactionCoversEveryAnnotatedField",},v1connect.ProjectServiceTestWebhookProcedure: {[]string{"project", "webhook"},- "redactWebhook masks the URL, which is a bearer credential; the project stays",+ "Webhook.url is annotated SENSITIVE because it is a bearer credential; the project stays",},v1connect.SettingServiceTestEmailSettingProcedure: {[]string{"email_setting", "to"},- "redactEmailSetting masks the SMTP password; the address the agent chose to mail stays, which is the point of the row",+ "the SMTP password is annotated SENSITIVE; the address the agent chose to mail stays, which is the point of the row",},// Recorded as sent. None of it is a credential, and what the row keeps is@@ -1488,11 +1488,16 @@ func mcpRequestFieldNeedsReview(field protoreflect.FieldDescriptor) bool {// on one, joins the population above without anyone deciding what its denial// row may carry.//-// It cannot check that a redactor exists — getRequestString's type switch is-// not readable from here — so it checks the thing that failed twice: that-// somebody looked, at each field rather than at the method as a whole. A new-// unaudited RPC, or a new field on one already reviewed, breaks the build; the-// fix is a redactor plus a row here, or a row here saying why none is needed.+// It checks the thing that failed twice: that somebody looked, at each field+// rather than at the method as a whole. A new unaudited RPC, or a new field on+// one already reviewed, breaks the build; the fix is an audit_behavior+// annotation plus a row here, or a row here saying why none is needed.+//+// It sits alongside the inventory in audit_redact_inventory_test.go rather than+// being replaced by it. The inventory is per (message, string-or-bytes field)+// across the whole audited surface; this is per (method, field) and covers the+// message-typed fields the inventory's scalar scope does not — DiffMetadata's+// target_metadata is a whole schema, and no string field names it.func TestLintDenialRequestsAreReviewedForRedaction(t *testing.T) {needsReview := map[string][]string{}for _, row := range mcpClassificationsFromDescriptors(t) {diff --git a/backend/api/v1/setting_service_directory_sync_test.go b/backend/api/v1/setting_service_directory_sync_test.goindex f23c589c6a..58f9fd8cdb 100644--- a/backend/api/v1/setting_service_directory_sync_test.go+++ b/backend/api/v1/setting_service_directory_sync_test.go@@ -80,8 +80,7 @@ func TestRotateDirectorySyncTokenRejectsMalformedNames(t *testing.T) {func TestRotateDirectorySyncTokenResponseIsRedactedForAudit(t *testing.T) {const token = "b17c0d9e-2222-4000-8000-fedcbafedcba"- got, err := getResponseString(&v1pb.RotateDirectorySyncTokenResponse{Token: token})- require.NoError(t, err)+ got := marshalAuditPayload(&v1pb.RotateDirectorySyncTokenResponse{Token: token})require.NotContains(t, got, token, "minted SCIM token written to the audit log")}diff --git a/docs/design/audit-payload-redaction-from-proto.md b/docs/design/audit-payload-redaction-from-proto.mdindex 4a438199e4..f0c4cb5de1 100644--- a/docs/design/audit-payload-redaction-from-proto.md+++ b/docs/design/audit-payload-redaction-from-proto.md@@ -1,6 +1,6 @@# Audit payload redaction from proto annotations-Status: proposal · 2026-08-20 · addresses BYT-10090+Status: implemented · 2026-08-21 · addresses BYT-10090## Background@@ -157,7 +157,8 @@ lints are the whole enforcement story, and an annotation implying otherwise is w### Redaction-The hook point is `getRequestString` and `getResponseString` themselves, not `WrapUnary`. Unary and+The hook point is `marshalAuditPayload` (originally two functions, `getRequestString` and+`getResponseString`), not `WrapUnary`. Unary andstreaming rows reach redaction only through those two: `auditConnectStreamingConn.Send`(`audit.go:171-193`) builds its own `auditEntry` and calls `createAuditLog` directly. `AdminExecute`is the only streaming RPC, is audited, and its response carries every row of an admin-mode query, so@@ -253,7 +254,7 @@ Constraints:- **Every `Any` that reaches the row is registered and redacted.** The descriptor walk sees`type_url` and `value`, never the packed message, so an annotated field inside one is invisibleto the redactor, the coverage lint and the inventory alike. Three paths put an `Any` on an audit- row and none passes `getRequestString` or `getResponseString`:+ row and none passes `marshalAuditPayload`:| source | call sites today | packed types ||---|---|---|@@ -361,8 +362,8 @@ walking each of the ten rebuilds for what it newly admits, is a precondition for- **Inventory** — the half that makes goal 4 real. A sentinel sweep cannot distinguish anunannotated credential from an ordinary field the audit row intentionally keeps; both surviveredaction identically, so the sweep alone has no oracle. The oracle is a checked-in inventory of- every string and bytes field reachable from a message that can reach `getRequestString` or- `getResponseString`. A field missing from it fails the build; clearing that failure means+ every string and bytes field reachable from a message that can reach `marshalAuditPayload`.+ A field missing from it fails the build; clearing that failure meansannotating the field or recording that it is not a credential. Same shape as`mcpDenialRequestsUnderReview` (`mcp_gate_test.go:1322`), which it replaces once the populationmatches.@@ -494,3 +495,95 @@ darwin/arm64, one run, with `INPUT_ONLY` standing in for the annotation set.- **AIP-147** prescribes patterns, not an annotation: `INPUT_ONLY` plus `OUTPUT_ONLY bool<name>_set`. We follow it inconsistently — `ssl_ca_set` does, `directory_sync_token_configured`does not, and that field is in no tagged release yet.++## Implementation++- **Annotation** — `AuditBehavior` and `audit_behavior = 100010` in+ `proto/v1/v1/annotation.proto`; 77 fields annotated across 17 service protos.+ `OMIT` is defined as "must not be recorded, for any reason", which is the+ reading the rest of this design assumes: it carries `User.phone`, `groups` and+ `profile`, `MaskingReason.semantic_type_icon` and `AIChatRequest.messages`+ alongside the bulk content fields, so deleting the ten rebuilds admits nothing+ new.+- **Redactor** — `backend/api/v1/audit_redact.go`. Plan per message type, cached+ on descriptor identity, with `descend` arms holding the sub-descriptor rather+ than a nested plan so a cycle is followed lazily to any depth; copy-and-share+ by `Range`; maps and repeated messages rebuilt; `SENSITIVE` oneof arms blanked+ and `OMIT` ones dropped. All 35 hand-written redactors are gone. With nothing+ left to switch on, `getRequestString` and `getResponseString` collapse into one+ `marshalAuditPayload`: the annotation says what to do with a field, and which+ side of the call it arrived on does not change that. The hook point is+ unchanged — it is still called from `createAuditLog`, not `WrapUnary`, which is+ what keeps the streaming path covered. The row itself is **not** walked as a+ whole: `service_data` and `status` are redacted where they are assigned, by+ `redactAuditServiceData` and `redactAuditStatus`, and `request_metadata` and+ `mcp_delegation` are assigned untouched because nothing under them is+ annotated. `TestAuditRowNeedsNoRedactionBeyondTheAnyPayloads` asserts that+ stays true.+- **`Any` registry** — `backend/api/v1/audit_redact.go`, keyed by+ destination field rather than by type, so `Status.details` and `service_data`+ permit different things. `TestLintAuditAnyFieldsAreRegistered` enforces the+ descriptor half: a new `Any` field reaching the row must name what may be+ packed in it. The other half — a new type packed into an existing field —+ changes no descriptor, so it is caught at runtime instead, by the `slog.Warn`+ `redactPackedAny` emits naming the type it dropped.+- **Coverage** — the redactor owes the row two things, and the tests are split+ on that line rather than by message type, and both halves live in+ `audit_redact_test.go`. `TestAuditRedactionCoversEveryAnnotatedField`+ sweeps 567 annotated field paths across 321 request/response types, one+ message per field so every oneof arm is exercised on its own rather than only+ the last-numbered one — that is the whole of "the secret leaves", so the+ hand-written per-credential tables that used to assert it by example are gone.+ `TestAuditRowKeepsItsSubstance` in the same file is the other half+ and has to stay hand-written: the sweep asserts absence, so a redactor that+ blanked the entire row would pass it. Mutation-checked in both directions —+ disabling redaction fails 91 subtests, replacing the output with an empty+ message fails all 13 survival rows and leaves the sweep green.+- **Inventory** — `audit_redact_inventory_test.go` holds the declaration lints,+ as against `audit_redact_test.go` which tests the code: 866 recorded+ `(message, string-or-bytes field)` pairs. Annotated fields are absent by+ design, so the list reads as "what the audit row writes down" and a new+ credential lands in it as one line of diff.+- **Read path** — `assertNoInputOnlyValues` is generalized to+ `assertNoSensitiveValues` (INPUT_ONLY ∪ SENSITIVE, traversing map values) and+ runs unconditionally on the converters a test can construct. A static+ call-graph lint over every `v1pb`-producing function in `backend/api/v1` was+ prototyped and dropped: the read path is a different surface from the audit+ row, and a Go AST check that reports a credential leak when a converter is+ merely *renamed* is the wrong price to pay inside this change. If we want it,+ it belongs in its own PR.+- **Kept, not replaced** — `mcpDenialRequestsUnderReview` stays alongside the+ inventory. It is per `(method, field)` and covers the message-typed request+ fields the inventory's scalar scope does not: `DiffMetadata.target_metadata`+ is a whole schema, and no string field names it.+- **Deliberately not fail-open** — `AWSCredential.role_arn` and `external_id`+ are annotated `SENSITIVE`. They are not credentials, and the Redaction section+ above notes that deleting `redactIAMExtension` would start recording them;+ annotating them keeps today's behavior instead of taking that trade silently.+- **Widened on purpose, recorded here rather than discovered later** —+ `redactDataSource` used to replace the whole `external_secret` submessage with+ an empty one; only its credential leaves are annotated, so+ `DataSourceExternalSecret`'s `url`, `engine_name`, `secret_name`,+ `password_key_name`, `mount_path` and the type enums now reach the row. Those+ say WHERE a secret lives, not what it is, and knowing which Vault path an+ instance was pointed at is the kind of thing the row exists for — but a+ customer's internal Vault endpoint and secret path are newly readable by+ anyone with `bb.auditLogs.search`, so this is a sign-off, not a side effect.+- **`QueryResult.messages` is `OMIT`** — the deleted `redactQueryResponse` was+ an allowlist that predated the field, so it dropped it by omission. Engine+ output (`RAISE NOTICE`, `PRINT`, `DBMS_OUTPUT`) is a result-data channel like+ `rows`, and column masking does not reach it.+- **`v1.AuditLog.service_data` is `OMIT`** — otherwise SearchAuditLogs' own+ audit row unpacks, redacts and re-transcribes the before-image of every row+ the caller read, which is the one place cost scaled with payload size.+- **A SENSITIVE scalar with explicit presence is blanked, not dropped.** The+ Redaction section above argues for blanking a oneof arm so the row still+ records which credential was supplied; a proto3 `optional` credential is the+ same question, and `LoginRequest.otp_code` is the case that matters — a failed+ login has to stay distinguishable from one where no second factor was+ presented. Non-optional scalars are still dropped, which protojson renders+ identically.+- **Still uncovered** — `Status.message`, exactly as the Redaction section says.+ `idp_service.go:283` still interpolates an oauth2 error whose `Error()` can+ embed the IdP's raw response body. It has no descriptor, so nothing here+ reaches it; fixing it means deciding what that error should say instead.