Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: use latest schema snapshot for changelog diff (#21266)
p0nyb3a23496131a
diff --git a/backend/api/mcp/gen/openapi.yaml b/backend/api/mcp/gen/openapi.yamlindex 55e6c519b8..b49cb4d706 100644--- a/backend/api/mcp/gen/openapi.yaml+++ b/backend/api/mcp/gen/openapi.yaml@@ -14038,12 +14038,13 @@ components:Supported filter:- status: the changelog status, support "==" operation. check Changelog.Status for available values.- - create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+ - has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported.+ - create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.Example:status == "DONE"- status == "FAILED" && type == "SDL"- create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z"+ status == "FAILED"+ has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z"title: ListChangelogsRequestadditionalProperties: falsebytebase.v1.ListChangelogsResponse:diff --git a/backend/api/v1/changelog_service.go b/backend/api/v1/changelog_service.goindex 4b410a6790..e55937ab4a 100644--- a/backend/api/v1/changelog_service.go+++ b/backend/api/v1/changelog_service.go@@ -60,35 +60,49 @@ func parseChangelogFilter(filter string, find *store.FindChangelogMessage) error}case celoperators.Equals:variable, value := getVariableAndValueFromExpr(expr)- strValue, ok := value.(string)- if !ok {- return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected string but found %q", value))- }switch variable {case "status":+ strValue, ok := value.(string)+ if !ok {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected string but found %q", value))+ }v1Status := v1pb.Changelog_Status_value[strValue]storeStatus := convertToChangelogStoreStatus(v1pb.Changelog_Status(v1Status))find.Status = &storeStatus+ case "has_schema_snapshot":+ hasSchemaSnapshot, ok := value.(bool)+ if !ok {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected bool but found %q", value))+ }+ if !hasSchemaSnapshot {+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`"has_schema_snapshot" only supports true`))+ }+ find.HasSyncHistory = truedefault:return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unsupport variable %v", variable))}- case celoperators.GreaterEquals, celoperators.LessEquals:+ case celoperators.GreaterEquals, celoperators.LessEquals, celoperators.Less:variable, rawValue := getVariableAndValueFromExpr(expr)value, ok := rawValue.(string)if !ok {return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("expect string, got %T, hint: filter literals should be string", rawValue))}if variable != "create_time" {- return connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`">=" and "<=" are only supported for "create_time"`))+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf(`">=", "<=", and "<" are only supported for "create_time"`))}t, err := time.Parse(time.RFC3339, value)if err != nil {return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("failed to parse time %v, error: %v", value, err))}- if functionName == celoperators.GreaterEquals {+ switch functionName {+ case celoperators.GreaterEquals:find.CreatedAtAfter = &t- } else {+ case celoperators.LessEquals:find.CreatedAtBefore = &t+ case celoperators.Less:+ find.CreatedAtStrictlyBefore = &t+ default:+ return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))}default:return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("unexpected function %v", functionName))diff --git a/backend/api/v1/changelog_service_test.go b/backend/api/v1/changelog_service_test.goindex a8ccc4cab1..cc4bebbcc0 100644--- a/backend/api/v1/changelog_service_test.go+++ b/backend/api/v1/changelog_service_test.go@@ -11,13 +11,15 @@ import (func TestParseChangelogFilter(t *testing.T) {tests := []struct {- name string- filter string- wantStatus *store.ChangelogStatus- wantAfter *time.Time- wantBefore *time.Time- wantErr bool- errContains string+ name string+ filter string+ wantStatus *store.ChangelogStatus+ wantAfter *time.Time+ wantBefore *time.Time+ wantStrictBefore *time.Time+ wantSchemaSnapshot bool+ wantErr bool+ errContains string}{{name: "empty filter",@@ -38,6 +40,12 @@ func TestParseChangelogFilter(t *testing.T) {filter: `create_time <= "2024-12-31T23:59:59Z"`,wantBefore: ptr(mustParseTime(t, "2024-12-31T23:59:59Z")),},+ {+ name: "schema snapshot before create time",+ filter: `has_schema_snapshot == true && create_time < "2024-12-31T23:59:59.123456789Z"`,+ wantSchemaSnapshot: true,+ wantStrictBefore: ptr(mustParseTime(t, "2024-12-31T23:59:59.123456789Z")),+ },{name: "create_time range",filter: `create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z"`,@@ -60,7 +68,7 @@ func TestParseChangelogFilter(t *testing.T) {name: "time comparison on wrong field",filter: `status >= "2024-01-01T00:00:00Z"`,wantErr: true,- errContains: `">=" and "<=" are only supported for "create_time"`,+ errContains: `">=", "<=", and "<" are only supported for "create_time"`,},{name: "unsupported variable",@@ -102,6 +110,11 @@ func TestParseChangelogFilter(t *testing.T) {require.NotNil(t, find.CreatedAtBefore)require.Equal(t, *tt.wantBefore, *find.CreatedAtBefore)}+ if tt.wantStrictBefore != nil {+ require.NotNil(t, find.CreatedAtStrictlyBefore)+ require.Equal(t, *tt.wantStrictBefore, *find.CreatedAtStrictlyBefore)+ }+ require.Equal(t, tt.wantSchemaSnapshot, find.HasSyncHistory)})}}diff --git a/backend/generated-go/v1/changelog_service.pb.go b/backend/generated-go/v1/changelog_service.pb.goindex 6ef48b3626..a071750267 100644--- a/backend/generated-go/v1/changelog_service.pb.go+++ b/backend/generated-go/v1/changelog_service.pb.go@@ -147,12 +147,13 @@ type ListChangelogsRequest struct {//// Supported filter:// - status: the changelog status, support "==" operation. check Changelog.Status for available values.- // - create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+ // - has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported.+ // - create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.//// Example:// status == "DONE"- // status == "FAILED" && type == "SDL"- // create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z"+ // status == "FAILED"+ // has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z"Filter string `protobuf:"bytes,5,opt,name=filter,proto3" json:"filter,omitempty"`unknownFields protoimpl.UnknownFieldssizeCache protoimpl.SizeCachediff --git a/backend/store/changelog.go b/backend/store/changelog.goindex 0996623ab7..893359fba3 100644--- a/backend/store/changelog.go+++ b/backend/store/changelog.go@@ -44,9 +44,10 @@ type FindChangelogMessage struct {ResourceID *stringDatabaseName *string- Status *ChangelogStatus- CreatedAtBefore *time.Time- CreatedAtAfter *time.Time+ Status *ChangelogStatus+ CreatedAtBefore *time.Time+ CreatedAtAfter *time.Time+ CreatedAtStrictlyBefore *time.TimeLimit *intOffset *int@@ -137,7 +138,7 @@ func (s *Store) UpdateChangelog(ctx context.Context, update *UpdateChangelogMessreturn nil}-func (s *Store) ListChangelogs(ctx context.Context, find *FindChangelogMessage) ([]*ChangelogMessage, error) {+func buildListChangelogsQuery(find *FindChangelogMessage) (string, []any, error) {// Avoid SQL-level string functions (e.g. LEFT()) on raw_dump — the column may// contain invalid UTF-8 from TiDB/OceanBase schema syncs (SQLSTATE 22021).// BASIC view skips the column entirely; FULL view fetches it and sanitizes in Go.@@ -187,8 +188,11 @@ func (s *Store) ListChangelogs(ctx context.Context, find *FindChangelogMessage)if v := find.CreatedAtAfter; v != nil {q.And("changelog.created_at >= ?", *v)}+ if v := find.CreatedAtStrictlyBefore; v != nil {+ q.And("changelog.created_at < ?", *v)+ }- q.Space("ORDER BY changelog.created_at DESC")+ q.Space("ORDER BY changelog.created_at DESC, changelog.resource_id DESC")if v := find.Limit; v != nil {q.Space("LIMIT ?", *v)}@@ -196,7 +200,11 @@ func (s *Store) ListChangelogs(ctx context.Context, find *FindChangelogMessage)q.Space("OFFSET ?", *v)}- query, args, err := q.ToSQL()+ return q.ToSQL()+}++func (s *Store) ListChangelogs(ctx context.Context, find *FindChangelogMessage) ([]*ChangelogMessage, error) {+ query, args, err := buildListChangelogsQuery(find)if err != nil {return nil, errors.Wrapf(err, "failed to build sql")}diff --git a/backend/store/changelog_test.go b/backend/store/changelog_test.goindex 72440ea2a6..4ebd470439 100644--- a/backend/store/changelog_test.go+++ b/backend/store/changelog_test.go@@ -1 +1,29 @@package store++import (+ "strings"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++func TestBuildListChangelogsQuerySchemaSnapshotBefore(t *testing.T) {+ before, err := time.Parse(time.RFC3339Nano, "2024-12-31T23:59:59.123456789Z")+ require.NoError(t, err)+ database := "db"+ limit := 1++ query, args, err := buildListChangelogsQuery(&FindChangelogMessage{+ InstanceID: "instance",+ DatabaseName: &database,+ HasSyncHistory: true,+ CreatedAtStrictlyBefore: &before,+ Limit: &limit,+ })+ require.NoError(t, err)+ require.Contains(t, strings.Join(strings.Fields(query), " "), "changelog.sync_history IS NOT NULL")+ require.Contains(t, strings.Join(strings.Fields(query), " "), "changelog.created_at < $3")+ require.Contains(t, strings.Join(strings.Fields(query), " "), "ORDER BY changelog.created_at DESC, changelog.resource_id DESC LIMIT $4")+ require.Equal(t, []any{"instance", "db", before, 1}, args)+}diff --git a/frontend/src/stores/app/changelog.ts b/frontend/src/stores/app/changelog.tsindex d9ff15b0ab..0e2fda2e36 100644--- a/frontend/src/stores/app/changelog.ts+++ b/frontend/src/stores/app/changelog.ts@@ -1,4 +1,5 @@-import { create as createProto } from "@bufbuild/protobuf";+import { create as createProto, toJson } from "@bufbuild/protobuf";+import { TimestampSchema } from "@bufbuild/protobuf/wkt";import { changelogServiceClientConnect } from "@/api";import { UNKNOWN_ID } from "@/types";import {@@ -189,21 +190,21 @@ export const createChangelogSlice: AppSliceCreator<ChangelogSlice> = (return undefined;}- const { changelogs } = await get().listChangelogs({- parent: database,- pageSize: 1000,- view: ChangelogView.BASIC,- });- const index = changelogs.findIndex(- (changelog) => extractChangelogUID(changelog.name) === currentUid+ const current = await get().getOrFetchChangelogByName(+ name,+ ChangelogView.FULL);- if (index < 0 || index === changelogs.length - 1) {+ if (!current?.createTime) {return undefined;}- return get().getOrFetchChangelogByName(- changelogs[index + 1].name,- ChangelogView.FULL- );+ const createTime = toJson(TimestampSchema, current.createTime);+ const { changelogs } = await get().listChangelogs({+ parent: database,+ pageSize: 1,+ view: ChangelogView.FULL,+ filter: `has_schema_snapshot == true && create_time < "${createTime}"`,+ });+ return changelogs[0];},});diff --git a/frontend/src/stores/app/index.test.ts b/frontend/src/stores/app/index.test.tsindex 689d26e19c..f1eec55ade 100644--- a/frontend/src/stores/app/index.test.ts+++ b/frontend/src/stores/app/index.test.ts@@ -1,4 +1,5 @@import { create as createProto } from "@bufbuild/protobuf";+import { TimestampSchema } from "@bufbuild/protobuf/wkt";import { beforeEach, describe, expect, test, vi } from "vitest";import { silentContextKey } from "@/api/context-key";import { isValidDatabaseGroupName, UNKNOWN_PROJECT_NAME } from "@/types";@@ -439,6 +440,11 @@ const changelogB = createProto(ChangelogSchema, {schema: "full",});+const changelogCreateTime = createProto(TimestampSchema, {+ seconds: 1735689599n,+ nanos: 123456789,+});+const webhookA = createProto(WebhookSchema, {name: "projects/a/webhooks/hook-a",title: "Hook A",@@ -1511,6 +1517,37 @@ describe("useAppStore", () => {).toBeUndefined();});+ test("fetches the latest prior changelog with a schema snapshot", async () => {+ const current = createProto(ChangelogSchema, {+ ...changelogA,+ createTime: changelogCreateTime,+ });+ mocks.getChangelog.mockResolvedValue(current);+ mocks.listChangelogs.mockResolvedValue({ changelogs: [changelogB] });+ const store = createAppStore();++ const previous = await store+ .getState()+ .fetchPreviousChangelog(current.name);++ expect(previous).toBe(changelogB);+ expect(mocks.getChangelog).toHaveBeenCalledWith(+ expect.objectContaining({+ name: current.name,+ view: ChangelogView.FULL,+ })+ );+ expect(mocks.listChangelogs).toHaveBeenCalledWith(+ expect.objectContaining({+ parent: "instances/i1/databases/db1",+ pageSize: 1,+ view: ChangelogView.FULL,+ filter:+ 'has_schema_snapshot == true && create_time < "2024-12-31T23:59:59.123456789Z"',+ })+ );+ });+test("project webhook operations call through and lookup by id", async () => {const projectWithWebhook = createProto(ProjectSchema, {...projectA,diff --git a/frontend/src/types/proto-es/v1/changelog_service_pb.d.ts b/frontend/src/types/proto-es/v1/changelog_service_pb.d.tsindex cc98263578..be95a3f97a 100644--- a/frontend/src/types/proto-es/v1/changelog_service_pb.d.ts+++ b/frontend/src/types/proto-es/v1/changelog_service_pb.d.ts@@ -54,12 +54,13 @@ export declare type ListChangelogsRequest = Message<"bytebase.v1.ListChangelogsR** Supported filter:* - status: the changelog status, support "==" operation. check Changelog.Status for available values.- * - create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+ * - has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported.+ * - create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.** Example:* status == "DONE"- * status == "FAILED" && type == "SDL"- * create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z"+ * status == "FAILED"+ * has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z"** @generated from field: string filter = 5;*/diff --git a/proto/gen/grpc-doc/openapi.yaml b/proto/gen/grpc-doc/openapi.yamlindex 289d22e541..98ddc926ac 100644--- a/proto/gen/grpc-doc/openapi.yaml+++ b/proto/gen/grpc-doc/openapi.yaml@@ -2679,12 +2679,13 @@ paths:Supported filter:- status: the changelog status, support "==" operation. check Changelog.Status for available values.- - create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+ - has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported.+ - create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.Example:status == "DONE"- status == "FAILED" && type == "SDL"- create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z"+ status == "FAILED"+ has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z"schema:type: stringtitle: filter@@ -7818,12 +7819,13 @@ paths:Supported filter:- status: the changelog status, support "==" operation. check Changelog.Status for available values.- - create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+ - has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported.+ - create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.Example:status == "DONE"- status == "FAILED" && type == "SDL"- create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z"+ status == "FAILED"+ has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z"schema:type: stringtitle: filter@@ -25143,12 +25145,13 @@ components:Supported filter:- status: the changelog status, support "==" operation. check Changelog.Status for available values.- - create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+ - has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported.+ - create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.Example:status == "DONE"- status == "FAILED" && type == "SDL"- create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z"+ status == "FAILED"+ has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z"title: ListChangelogsRequestrequired:- parentdiff --git a/proto/gen/grpc-doc/v1/README.md b/proto/gen/grpc-doc/v1/README.mdindex 01191e1160..6f8122f4c9 100644--- a/proto/gen/grpc-doc/v1/README.md+++ b/proto/gen/grpc-doc/v1/README.md@@ -6281,9 +6281,9 @@ When paginating, all other parameters provided must match the call that provided| view | [ChangelogView](#bytebase-v1-ChangelogView) | | || filter | [string](#string) | | Filter is used to filter changelogs returned in the list. The syntax and semantics of CEL are documented at https://github.com/google/cel-spec-Supported filter: - status: the changelog status, support "==" operation. check Changelog.Status for available values. - create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+Supported filter: - status: the changelog status, support "==" operation. check Changelog.Status for available values. - has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported. - create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.-Example: status == "DONE" status == "FAILED" && type == "SDL" create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z" |+Example: status == "DONE" status == "FAILED" has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z" |diff --git a/proto/gen/grpc-doc/v1/index.html b/proto/gen/grpc-doc/v1/index.htmlindex 059dd75b64..c63bf8ee6b 100644--- a/proto/gen/grpc-doc/v1/index.html+++ b/proto/gen/grpc-doc/v1/index.html@@ -17944,12 +17944,13 @@ The syntax and semantics of CEL are documented at https://github.com/google/cel-Supported filter:- status: the changelog status, support "==" operation. check Changelog.Status for available values.-- create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+- has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported.+- create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.Example:status == "DONE"-status == "FAILED" && type == "SDL"-create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z" </p></td>+status == "FAILED"+has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z" </p></td></tr></tbody>diff --git a/proto/v1/v1/changelog_service.proto b/proto/v1/v1/changelog_service.protoindex db640d659a..182dc00e17 100644--- a/proto/v1/v1/changelog_service.proto+++ b/proto/v1/v1/changelog_service.proto@@ -75,12 +75,13 @@ message ListChangelogsRequest {//// Supported filter:// - status: the changelog status, support "==" operation. check Changelog.Status for available values.- // - create_time: the changelog create time in "2006-01-02T15:04:05Z07:00" format, support ">=" or "<=" operator.+ // - has_schema_snapshot: filters to changelogs with a schema snapshot; only "has_schema_snapshot == true" is supported.+ // - create_time: the changelog create time in RFC 3339 format, supports ">=", "<=", or "<" operator.//// Example:// status == "DONE"- // status == "FAILED" && type == "SDL"- // create_time >= "2024-01-01T00:00:00Z" && create_time <= "2024-01-02T00:00:00Z"+ // status == "FAILED"+ // has_schema_snapshot == true && create_time < "2024-01-02T00:00:00Z"string filter = 5;}