Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: avoid UTF-8 validation crash in ListChangelogs for corrupted schema data (#19439)

Vincent Huangcd7bb085add2
diff --git a/backend/migrator/changelog_utf8_test.go b/backend/migrator/changelog_utf8_test.go
new file mode 100644
index 0000000000..b246de68bd
--- /dev/null
+++ b/backend/migrator/changelog_utf8_test.go
@@ -0,0 +1,112 @@
+package migrator
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "unicode/utf8"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+)
+
+// TestListChangelogsWithInvalidUTF8 verifies that reading sync_history.raw_dump
+// containing invalid UTF-8 bytes does not crash. Regression test for SQLSTATE 22021
+// caused by LEFT(raw_dump, N) triggering PostgreSQL UTF-8 validation.
+//
+// The customer reported: "failed to ensure baseline changelog: failed to check for
+// existing changelogs: failed to query: ERROR: invalid byte sequence for encoding
+// "UTF8": 0xe4 0xb8 (SQLSTATE 22021)"
+func TestListChangelogsWithInvalidUTF8(t *testing.T) {
+ ctx := context.Background()
+ container := testcontainer.GetTestPgContainer(ctx, t)
+ t.Cleanup(func() { container.Close(ctx) })
+
+ db := container.GetDB()
+
+ // Create minimal schema with raw_dump as bytea initially.
+ // We insert the invalid bytes as bytea (no encoding validation), then flip
+ // the column type to text via pg_attribute. This simulates how corrupted data
+ // ends up in production (e.g., via server-side INSERT INTO...SELECT during
+ // the 3.2 migration, or older PostgreSQL versions with laxer validation).
+ setup := `
+ CREATE TABLE sync_history (
+ id bigserial PRIMARY KEY,
+ created_at timestamptz NOT NULL DEFAULT now(),
+ instance text NOT NULL,
+ db_name text NOT NULL,
+ metadata json NOT NULL DEFAULT '{}',
+ raw_dump bytea NOT NULL DEFAULT ''::bytea
+ );
+
+ CREATE TABLE changelog (
+ id bigserial PRIMARY KEY,
+ created_at timestamptz NOT NULL DEFAULT now(),
+ instance text NOT NULL,
+ db_name text NOT NULL,
+ status text NOT NULL,
+ sync_history_id bigint REFERENCES sync_history(id),
+ payload jsonb NOT NULL DEFAULT '{}'
+ );
+
+ CREATE TABLE task (
+ id serial PRIMARY KEY,
+ plan_id bigint NOT NULL
+ );
+
+ CREATE TABLE plan (
+ id bigserial PRIMARY KEY,
+ name text NOT NULL
+ );
+
+ -- Insert invalid UTF-8 as bytea (no encoding validation).
+ -- 0xe4 0xb8 is a truncated 3-byte Chinese UTF-8 character (e.g. 中 = 0xe4 0xb8 0xad).
+ INSERT INTO sync_history (instance, db_name, metadata, raw_dump)
+ VALUES ('test-instance', 'test-db', '{}', '\xE4B8'::bytea);
+
+ -- Flip column type from bytea to text via system catalog.
+ -- bytea and text share the same varlena storage format, so the raw bytes
+ -- are preserved but now treated as text — bypassing UTF-8 validation.
+ UPDATE pg_attribute
+ SET atttypid = 'text'::regtype
+ WHERE attrelid = 'sync_history'::regclass
+ AND attname = 'raw_dump';
+
+ INSERT INTO changelog (instance, db_name, status, sync_history_id, payload)
+ VALUES ('test-instance', 'test-db', 'DONE', 1, '{}');
+ `
+ _, err := db.ExecContext(ctx, setup)
+ require.NoError(t, err)
+
+ // Reproduce the bug: LEFT() on text containing invalid UTF-8 triggers SQLSTATE 22021.
+ // This mirrors the query that ListChangelogs generates when ShowFull is false.
+ var schema string
+ err = db.QueryRowContext(ctx,
+ `SELECT COALESCE(LEFT(sh_cur.raw_dump, 512), '')
+ FROM changelog
+ LEFT JOIN sync_history sh_cur ON sh_cur.id = changelog.sync_history_id
+ WHERE changelog.id = $1`,
+ 1,
+ ).Scan(&schema)
+ assert.Error(t, err, "LEFT() on invalid UTF-8 text must fail — confirming the bug exists")
+ if err != nil {
+ assert.Contains(t, err.Error(), "22021", "error must be SQLSTATE 22021 (invalid byte sequence)")
+ }
+
+ // Verify the fix: reading the raw column without LEFT() succeeds because
+ // a plain column reference does not trigger PostgreSQL's UTF-8 string-function validation.
+ err = db.QueryRowContext(ctx,
+ `SELECT COALESCE(sh_cur.raw_dump, '')
+ FROM changelog
+ LEFT JOIN sync_history sh_cur ON sh_cur.id = changelog.sync_history_id
+ WHERE changelog.id = $1`,
+ 1,
+ ).Scan(&schema)
+ require.NoError(t, err, "reading raw_dump without LEFT() must succeed even with invalid UTF-8")
+
+ // After Go-side sanitization, the result must be valid UTF-8.
+ sanitized := strings.ToValidUTF8(schema, "")
+ assert.True(t, utf8.ValidString(sanitized), "sanitized schema must be valid UTF-8")
+}
diff --git a/backend/store/changelog.go b/backend/store/changelog.go
index d40514645a..cddbc5949b 100644
--- a/backend/store/changelog.go
+++ b/backend/store/changelog.go
@@ -2,7 +2,7 @@ package store
import (
"context"
- "fmt"
+ "strings"
"time"
"google.golang.org/protobuf/encoding/protojson"
@@ -50,7 +50,7 @@ type FindChangelogMessage struct {
Limit *int
Offset *int
- // If false, PrevSchema, Schema are truncated
+ // If false, Schema is omitted (empty string).
ShowFull bool
HasSyncHistory bool
}
@@ -128,16 +128,15 @@ func (s *Store) UpdateChangelog(ctx context.Context, update *UpdateChangelogMess
}
func (s *Store) ListChangelogs(ctx context.Context, find *FindChangelogMessage) ([]*ChangelogMessage, error) {
- truncateSize := 512
- if common.IsDev() {
- truncateSize = 4
- }
- shCurField := fmt.Sprintf("LEFT(sh_cur.raw_dump, %d)", truncateSize)
+ // Avoid SQL-level string functions (e.g. LEFT()) on raw_dump — the column may
+ // contain invalid UTF-8 from TiDB/OceanBase schema syncs (SQLSTATE 22021).
+ // BASIC view skips the column entirely; FULL view fetches it and sanitizes in Go.
+ schemaExpr := "''"
if find.ShowFull {
- shCurField = "sh_cur.raw_dump"
+ schemaExpr = "COALESCE(sh_cur.raw_dump, '')"
}
- q := qb.Q().Space(fmt.Sprintf(`
+ q := qb.Q().Space(`
SELECT
changelog.id,
changelog.created_at,
@@ -145,7 +144,7 @@ func (s *Store) ListChangelogs(ctx context.Context, find *FindChangelogMessage)
changelog.db_name,
changelog.status,
changelog.sync_history_id,
- COALESCE(%s, ''),
+ ` + schemaExpr + `,
changelog.payload,
COALESCE(plan.name, '')
FROM changelog
@@ -157,9 +156,7 @@ func (s *Store) ListChangelogs(ctx context.Context, find *FindChangelogMessage)
) task_info ON TRUE
LEFT JOIN plan ON plan.id = task_info.plan_id
WHERE TRUE
- `,
- shCurField,
- ))
+ `)
if v := find.UID; v != nil {
q.And("changelog.id = ?", *v)
@@ -227,6 +224,9 @@ func (s *Store) ListChangelogs(ctx context.Context, find *FindChangelogMessage)
return nil, errors.Wrapf(err, "failed to unmarshal")
}
+ // Sanitize invalid UTF-8 from external database schema dumps (e.g. TiDB/OceanBase with non-UTF-8 Chinese text).
+ c.Schema = strings.ToValidUTF8(c.Schema, "")
+
changelogs = append(changelogs, &c)
}
diff --git a/backend/store/sync_history.go b/backend/store/sync_history.go
index 6820c554e7..8798867f62 100644
--- a/backend/store/sync_history.go
+++ b/backend/store/sync_history.go
@@ -2,6 +2,7 @@ package store
import (
"context"
+ "strings"
"time"
"github.com/pkg/errors"
@@ -60,11 +61,17 @@ func (s *Store) GetSyncHistoryByUID(ctx context.Context, uid int64) (*SyncHistor
return nil, errors.Wrapf(err, "failed to unmarshal")
}
+ // Sanitize invalid UTF-8 from historical rows (e.g. TiDB/OceanBase schema syncs).
+ h.Schema = strings.ToValidUTF8(h.Schema, "")
+
return &h, nil
}
// UpsertDBSchema upserts a database schema.
func (s *Store) CreateSyncHistory(ctx context.Context, instanceID, databaseName string, metadata *storepb.DatabaseSchemaMetadata, schema string) (int64, error) {
+ // Sanitize schema to prevent storing invalid UTF-8 bytes from external databases.
+ schema = strings.ToValidUTF8(schema, "")
+
metadataBytes, err := protojson.Marshal(metadata)
if err != nil {
return 0, errors.Wrapf(err, "failed to marshal")