Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: load every engine's catalog with omni's LoadMetadata (#21429)
Danny Xu650aa31db836
diff --git a/backend/plugin/advisor/sql_review.go b/backend/plugin/advisor/sql_review.goindex f0d46e2efe..6b7b87e1a7 100644--- a/backend/plugin/advisor/sql_review.go+++ b/backend/plugin/advisor/sql_review.go@@ -133,7 +133,7 @@ func SQLReviewCheck(SessionUser: checkContext.SessionUser,RawSQL: statements,}- if advice := schema.WalkThroughWithContext(checkContext.DBType, walkThroughContext, checkContext.FinalMetadata, asts); advice != nil {+ if advice := schema.WalkThroughWithContext(ctx, checkContext.DBType, walkThroughContext, checkContext.FinalMetadata, asts); advice != nil {for _, rule := range ruleList {if rule.Engine == checkContext.DBType && rule.Type == storepb.SQLReviewRule_BUILTIN_WALK_THROUGH_CHECK {if status, err := NewStatusBySQLReviewRuleLevel(rule.Level); err == nil {diff --git a/backend/plugin/parser/mongodb/completion.go b/backend/plugin/parser/mongodb/completion.goindex 0c6732dbfc..07c0f50121 100644--- a/backend/plugin/parser/mongodb/completion.go+++ b/backend/plugin/parser/mongodb/completion.go@@ -82,16 +82,7 @@ func buildCatalog(ctx context.Context, cCtx base.CompletionContext) *catalog.Catreturn cat}- for _, schema := range metadata.ListSchemaNames() {- schemaMeta := metadata.GetSchemaMetadata(schema)- if schemaMeta == nil {- continue- }- for _, table := range schemaMeta.ListTableNames() {- cat.AddCollection(table)- }- }-+ cat.LoadMetadata(metadata.GetProto())return cat}diff --git a/backend/plugin/parser/partiql/completion.go b/backend/plugin/parser/partiql/completion.goindex 2a9dc6fa5e..db293c6f2e 100644--- a/backend/plugin/parser/partiql/completion.go+++ b/backend/plugin/parser/partiql/completion.go@@ -30,12 +30,7 @@ func Completion(ctx context.Context, cCtx base.CompletionContext, statement stri_, dbMeta, err := cCtx.Metadata(ctx, cCtx.InstanceID, cCtx.DefaultDatabase)if err == nil && dbMeta != nil {databaseMetadata = dbMeta- schema := dbMeta.GetSchemaMetadata("")- if schema != nil {- for _, table := range schema.ListTableNames() {- cat.AddTable(table)- }- }+ cat.LoadMetadata(dbMeta.GetProto())}}diff --git a/backend/plugin/parser/pg/query_span.go b/backend/plugin/parser/pg/query_span.goindex 9dfc978677..0bf1f2e12b 100644--- a/backend/plugin/parser/pg/query_span.go+++ b/backend/plugin/parser/pg/query_span.go@@ -75,20 +75,6 @@ type omniQuerySpanExtractor struct {// fallbackCTEAliases holds CTE column aliases (e.g., WITH t1(cc1, cc2) AS (...)).// Keyed by lowercase CTE name.fallbackCTEAliases map[string][]string-- // Catalog loader state — populated by initCatalog.- // degraded records objects whose real install failed and fell back to- // pseudo. Keyed by loader object key (schema.name with kind prefix).- degraded map[string]error- // trulyBroken records objects whose pseudo install also failed.- trulyBroken map[string]error- // loaderObjects records every object collected from metadata. Used by- // tests to distinguish loader bugs from metadata gaps.- loaderObjects map[string]bool- // lastFallbackReason is the classifier's verdict for the most recent- // AnalyzeSelectStmt error at any of the three fallback sites. Used by- // tests; not consumed by production code.- lastFallbackReason fallbackReason}// newOmniQuerySpanExtractor creates a new omni-based query span extractor.@@ -121,12 +107,9 @@ func (e *omniQuerySpanExtractor) getDatabaseMetadata(database string) (*model.Dareturn meta, nil}-// initCatalog initializes the omni catalog from the database metadata using-// the Catalog loader. Every metadata object is installed in dependency order,-// with an inline pseudo fallback at any failed slot (see query_span_e3_*.go).-//-// The DDL render + batch-parse path used before this refactor is deleted: BYT-9215 and-// BYT-9261 are unreachable by construction here.+// initCatalog loads the database metadata into a new omni catalog. An object+// that fails to install gets a text-typed stand-in or is left out; either way+// the rest of the catalog still loads.func (e *omniQuerySpanExtractor) initCatalog() error {meta, err := e.getDatabaseMetadata(e.defaultDatabase)if err != nil {@@ -140,24 +123,20 @@ func (e *omniQuerySpanExtractor) initCatalog() error {return nil}- loader := newCatalogLoader(e.cat, meta.GetProto())- if err := loader.Load(e.ctx); err != nil {- return errors.Wrapf(err, "catalog loader failed for %q", e.defaultDatabase)+ report, err := e.cat.LoadMetadata(e.ctx, meta.GetProto(), catalog.LoadMetadataOptions{})+ if err != nil {+ return errors.Wrapf(err, "failed to load the catalog for %q", e.defaultDatabase)}- e.degraded = loader.degraded- e.trulyBroken = loader.trulyBroken- e.loaderObjects = loader.loaderObjects-- if len(loader.degraded) > 0 || len(loader.trulyBroken) > 0 {- slog.Debug("catalog loader degraded objects",+ if len(report.Degraded) > 0 || len(report.Missing) > 0 {+ slog.Debug("catalog loaded with stand-ins",slog.String("database", e.defaultDatabase),- slog.Int("degraded", len(loader.degraded)),- slog.Int("truly_broken", len(loader.trulyBroken)),+ slog.Int("degraded", len(report.Degraded)),+ slog.Int("missing", len(report.Missing)),)}// Populate original (non-stubbed) function definitions for PL/pgSQL body- // analysis. This is independent of catalog loading and unchanged by the loader.+ // analysis.for _, s := range meta.GetProto().GetSchemas() {for _, f := range s.GetFunctions() {if f.Definition == "" {@@ -340,10 +319,6 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)query, err := e.cat.AnalyzeSelectStmt(selStmt)if err != nil {- // Record the classifier's verdict for test inspection. This does not- // gate behavior — the fallback below always runs on analyzer errors.- e.lastFallbackReason = classifyAnalyzeError(err)-// Before falling back, try to handle user-defined table-returning functions// that omni's AnalyzeSelectStmt can't resolve (e.g., RETURNS TABLE functions// used as table sources: SELECT * FROM func()).diff --git a/backend/plugin/parser/pg/query_span_composite_test.go b/backend/plugin/parser/pg/query_span_composite_test.godeleted file mode 100644index 4a1c885047..0000000000--- a/backend/plugin/parser/pg/query_span_composite_test.go+++ /dev/null@@ -1,182 +0,0 @@-package pg--import (- "context"- "testing"-- metadatapb "github.com/bytebase/omni/metadata"- "github.com/bytebase/omni/pg/ast"- "github.com/bytebase/omni/pg/catalog"-)--// TestLoaderInstallsCompositeTypes proves the query-span catalog loader-// installs composite types from metadata with real attributes, in dependency-// order (aa_nested sorts before its dependency zz_base alphabetically), and-// that tables typed with them install cleanly.-func TestLoaderInstallsCompositeTypes(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{- {- Name: "public",- EnumTypes: []*metadatapb.EnumTypeMetadata{- {Name: "status", Values: []string{"a", "b"}},- },- CompositeTypes: []*metadatapb.CompositeTypeMetadata{- {- // References zz_base only through an array suffix.- Name: "aa_array_only",- Attributes: []*metadatapb.CompositeTypeAttribute{- {Name: "items", Type: "public.zz_base[]"},- },- },- {- Name: "aa_nested",- Attributes: []*metadatapb.CompositeTypeAttribute{- {Name: "home", Type: "public.zz_base"},- {Name: "s", Type: "public.status"},- },- },- {- Name: "zz_base",- Attributes: []*metadatapb.CompositeTypeAttribute{- {Name: "street", Type: "text"},- {Name: "city", Type: "character varying(50)"},- },- },- },- Tables: []*metadatapb.TableMetadata{- {- Name: "users",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "integer"},- {Name: "home", Type: "public.zz_base", Nullable: true},- },- },- },- },- },- }-- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- loader := newCatalogLoader(cat, meta)- if err := loader.Load(context.Background()); err != nil {- t.Fatalf("Load: %v", err)- }-- base := cat.GetRelation("public", "zz_base")- if base == nil {- t.Fatal("composite type zz_base not installed")- }- if len(base.Columns) != 2 {- t.Errorf("zz_base attribute count: got %d, want 2", len(base.Columns))- }- if base.Columns[0].Name != "street" {- t.Errorf("zz_base first attribute: got %q, want street", base.Columns[0].Name)- }- nested := cat.GetRelation("public", "aa_nested")- if nested == nil {- t.Fatal("composite type aa_nested not installed")- }- if len(nested.Columns) != 2 {- t.Errorf("aa_nested attribute count: got %d, want 2", len(nested.Columns))- }- arrayOnly := cat.GetRelation("public", "aa_array_only")- if arrayOnly == nil {- t.Fatal("composite type aa_array_only not installed")- }- if len(arrayOnly.Columns) != 1 || arrayOnly.Columns[0].Name != "items" {- t.Errorf("aa_array_only must install with its real attribute, got %d columns", len(arrayOnly.Columns))- }- if cat.GetRelation("public", "users") == nil {- t.Fatal("table with composite column not installed")- }-}--// TestLoaderCompositeFallbackPreservesAttributeNames forces the pseudo-// fallback via a domain-typed attribute (domains are not loader objects) and-// asserts the fallback keeps the metadata attribute names.-func TestLoaderCompositeFallbackPreservesAttributeNames(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{- {- Name: "public",- CompositeTypes: []*metadatapb.CompositeTypeMetadata{- {- Name: "with_domain",- Attributes: []*metadatapb.CompositeTypeAttribute{- {Name: "p", Type: "public.pos_int"},- {Name: "note", Type: "text"},- },- },- },- },- },- }-- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- loader := newCatalogLoader(cat, meta)- if err := loader.Load(context.Background()); err != nil {- t.Fatalf("Load: %v", err)- }-- rel := cat.GetRelation("public", "with_domain")- if rel == nil {- t.Fatal("composite must fall back, not vanish")- }- if len(rel.Columns) != 2 {- t.Fatalf("fallback must preserve attribute names, got %d columns", len(rel.Columns))- }- if rel.Columns[0].Name != "p" || rel.Columns[1].Name != "note" {- t.Errorf("fallback attribute names: got %q, %q", rel.Columns[0].Name, rel.Columns[1].Name)- }-}--// TestRangeVarFallbackSkipsCompositeTypes proves the query-span fallback does-// not expand a standalone composite type as a FROM source — PostgreSQL-// rejects composite types as table sources.-func TestRangeVarFallbackSkipsCompositeTypes(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{- {- Name: "public",- CompositeTypes: []*metadatapb.CompositeTypeMetadata{- {- Name: "addr",- Attributes: []*metadatapb.CompositeTypeAttribute{- {Name: "street", Type: "text"},- },- },- },- Tables: []*metadatapb.TableMetadata{- {- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "integer"},- },- },- },- },- },- }-- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- loader := newCatalogLoader(cat, meta)- if err := loader.Load(context.Background()); err != nil {- t.Fatalf("Load: %v", err)- }-- e := &omniQuerySpanExtractor{- cat: cat,- searchPath: []string{"public"},- defaultDatabase: "db",- }- if results := e.extractColumnsFromRangeVar(&ast.RangeVar{Schemaname: "public", Relname: "addr"}); results != nil {- t.Errorf("composite type must not expand as a FROM source, got %d columns", len(results))- }- if results := e.extractColumnsFromRangeVar(&ast.RangeVar{Schemaname: "public", Relname: "t"}); len(results) != 1 {- t.Errorf("real table must still expand, got %d columns", len(results))- }-}diff --git a/backend/plugin/parser/pg/query_span_loader.go b/backend/plugin/parser/pg/query_span_loader.godeleted file mode 100644index 4fd76a1dc3..0000000000--- a/backend/plugin/parser/pg/query_span_loader.go+++ /dev/null@@ -1,745 +0,0 @@-package pg--import (- "cmp"- "context"- "slices"-- "github.com/pkg/errors"-- metadatapb "github.com/bytebase/omni/metadata"- "github.com/bytebase/omni/pg/ast"- "github.com/bytebase/omni/pg/catalog"-)--// catalogLoader installs every schema object from a metadatapb.DatabaseSchemaMetadata-// into an omni catalog, in dependency-topological order, with an inline-// pseudo fallback at each failed slot.-//-// Invariants after Load() returns without error:-// 1. Every object from the metadata is present in the catalog — as either a-// real install or a pseudo install. `loaderObjects` records its identity.-// 2. `degraded[key]` contains the real-install error for objects whose real-// install failed and fell back to pseudo.-// 3. `trulyBroken[key]` contains the pseudo-install error for objects whose-// even-pseudo install failed. These are the few objects callers can-// expect analyzer errors on.-//-// The loader holds no reference to the catalog after Load() completes; the-// catalog object belongs to the caller.-type catalogLoader struct {- cat *catalog.Catalog- meta *metadatapb.DatabaseSchemaMetadata-- // loaderObjects maps canonical object keys (schema.name) to true for- // every object the loader collected from metadata. Used by the classifier- // to tell "loader knew about X and something went wrong" from "X was- // never in metadata."- loaderObjects map[string]bool-- // degraded records objects whose real install failed. The pseudo version- // is in the catalog for these entries.- degraded map[string]error-- // trulyBroken records objects whose pseudo install also failed. The- // catalog has nothing at this slot.- trulyBroken map[string]error-}--// objectKind identifies the kind of an Catalog loader object. It drives the-// install switch in installReal/installPseudo and the dependency-edge-// computation in buildDependencyGraph.-type objectKind int--const (- kindSchema objectKind = iota- kindEnum- kindComposite- kindTable- kindView- kindMatView- kindFunction-)--// objectEntry is the flattened, order-free unit of work the loader processes.-// One entry per schema object. Entries for the same (schema, name) across-// different kinds are allowed (e.g. a table and a function with the same-// name) and treated as independent nodes in the dep graph.-type objectEntry struct {- kind objectKind- schema string- name string-- // Exactly one of the following is populated based on kind.- enumMeta *metadatapb.EnumTypeMetadata- compositeMeta *metadatapb.CompositeTypeMetadata- tableMeta *metadatapb.TableMetadata- viewMeta *metadatapb.ViewMetadata- matViewMeta *metadatapb.MaterializedViewMetadata- funcMeta *metadatapb.FunctionMetadata-}--// key returns a canonical kind-prefixed identifier used for dependency-// lookups and set membership. Tables, views, and matviews share a namespace-// (relation namespace in PG), so they use the same prefix "rel:".-//-// Functions are identified by schema, name AND signature so that overloaded-// names (e.g. `public.fn(int4)` and `public.fn(text)`) do not collide and-// collapse into a single entry during topo-sort bookkeeping.-func (e *objectEntry) key() string {- switch e.kind {- case kindSchema:- return "schema:" + e.schema- case kindEnum, kindComposite:- return "type:" + e.schema + "." + e.name- case kindTable, kindView, kindMatView:- return "rel:" + e.schema + "." + e.name- case kindFunction:- return "func:" + e.schema + "." + e.name + "|" + funcSignatureKey(e.funcMeta)- }- return "unknown:" + e.schema + "." + e.name-}--// sortKey returns the intra-SCC lexicographic key. Matches hard contract C10.-// For functions the signature is appended so overloads have distinct sort-// keys and the install order is fully deterministic.-func (e *objectEntry) sortKey() string {- base := e.schema + "\x00" + e.name + "\x00" + kindLabel(e.kind)- if e.kind == kindFunction {- base += "\x00" + funcSignatureKey(e.funcMeta)- }- return base-}--// funcSignatureKey returns a stable, comparable identifier for a function-// signature. Uses the metadata Signature string when present; falls back to-// the definition hash to keep distinct overloads distinguishable even when-// sync emitted an empty Signature field.-func funcSignatureKey(fn *metadatapb.FunctionMetadata) string {- if fn == nil {- return ""- }- if fn.Signature != "" {- return fn.Signature- }- return fn.Definition-}--func kindLabel(k objectKind) string {- switch k {- case kindSchema:- return "0schema"- case kindEnum:- return "1enum"- case kindComposite:- return "1composite"- case kindTable:- return "2table"- case kindView:- return "3view"- case kindMatView:- return "4matview"- case kindFunction:- return "5function"- }- return "9unknown"-}--// newCatalogLoader returns a loader primed for Load(). It does not touch the-// catalog until Load() is called.-func newCatalogLoader(cat *catalog.Catalog, meta *metadatapb.DatabaseSchemaMetadata) *catalogLoader {- return &catalogLoader{- cat: cat,- meta: meta,- loaderObjects: make(map[string]bool),- degraded: make(map[string]error),- trulyBroken: make(map[string]error),- }-}--// Load collects every object from metadata, topologically orders them, and-// installs each either real or (on failure) pseudo. The returned error is-// non-nil only for catastrophic conditions (nil catalog, ctx cancellation);-// per-object failures are recorded in degraded/trulyBroken and do not stop-// the loader.-func (l *catalogLoader) Load(ctx context.Context) error {- if l.cat == nil {- return errors.New("catalogLoader: nil catalog")- }- if l.meta == nil {- return nil- }-- objects := l.collectObjects()- for _, obj := range objects {- l.loaderObjects[obj.key()] = true- }-- sorted := topoSortObjects(objects)- for _, obj := range sorted {- if err := ctx.Err(); err != nil {- return err- }- l.installOne(obj)- }- return nil-}--// collectObjects flattens DatabaseSchemaMetadata into a list of objectEntry-// values, one per schema object the loader will install. Schemas themselves-// become kindSchema entries so CREATE SCHEMA is part of the topo order.-func (l *catalogLoader) collectObjects() []*objectEntry {- var out []*objectEntry- for _, sm := range l.meta.Schemas {- if sm.Name == "" {- // PG sync sometimes emits a no-schema placeholder; skip.- continue- }- out = append(out, &objectEntry{- kind: kindSchema,- schema: sm.Name,- name: sm.Name,- })- for _, enum := range sm.EnumTypes {- out = append(out, &objectEntry{- kind: kindEnum,- schema: sm.Name,- name: enum.Name,- enumMeta: enum,- })- }- for _, composite := range sm.CompositeTypes {- out = append(out, &objectEntry{- kind: kindComposite,- schema: sm.Name,- name: composite.Name,- compositeMeta: composite,- })- }- for _, tbl := range sm.Tables {- out = append(out, &objectEntry{- kind: kindTable,- schema: sm.Name,- name: tbl.Name,- tableMeta: tbl,- })- }- for _, view := range sm.Views {- out = append(out, &objectEntry{- kind: kindView,- schema: sm.Name,- name: view.Name,- viewMeta: view,- })- }- for _, mv := range sm.MaterializedViews {- out = append(out, &objectEntry{- kind: kindMatView,- schema: sm.Name,- name: mv.Name,- matViewMeta: mv,- })- }- for _, fn := range sm.Functions {- out = append(out, &objectEntry{- kind: kindFunction,- schema: sm.Name,- name: fn.Name,- funcMeta: fn,- })- }- }- return out-}--// topoSortObjects orders objects by their dependency edges so each object is-// processed only after its dependencies. Cycles are broken via Tarjan SCC-// with intra-SCC lexicographic ordering (C10): one member of an SCC installs-// first (and typically fails → pseudo), the rest install against the pseudo.-func topoSortObjects(objects []*objectEntry) []*objectEntry {- if len(objects) == 0 {- return nil- }-- index := make(map[string]*objectEntry, len(objects))- for _, obj := range objects {- index[obj.key()] = obj- }-- edges := buildDependencyEdges(objects, index)- sccs := tarjanSCC(objects, edges)-- // Build a condensed DAG of SCCs, then topo sort and flatten.- sccOf := make(map[string]int, len(objects))- for i, scc := range sccs {- for _, obj := range scc {- sccOf[obj.key()] = i- }- }- condensedEdges := make([][]int, len(sccs))- inDegree := make([]int, len(sccs))- seenEdge := make(map[[2]int]bool)- for src, dsts := range edges {- srcSCC, ok := sccOf[src]- if !ok {- continue- }- for _, dst := range dsts {- dstSCC, ok := sccOf[dst]- if !ok || dstSCC == srcSCC {- continue- }- edge := [2]int{dstSCC, srcSCC} // dst must come before src- if seenEdge[edge] {- continue- }- seenEdge[edge] = true- condensedEdges[dstSCC] = append(condensedEdges[dstSCC], srcSCC)- inDegree[srcSCC]++- }- }-- // Kahn's algorithm on the condensed graph with a deterministic tiebreak:- // ready SCCs are sorted by the lex-smallest sort key of their members.- readyHeap := make([]int, 0)- for i := range sccs {- if inDegree[i] == 0 {- readyHeap = append(readyHeap, i)- }- }- sortSCCsByMin(readyHeap, sccs)-- var flat []*objectEntry- for len(readyHeap) > 0 {- next := readyHeap[0]- readyHeap = readyHeap[1:]- flat = append(flat, sortedSCCMembers(sccs[next])...)- for _, nb := range condensedEdges[next] {- inDegree[nb]--- if inDegree[nb] == 0 {- readyHeap = append(readyHeap, nb)- }- }- sortSCCsByMin(readyHeap, sccs)- }-- // Safety net: if any objects were not emitted (e.g. invariant violation- // in Tarjan / edge building), emit them in deterministic order so the- // loader still covers them.- if len(flat) != len(objects) {- emitted := make(map[string]bool, len(flat))- for _, o := range flat {- emitted[o.key()] = true- }- var missed []*objectEntry- for _, o := range objects {- if !emitted[o.key()] {- missed = append(missed, o)- }- }- slices.SortStableFunc(missed, func(a, b *objectEntry) int {- return cmp.Compare(a.sortKey(), b.sortKey())- })- flat = append(flat, missed...)- }- return flat-}--// buildDependencyEdges returns a map from each object's key to the keys of-// objects it depends on. Only edges whose target actually exists in the-// collected set are recorded; references to unknown objects become no-ops-// (the install will fail and trigger pseudo).-func buildDependencyEdges(objects []*objectEntry, index map[string]*objectEntry) map[string][]string {- edges := make(map[string][]string, len(objects))- for _, obj := range objects {- var deps []string- // Every non-schema object depends on its schema.- if obj.kind != kindSchema {- if key := "schema:" + obj.schema; index[key] != nil {- deps = append(deps, key)- }- }- switch obj.kind {- case kindTable:- for _, col := range obj.tableMeta.Columns {- for _, ref := range extractUserTypeRefs(col.Type) {- if k := typeRefKey(ref); index[k] != nil {- deps = append(deps, k)- }- }- }- case kindView:- for _, dep := range obj.viewMeta.DependencyColumns {- if k := relationKey(dep.Schema, dep.Table); index[k] != nil {- deps = append(deps, k)- }- }- case kindMatView:- for _, dep := range obj.matViewMeta.DependencyColumns {- if k := relationKey(dep.Schema, dep.Table); index[k] != nil {- deps = append(deps, k)- }- }- case kindFunction:- argTypes, _ := parseFunctionSignatureArgTypes(obj.funcMeta.Signature)- for _, at := range argTypes {- for _, ref := range extractUserTypeRefs(at) {- if k := typeRefKey(ref); index[k] != nil {- deps = append(deps, k)- }- }- }- case kindComposite:- for _, attribute := range obj.compositeMeta.Attributes {- for _, ref := range extractUserTypeRefs(attribute.Type) {- if k := typeRefKey(ref); index[k] != nil {- deps = append(deps, k)- }- // An attribute may be typed with a table/view row type.- if k := relationKey(ref.Schema, ref.Name); index[k] != nil {- deps = append(deps, k)- }- }- }- case kindSchema, kindEnum:- // Schemas have no outgoing edges (they are the top of the graph).- // Enums depend only on their schema, already captured above.- default:- // Unknown kinds contribute no edges; install will surface the- // problem at real-install time.- }- if len(deps) > 0 {- edges[obj.key()] = dedupStrings(deps)- }- }- return edges-}--func typeRefKey(r UserTypeRef) string {- return "type:" + r.Schema + "." + r.Name-}--func relationKey(schema, name string) string {- return "rel:" + schema + "." + name-}--func dedupStrings(in []string) []string {- if len(in) < 2 {- return in- }- seen := make(map[string]bool, len(in))- out := in[:0]- for _, s := range in {- if !seen[s] {- seen[s] = true- out = append(out, s)- }- }- return out-}--// tarjanSCC runs Tarjan's strongly-connected-components algorithm on the-// dependency graph and returns SCCs. Each SCC's members are returned in the-// order the algorithm discovered them; callers sort for determinism.-func tarjanSCC(objects []*objectEntry, edges map[string][]string) [][]*objectEntry {- type state struct {- index, low int- onStack bool- }- st := make(map[string]*state, len(objects))- byKey := make(map[string]*objectEntry, len(objects))- for _, obj := range objects {- byKey[obj.key()] = obj- }-- // Walk nodes in deterministic order so tests and shadow diffs reproduce.- keys := make([]string, 0, len(objects))- for _, obj := range objects {- keys = append(keys, obj.key())- }- slices.Sort(keys)-- var (- index int- stack []string- result [][]*objectEntry- )-- var strongconnect func(v string)- strongconnect = func(v string) {- st[v] = &state{index: index, low: index, onStack: true}- index++- stack = append(stack, v)-- for _, w := range edges[v] {- if _, ok := byKey[w]; !ok {- continue- }- if _, seen := st[w]; !seen {- strongconnect(w)- st[v].low = min(st[v].low, st[w].low)- } else if st[w].onStack {- st[v].low = min(st[v].low, st[w].index)- }- }-- if st[v].low == st[v].index {- var scc []*objectEntry- for {- w := stack[len(stack)-1]- stack = stack[:len(stack)-1]- st[w].onStack = false- scc = append(scc, byKey[w])- if w == v {- break- }- }- result = append(result, scc)- }- }-- for _, k := range keys {- if _, seen := st[k]; !seen {- strongconnect(k)- }- }- return result-}--// sortedSCCMembers returns a copy of scc sorted by sortKey for deterministic-// install order.-func sortedSCCMembers(scc []*objectEntry) []*objectEntry {- out := make([]*objectEntry, len(scc))- copy(out, scc)- slices.SortStableFunc(out, func(a, b *objectEntry) int {- return cmp.Compare(a.sortKey(), b.sortKey())- })- return out-}--// sortSCCsByMin sorts a slice of SCC indices in place by the lex-smallest-// sortKey among each SCC's members. Stable ordering.-func sortSCCsByMin(indices []int, sccs [][]*objectEntry) {- slices.SortStableFunc(indices, func(a, b int) int {- return cmp.Compare(minSortKey(sccs[a]), minSortKey(sccs[b]))- })-}--func minSortKey(scc []*objectEntry) string {- if len(scc) == 0 {- return ""- }- best := scc[0].sortKey()- for _, o := range scc[1:] {- if k := o.sortKey(); k < best {- best = k- }- }- return best-}--// installOne attempts a real install; on failure installs a pseudo at the-// same slot. All outcomes are recorded in l.degraded / l.trulyBroken; this-// function never returns an error to the caller because per-object failures-// are not fatal to the loader.-//-// Functions are a special case: when a function's real install fails, we-// deliberately leave the slot empty. Query span has a dedicated fallback for-// user functions (`tryMetadataFuncLookup` in query_span.go) that-// reconstructs a synthetic proc from metadata at query time. Installing a-// text-backed pseudo function would crowd out that path, producing wrong-// overload resolution and losing body-level lineage. Tables/views/types are-// fine to pseudo-install because their lookups only need the object to exist.-func (l *catalogLoader) installOne(obj *objectEntry) {- realErr := l.installReal(obj)- if realErr == nil {- return- }- l.degraded[obj.key()] = realErr- if obj.kind == kindFunction {- // No pseudo — rely on tryMetadataFuncLookup at query time.- return- }- if pErr := l.installPseudo(obj); pErr != nil {- l.trulyBroken[obj.key()] = pErr- }-}--// installReal translates the object's metadata into an AST and calls the-// matching omni DefineX / ExecCreateTableAs / CreateFunctionStmt. Returns-// the first error encountered.-func (l *catalogLoader) installReal(obj *objectEntry) error {- switch obj.kind {- case kindSchema:- err := l.cat.CreateSchemaCommand(&ast.CreateSchemaStmt{- Schemaname: obj.schema,- })- // Built-in schemas (public, pg_catalog, pg_toast) are preloaded by- // catalog.New(). Treating duplicate-schema as success keeps the- // loader's "every metadata object is present post-Load" invariant- // without needing a hardcoded exclusion list.- var cErr *catalog.Error- if errors.As(err, &cErr) && cErr.Code == catalog.CodeDuplicateSchema {- return nil- }- return err- case kindEnum:- return l.cat.DefineEnum(buildCreateEnumStmt(obj.schema, obj.enumMeta))- case kindComposite:- stmt, err := buildCompositeTypeStmt(obj.schema, obj.compositeMeta)- if err != nil {- return err- }- return l.cat.DefineCompositeType(stmt)- case kindTable:- stmt, err := buildCreateStmt(obj.schema, obj.tableMeta)- if err != nil {- return err- }- return l.cat.DefineRelation(stmt, 'r')- case kindView:- stmt, err := buildViewStmt(obj.schema, obj.viewMeta)- if err != nil {- return err- }- return l.cat.DefineView(stmt)- case kindMatView:- stmt, err := buildCreateTableAsStmt(obj.schema, obj.matViewMeta)- if err != nil {- return err- }- return l.cat.ExecCreateTableAs(stmt)- case kindFunction:- stmt, err := buildCreateFunctionStmt(obj.schema, obj.funcMeta)- if err != nil {- return err- }- return l.cat.CreateFunctionStmt(stmt)- }- return errors.Errorf("unknown object kind %d", obj.kind)-}--// installPseudo builds and installs the pseudo variant of an object. Every-// pseudo form is text-backed and free of user-object dependencies, so this-// function essentially never fails; the few error paths exist for defense in-// depth.-func (l *catalogLoader) installPseudo(obj *objectEntry) error {- switch obj.kind {- case kindSchema:- // Schemas have no pseudo form — they either exist or they don't.- // If CreateSchemaCommand failed, treat as trulyBroken.- return errors.New("schema has no pseudo form")- case kindEnum:- return l.cat.DefineEnum(pseudoCreateEnumStmt(obj.schema, obj.name))- case kindComposite:- // Name-preserving, text-backed fallback when metadata attributes are- // known, so (col).field access still resolves; the "_broken" shape- // remains for composites with no attribute information at all.- if len(obj.compositeMeta.GetAttributes()) > 0 {- items := make([]ast.Node, 0, len(obj.compositeMeta.Attributes))- for _, attribute := range obj.compositeMeta.Attributes {- if attribute.Name == "" {- continue- }- items = append(items, &ast.ColumnDef{- Colname: attribute.Name,- TypeName: pseudoTextTypeName(),- })- }- return l.cat.DefineCompositeType(&ast.CompositeTypeStmt{- Typevar: &ast.RangeVar{Schemaname: obj.schema, Relname: obj.name},- Coldeflist: &ast.List{Items: items},- })- }- return l.cat.DefineCompositeType(pseudoCompositeTypeStmt(obj.schema, obj.name))- case kindTable:- cols := columnNamesFromTableMetadata(obj.tableMeta)- return l.cat.DefineRelation(pseudoCreateTableStmt(obj.schema, obj.name, cols), 'r')- case kindView:- cols := viewColumnNames(obj.viewMeta)- stmt, err := pseudoViewStmt(obj.schema, obj.name, cols)- if err != nil {- return err- }- return l.cat.DefineView(stmt)- case kindMatView:- cols := matviewColumnNames(obj.matViewMeta)- stmt, err := pseudoCreateTableAsStmt(obj.schema, obj.name, cols)- if err != nil {- return err- }- return l.cat.ExecCreateTableAs(stmt)- case kindFunction:- argCount := functionArgCountFromSignature(obj.funcMeta.Signature)- return l.cat.CreateFunctionStmt(pseudoCreateFunctionStmt(obj.schema, obj.name, argCount))- }- return errors.Errorf("unknown object kind %d", obj.kind)-}--// columnNamesFromTableMetadata returns the deduplicated column names of a-// table's metadata, in order, for use as pseudo column names.-func columnNamesFromTableMetadata(tbl *metadatapb.TableMetadata) []string {- if tbl == nil {- return nil- }- seen := make(map[string]bool, len(tbl.Columns))- out := make([]string, 0, len(tbl.Columns))- for _, col := range tbl.Columns {- if col.Name == "" || seen[col.Name] {- continue- }- seen[col.Name] = true- out = append(out, col.Name)- }- return out-}--// viewColumnNames returns the pseudo column-name list for a view. Prefers-// explicit Columns (when sync populated them); falls back to the set of-// dependency_columns.column values.-func viewColumnNames(v *metadatapb.ViewMetadata) []string {- if v == nil {- return nil- }- if len(v.Columns) > 0 {- seen := make(map[string]bool, len(v.Columns))- out := make([]string, 0, len(v.Columns))- for _, col := range v.Columns {- if col.Name == "" || seen[col.Name] {- continue- }- seen[col.Name] = true- out = append(out, col.Name)- }- return out- }- seen := make(map[string]bool, len(v.DependencyColumns))- out := make([]string, 0, len(v.DependencyColumns))- for _, dc := range v.DependencyColumns {- if dc.Column == "" || seen[dc.Column] {- continue- }- seen[dc.Column] = true- out = append(out, dc.Column)- }- return out-}--// matviewColumnNames returns pseudo column names for a materialized view.-// MaterializedViewMetadata does not carry a Columns field, so we fall back-// to dependency_columns.-func matviewColumnNames(m *metadatapb.MaterializedViewMetadata) []string {- if m == nil {- return nil- }- seen := make(map[string]bool, len(m.DependencyColumns))- out := make([]string, 0, len(m.DependencyColumns))- for _, dc := range m.DependencyColumns {- if dc.Column == "" || seen[dc.Column] {- continue- }- seen[dc.Column] = true- out = append(out, dc.Column)- }- return out-}diff --git a/backend/plugin/parser/pg/query_span_loader_builders.go b/backend/plugin/parser/pg/query_span_loader_builders.godeleted file mode 100644index 9d4b6c120c..0000000000--- a/backend/plugin/parser/pg/query_span_loader_builders.go+++ /dev/null@@ -1,290 +0,0 @@-package pg--import (- "strings"-- "github.com/pkg/errors"-- metadatapb "github.com/bytebase/omni/metadata"- "github.com/bytebase/omni/pg/ast"-)--// This file translates storepb metadata into omni AST nodes. Each build*-// function is deterministic, allocates fresh nodes, and does not touch the-// catalog. Builders may return errors for inputs the cheat parser rejects;-// the loader catches those errors and installs a pseudo at the same slot.--// buildCreateEnumStmt translates EnumTypeMetadata + schema into a-// CreateEnumStmt. Enum values are preserved as plain string literals.-func buildCreateEnumStmt(schema string, enum *metadatapb.EnumTypeMetadata) *ast.CreateEnumStmt {- vals := make([]ast.Node, 0, len(enum.Values))- for _, v := range enum.Values {- vals = append(vals, &ast.String{Str: v})- }- return &ast.CreateEnumStmt{- TypeName: qualifiedNameList(schema, enum.Name),- Vals: &ast.List{Items: vals},- }-}--// buildCompositeTypeStmt translates CompositeTypeMetadata + schema into a-// CompositeTypeStmt. Attribute types are parsed via typeNameFromString; an-// error on any attribute propagates so the loader can pseudo the whole type.-func buildCompositeTypeStmt(schema string, composite *metadatapb.CompositeTypeMetadata) (*ast.CompositeTypeStmt, error) {- cols := make([]ast.Node, 0, len(composite.Attributes))- for _, attribute := range composite.Attributes {- if attribute.Name == "" {- continue- }- if attribute.Type == "" {- return nil, errors.Errorf("attribute %q: empty type", attribute.Name)- }- tn, err := typeNameFromString(attribute.Type)- if err != nil {- return nil, errors.Wrapf(err, "attribute %q", attribute.Name)- }- cols = append(cols, &ast.ColumnDef{- Colname: attribute.Name,- TypeName: tn,- CollClause: qsCollateClauseFromReference(attribute.Collation),- })- }- return &ast.CompositeTypeStmt{- Typevar: &ast.RangeVar{Schemaname: schema, Relname: composite.Name},- Coldeflist: &ast.List{Items: cols},- }, nil-}--// buildCreateStmt translates TableMetadata into a CreateStmt with real-// column types. Column types are parsed via typeNameFromString; an error on-// any single column propagates back so the loader can pseudo the whole-// relation rather than install a partially-typed catalog entry.-//-// Columns with empty names are skipped.-func buildCreateStmt(schema string, table *metadatapb.TableMetadata) (*ast.CreateStmt, error) {- items := make([]ast.Node, 0, len(table.Columns))- for _, col := range table.Columns {- if col.Name == "" {- continue- }- if col.Type == "" {- return nil, errors.Errorf("column %q: empty type", col.Name)- }- tn, err := typeNameFromString(col.Type)- if err != nil {- return nil, errors.Wrapf(err, "column %q", col.Name)- }- items = append(items, &ast.ColumnDef{- Colname: col.Name,- TypeName: tn,- IsNotNull: !col.Nullable,- })- }- return &ast.CreateStmt{- Relation: &ast.RangeVar{- Schemaname: schema,- Relname: table.Name,- Relpersistence: 'p',- },- TableElts: &ast.List{Items: items},- }, nil-}--// buildViewStmt translates ViewMetadata into a ViewStmt. The view body is-// parsed via ParsePg; the result must be a *ast.SelectStmt (not a RawStmt-// wrapper) because omni's DefineView type-asserts directly.-//-// If the body is empty or unparseable, returns an error; the loader will-// pseudo the view using dependency_columns metadata.-func buildViewStmt(schema string, view *metadatapb.ViewMetadata) (*ast.ViewStmt, error) {- if view.Definition == "" {- return nil, errors.New("empty view definition")- }- sel, err := parseSelectBody(view.Definition)- if err != nil {- return nil, errors.Wrapf(err, "view %q body", view.Name)- }- return &ast.ViewStmt{- View: &ast.RangeVar{- Schemaname: schema,- Relname: view.Name,- Relpersistence: 'p',- },- Query: sel,- }, nil-}--// buildCreateTableAsStmt translates MaterializedViewMetadata into a-// CreateTableAsStmt with Objtype OBJECT_MATVIEW. Same SelectStmt contract-// as buildViewStmt: body must parse to *ast.SelectStmt.-func buildCreateTableAsStmt(schema string, mv *metadatapb.MaterializedViewMetadata) (*ast.CreateTableAsStmt, error) {- if mv.Definition == "" {- return nil, errors.New("empty matview definition")- }- sel, err := parseSelectBody(mv.Definition)- if err != nil {- return nil, errors.Wrapf(err, "matview %q body", mv.Name)- }- return &ast.CreateTableAsStmt{- Query: sel,- Objtype: ast.OBJECT_MATVIEW,- Into: &ast.IntoClause{- Rel: &ast.RangeVar{- Schemaname: schema,- Relname: mv.Name,- Relpersistence: 'p',- },- },- }, nil-}--// buildCreateFunctionStmt translates FunctionMetadata into a-// *ast.CreateFunctionStmt. The primary path parses the function's full-// Definition (an `omni`-compatible CREATE FUNCTION/PROCEDURE statement),-// which preserves parameter types, return-type signatures (including-// RETURNS TABLE / RETURNS SETOF), and the dollar-quoted body verbatim.-//-// When the definition string is missing or cannot be parsed as a-// CreateFunctionStmt, we fall back to a minimal text-typed shape-// reconstructed from the signature string. The fallback loses return-type-// fidelity but keeps analyzer name resolution working for queries that-// reference the function by name.-func buildCreateFunctionStmt(schema string, fn *metadatapb.FunctionMetadata) (*ast.CreateFunctionStmt, error) {- if fn.Definition != "" {- stmts, err := ParsePg(fn.Definition)- if err == nil && len(stmts) == 1 {- node := stmts[0].AST- if raw, ok := node.(*ast.RawStmt); ok {- node = raw.Stmt- }- if parsed, ok := node.(*ast.CreateFunctionStmt); ok {- return parsed, nil- }- }- }- return buildCreateFunctionStmtFromSignature(schema, fn)-}--// buildCreateFunctionStmtFromSignature is the minimal-shape fallback used-// when the full definition is missing or unparseable. Parameters and return-// type collapse to text; the body is a trivial `SELECT NULL::text`.-func buildCreateFunctionStmtFromSignature(schema string, fn *metadatapb.FunctionMetadata) (*ast.CreateFunctionStmt, error) {- argTypes, err := parseFunctionSignatureArgTypes(fn.Signature)- if err != nil {- return nil, errors.Wrapf(err, "function %q signature %q", fn.Name, fn.Signature)- }- params := make([]ast.Node, 0, len(argTypes))- for i, argType := range argTypes {- tn, err := typeNameFromString(argType)- if err != nil {- return nil, errors.Wrapf(err, "function %q arg %d", fn.Name, i)- }- params = append(params, &ast.FunctionParameter{- ArgType: tn,- Mode: ast.FUNC_PARAM_IN,- })- }- stmt := &ast.CreateFunctionStmt{- Funcname: qualifiedNameList(schema, fn.Name),- ReturnType: pseudoTextTypeName(),- }- if len(params) > 0 {- stmt.Parameters = &ast.List{Items: params}- }- stmt.Options = &ast.List{Items: []ast.Node{- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},- &ast.DefElem{- Defname: "as",- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT NULL::text"}}},- },- }}- return stmt, nil-}--// parseSelectBody parses a SQL string expected to be a single SELECT-// statement and returns the *ast.SelectStmt. Rejects empty input, multiple-// statements, and non-SELECT top-level nodes. Unwraps a RawStmt wrapper if-// ParsePg returns one.-func parseSelectBody(sql string) (*ast.SelectStmt, error) {- stmts, err := ParsePg(sql)- if err != nil {- return nil, errors.Wrap(err, "parse")- }- if len(stmts) != 1 {- return nil, errors.Errorf("expected 1 statement, got %d", len(stmts))- }- node := stmts[0].AST- if raw, ok := node.(*ast.RawStmt); ok {- node = raw.Stmt- }- sel, ok := node.(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("expected SelectStmt, got %T", node)- }- return sel, nil-}--// parseFunctionSignatureArgTypes extracts the comma-separated argument types-// from a signature like "fn(integer, text)" or "fn(numeric(10,2), boolean)".-// An empty parameter list returns an empty slice.-func parseFunctionSignatureArgTypes(signature string) ([]string, error) {- open := strings.Index(signature, "(")- if open < 0 {- return nil, nil- }- closeIdx := strings.LastIndex(signature, ")")- if closeIdx < 0 || closeIdx <= open {- return nil, errors.Errorf("signature %q: unbalanced parentheses", signature)- }- return splitTopLevelCommas(signature[open+1 : closeIdx]), nil-}--// splitTopLevelCommas splits a string on commas that are not nested inside-// parentheses. Used for function signature argument lists so that types like-// `numeric(10,2)` are not split.-func splitTopLevelCommas(s string) []string {- s = strings.TrimSpace(s)- if s == "" {- return nil- }- var parts []string- depth := 0- start := 0- for i := 0; i < len(s); i++ {- switch s[i] {- case '(':- depth++- case ')':- if depth > 0 {- depth--- }- case ',':- if depth == 0 {- parts = append(parts, strings.TrimSpace(s[start:i]))- start = i + 1- }- default:- // other bytes are part of the current segment- }- }- parts = append(parts, strings.TrimSpace(s[start:]))- return parts-}--// qsCollateClauseFromReference converts a stored emit-ready collation-// reference (quoted as needed, schema-qualified outside pg_catalog) into an-// AST collate clause. Returns nil for no collation.-func qsCollateClauseFromReference(reference string) *ast.CollateClause {- if reference == "" {- return nil- }- var items []ast.Node- if schemaName, collationName, ok := splitQualifiedName(reference); ok {- items = []ast.Node{&ast.String{Str: schemaName}, &ast.String{Str: collationName}}- } else {- name := strings.Trim(reference, `"`)- name = strings.ReplaceAll(name, `""`, `"`)- items = []ast.Node{&ast.String{Str: name}}- }- return &ast.CollateClause{Collname: &ast.List{Items: items}}-}diff --git a/backend/plugin/parser/pg/query_span_loader_classify.go b/backend/plugin/parser/pg/query_span_loader_classify.godeleted file mode 100644index e10d1d6c71..0000000000--- a/backend/plugin/parser/pg/query_span_loader_classify.go+++ /dev/null@@ -1,83 +0,0 @@-package pg--import (- "github.com/pkg/errors"-- "github.com/bytebase/omni/pg/catalog"-)--// fallbackReason classifies why analyzer analysis fell through to-// extractFallbackColumns / fallbackExtractLineage. It is a pure classification-// of the error from omni's AnalyzeSelectStmt, used by tests to assert which-// branch a given query takes.-//-// This is NOT a runtime gate, a metric, or a telemetry signal. It exists so-// that unit tests can distinguish "pseudo-induced degradation" from-// "undefined reference" from "analyzer does not support this feature", and so-// that debugging from logs carries a meaningful category.-type fallbackReason int--const (- reasonNone fallbackReason = iota- // reasonExpectedPseudoSemantic: analyzer failed because a pseudo-installed- // object looks like text in place of its original user type. Common forms:- // UndefinedFunction (overload signature no longer matches), DatatypeMismatch- // (operator resolution against text-backed pseudo), AmbiguousFunction /- // AmbiguousColumn (candidate pile from pseudo-widening).- reasonExpectedPseudoSemantic- // reasonUndefinedReference: analyzer could not find a relation, column,- // schema, or object. This can be a loader bug (we should have installed- // it) OR a legitimate user error (query typos, stale metadata). Without a- // structured ErrorIdent on *catalog.Error we cannot programmatically- // distinguish the two; humans triage from logs.- reasonUndefinedReference- // reasonAnalyzerUnsupported: non-*catalog.Error internal analyzer failure- // (returned as fmt.Errorf from analyze.go code paths that hit an- // unsupported PG feature or a defensive check).- reasonAnalyzerUnsupported-)--// classifyAnalyzeError buckets an omni AnalyzeSelectStmt error into one of-// the fallbackReason categories. Non-nil errors always return a non-None-// reason; a nil error returns reasonNone.-func classifyAnalyzeError(err error) fallbackReason {- if err == nil {- return reasonNone- }- var cErr *catalog.Error- if !errors.As(err, &cErr) {- return reasonAnalyzerUnsupported- }- switch cErr.Code {- case catalog.CodeUndefinedFunction,- catalog.CodeAmbiguousFunction,- catalog.CodeDatatypeMismatch,- catalog.CodeFeatureNotSupported,- catalog.CodeAmbiguousColumn:- return reasonExpectedPseudoSemantic- case catalog.CodeUndefinedTable,- catalog.CodeUndefinedColumn,- catalog.CodeUndefinedObject,- catalog.CodeUndefinedSchema:- return reasonUndefinedReference- default:- return reasonAnalyzerUnsupported- }-}--// String returns a short lowercase label suitable for test assertions and-// log lines.-func (r fallbackReason) String() string {- switch r {- case reasonNone:- return "none"- case reasonExpectedPseudoSemantic:- return "expected_pseudo_semantic"- case reasonUndefinedReference:- return "undefined_reference"- case reasonAnalyzerUnsupported:- return "analyzer_unsupported"- default:- return "unknown"- }-}diff --git a/backend/plugin/parser/pg/query_span_loader_integration_test.go b/backend/plugin/parser/pg/query_span_loader_integration_test.godeleted file mode 100644index e1e53c3343..0000000000--- a/backend/plugin/parser/pg/query_span_loader_integration_test.go+++ /dev/null@@ -1,671 +0,0 @@-package pg--import (- "context"- "os"- "os/exec"- "runtime/debug"- "testing"-- metadatapb "github.com/bytebase/omni/metadata"- "github.com/bytebase/omni/pg/ast"- "github.com/bytebase/omni/pg/catalog"-- "github.com/bytebase/bytebase/backend/plugin/parser/base"-)--// ----------------------------------------------------------------------------// Catalog loader integration tests-// ----------------------------------------------------------------------------//-// These tests drive the full GetQuerySpan pipeline (parse → initCatalog via-// catalogLoader → AnalyzeSelectStmt → extractLineage) against hand-built-// DatabaseSchemaMetadata fixtures that reproduce the bugs the loader is designed to-// prevent, plus the cascade-prevention story.-//-// Each test asserts one of:-// - A query against an unrelated object succeeds even when a bad object is-// present in the schema (BYT-9215 / BYT-9261 class, blast-radius).-// - A query against a broken-but-pseudo'd object still returns lineage.-// - A cascade chain (bad root → dependent table → dependent view) installs-// downstream objects as real, with query lineage intact.--func TestLoaderIntegration_BYT9215_BadQuotedIdentifier(t *testing.T) {- // BYT-9215 class: a table name contains a character sequence the DDL- // deparse-and-reparse loop chokes on. Under the old path this would kill- // query span for the entire database. Under the loader, the bad table is- // pseudo-installed and unrelated queries succeed.- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Tables: []*metadatapb.TableMetadata{- // A table with a quoted identifier containing an apostrophe —- // the kind of input BYT-9215 reported failing under Exec(ddl).- // Real install should succeed (no DDL roundtrip), but even if- // it didn't, pseudo would catch it.- {- Name: "'weird'table",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- },- },- // An unrelated healthy table.- {- Name: "accounts",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- {Name: "email", Type: "text"},- },- },- },- }},- }-- span := mustGetQuerySpan(t, meta, `SELECT id, email FROM accounts`)- if len(span.Results) != 2 {- t.Fatalf("accounts query: got %d results, want 2", len(span.Results))- }- for _, r := range span.Results {- if len(r.SourceColumns) == 0 {- t.Errorf("result %q has empty sources (lineage lost)", r.Name)- }- }-}--func TestGetQuerySpanWithSelectedSchemaFallsBackToPublic(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{- {- Name: "app",- },- {- Name: "public",- Tables: []*metadatapb.TableMetadata{{- Name: "customer",- Columns: []*metadatapb.ColumnMetadata{- {Name: "ssn", Type: "text"},- },- }},- },- },- }- getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})- span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{- GetDatabaseMetadataFunc: getter,- ListDatabaseNamesFunc: lister,- }, base.Statement{Text: `SELECT ssn FROM customer`}, "db", "app", false)- if err != nil {- t.Fatalf("GetQuerySpan: %v", err)- }- if len(span.Results) != 1 {- t.Fatalf("got %d results, want 1", len(span.Results))- }- want := base.ColumnResource{Database: "db", Schema: "public", Table: "customer", Column: "ssn"}- if _, ok := span.Results[0].SourceColumns[want]; !ok {- t.Fatalf("result %q missing public fallback source %+v; have %+v", span.Results[0].Name, want, span.Results[0].SourceColumns)- }- wantAccess := base.ColumnResource{Database: "db", Schema: "public", Table: "customer"}- if _, ok := span.SourceColumns[wantAccess]; !ok {- t.Fatalf("span missing public fallback access source %+v; have %+v", wantAccess, span.SourceColumns)- }-}--func TestGetQuerySpanSelectedSchemaTakesPrecedenceOverPublic(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{- {- Name: "app",- Tables: []*metadatapb.TableMetadata{{- Name: "customer",- Columns: []*metadatapb.ColumnMetadata{- {Name: "email", Type: "text"},- },- }},- },- {- Name: "public",- Tables: []*metadatapb.TableMetadata{{- Name: "customer",- Columns: []*metadatapb.ColumnMetadata{- {Name: "ssn", Type: "text"},- },- }},- },- },- }- getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})- span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{- GetDatabaseMetadataFunc: getter,- ListDatabaseNamesFunc: lister,- }, base.Statement{Text: `SELECT email FROM customer`}, "db", "app", false)- if err != nil {- t.Fatalf("GetQuerySpan: %v", err)- }- if len(span.Results) != 1 {- t.Fatalf("got %d results, want 1", len(span.Results))- }- want := base.ColumnResource{Database: "db", Schema: "app", Table: "customer", Column: "email"}- if _, ok := span.Results[0].SourceColumns[want]; !ok {- t.Fatalf("result %q missing selected-schema source %+v; have %+v", span.Results[0].Name, want, span.Results[0].SourceColumns)- }-}--func TestLoaderIntegration_BrokenEnumCascade(t *testing.T) {- // Real failure chain: table references a user-defined type that does- // not exist in metadata. Real install of the table fails (omni cannot- // resolve the type). Pseudo install of the table succeeds with text- // columns. Query against the table returns its metadata column names.- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Tables: []*metadatapb.TableMetadata{{- Name: "tasks",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- // References an enum that is NOT declared in metadata —- // buildCreateStmt will succeed (typeNameFromString works)- // but DefineRelation fails (enum not in catalog).- {Name: "status", Type: "public.nonexistent_enum"},- {Name: "title", Type: "text"},- },- }},- }},- }-- span := mustGetQuerySpan(t, meta, `SELECT id, status, title FROM tasks`)- if len(span.Results) != 3 {- t.Fatalf("tasks query: got %d results, want 3", len(span.Results))- }- wantNames := []string{"id", "status", "title"}- for i, want := range wantNames {- if i >= len(span.Results) {- break- }- if span.Results[i].Name != want {- t.Errorf("result[%d]: got %q, want %q", i, span.Results[i].Name, want)- }- if len(span.Results[i].SourceColumns) == 0 {- t.Errorf("result[%d] (%s): empty sources", i, span.Results[i].Name)- }- }-}--func TestLoaderIntegration_BrokenRootTableAndHealthyNeighbor(t *testing.T) {- // A query against a healthy table must succeed even when an unrelated- // table in the same schema references a broken type. This is the core- // blast-radius claim: one bad object does not poison all queries.- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Tables: []*metadatapb.TableMetadata{- {- Name: "broken",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- {Name: "bad", Type: "public.nonexistent_type"},- },- },- {- Name: "healthy",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- {Name: "label", Type: "text"},- },- },- },- }},- }-- span := mustGetQuerySpan(t, meta, `SELECT id, label FROM healthy`)- if len(span.Results) != 2 {- t.Fatalf("healthy query: got %d results, want 2", len(span.Results))- }- for _, r := range span.Results {- if len(r.SourceColumns) == 0 {- t.Errorf("result %q lost lineage because of unrelated broken table", r.Name)- }- }-}--func TestLoaderIntegration_ViewOverBrokenTableStillResolves(t *testing.T) {- // Chain: enum missing → table T references it (degrades to pseudo) →- // view V on T installs real (against pseudo T) → query on V resolves.- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Tables: []*metadatapb.TableMetadata{{- Name: "records",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- {Name: "status", Type: "public.nonexistent_enum"},- {Name: "title", Type: "text"},- },- }},- Views: []*metadatapb.ViewMetadata{{- Name: "records_view",- Definition: "SELECT id, title, status FROM records",- DependencyColumns: []*metadatapb.DependencyColumn{- {Schema: "public", Table: "records", Column: "id"},- {Schema: "public", Table: "records", Column: "title"},- {Schema: "public", Table: "records", Column: "status"},- },- }},- }},- }-- span := mustGetQuerySpan(t, meta, `SELECT id, title, status FROM records_view`)- if len(span.Results) != 3 {- t.Fatalf("view query: got %d results, want 3", len(span.Results))- }- for i, r := range span.Results {- if len(r.SourceColumns) == 0 {- t.Errorf("result[%d] (%s) lost lineage", i, r.Name)- }- }-}--func TestLoaderIntegration_SimpleHealthyPath(t *testing.T) {- // Baseline sanity: a clean schema must produce exact lineage down to- // (schema, table, column) — no degraded flags.- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Tables: []*metadatapb.TableMetadata{{- Name: "users",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- {Name: "email", Type: "text"},- {Name: "created_at", Type: "timestamp with time zone"},- },- }},- }},- }-- span := mustGetQuerySpan(t, meta, `SELECT id, email FROM users WHERE id > 0`)- if len(span.Results) != 2 {- t.Fatalf("got %d results, want 2", len(span.Results))- }- mustHaveExactSource(t, span.Results[0], "users", "id")- mustHaveExactSource(t, span.Results[1], "users", "email")-}--func TestLoaderIntegration_EnumWorksWhenDeclared(t *testing.T) {- // When an enum IS declared in metadata, the table installs as real and- // enum-typed columns resolve through their real type. This is the- // positive control for TestLoaderIntegration_BrokenEnumCascade.- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- EnumTypes: []*metadatapb.EnumTypeMetadata{{- Name: "task_status",- Values: []string{"pending", "running", "done"},- }},- Tables: []*metadatapb.TableMetadata{{- Name: "tasks",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- {Name: "status", Type: "public.task_status"},- },- }},- }},- }-- span := mustGetQuerySpan(t, meta, `SELECT id, status FROM tasks WHERE status = 'pending'`)- if len(span.Results) != 2 {- t.Fatalf("got %d results, want 2", len(span.Results))- }- mustHaveExactSource(t, span.Results[0], "tasks", "id")- mustHaveExactSource(t, span.Results[1], "tasks", "status")-}--func TestLoaderIntegration_CorrelatedRangeFunctionSubquery(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Tables: []*metadatapb.TableMetadata{- {- Name: "compliance_case_record_audits",- Columns: []*metadatapb.ColumnMetadata{- {Name: "case_id", Type: "text"},- {Name: "entity_id", Type: "text"},- {Name: "reason", Type: "text"},- {Name: "remark", Type: "text"},- {Name: "case_record", Type: "jsonb"},- {Name: "reviewer_by", Type: "text"},- {Name: "deleted_at", Type: "timestamptz"},- },- },- {- Name: "compliance_cases",- Columns: []*metadatapb.ColumnMetadata{- {Name: "case_id", Type: "text"},- {Name: "case_info", Type: "jsonb"},- {Name: "deleted_at", Type: "timestamptz"},- },- },- },- }},- }- sql := `-select- a.case_id,- -- c.member_id,- a.entity_id as uid,- a.reason,- a.remark,- (- select elem->>'matchedDateTimeValue'- from jsonb_array_elements(a.case_record::jsonb->'secondaryFieldResults') elem- where elem->>'typeId' = '******'- limit 1- ) as wc_dob,- (- select elem->>'matchedValue'- from jsonb_array_elements(a.case_record::jsonb->'secondaryFieldResults') elem- where elem->>'typeId' = '******'- limit 1- ) as wc_citizenship,- c.case_info->>'birth_date' as kyc_dob,- c.case_info->>'nationality' as kyc_citizenship,- a.reviewer_by as review_by-from compliance_case_record_audits a-inner join compliance_cases c- on c.case_id = a.case_id and c.deleted_at is null-where a.reviewer_by in ('****** ', '******')- and a.deleted_at is null;-`- span := mustGetQuerySpan(t, meta, sql)- if len(span.Results) != 9 {- t.Fatalf("got %d results, want 9", len(span.Results))- }- mustHaveExactSource(t, span.Results[4], "compliance_case_record_audits", "case_record")- mustHaveExactSource(t, span.Results[5], "compliance_case_record_audits", "case_record")- mustHaveExactSource(t, span.Results[6], "compliance_cases", "case_info")- mustHaveExactSource(t, span.Results[7], "compliance_cases", "case_info")-}--func TestLoaderIntegration_MultipleCTEsWithLateralJoinKeepsJSONBLineage(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "db",- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Tables: []*metadatapb.TableMetadata{{- Name: "ai_conversation",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "text"},- {Name: "parts", Type: "jsonb"},- },- }},- }},- }-- tests := []struct {- name string- sql string- }{- {- name: "single_cte",- sql: `-WITH convs AS (- SELECT ac.id, ac.parts AS conv_parts- FROM ai_conversation ac- LIMIT 1-)-SELECT- LEFT(- COALESCE(- (SELECT string_agg(v::text, ' ')- FROM jsonb_array_elements_text(- jsonb_path_query_array(t.part->'bodyData', 'strict $.**.text', '{}'::jsonb, true)- ) AS v),- '<no text>'- ),- 2000- ) AS content-FROM convs c-CROSS JOIN LATERAL jsonb_array_elements(c.conv_parts) WITH ORDINALITY AS t(part, part_ord)-WHERE t.part->>'type' IN ('prompt', 'text');-`,- },- {- name: "multiple_ctes",- sql: `-WITH cte1 AS (- SELECT ac.id, ac.parts AS conv_parts- FROM ai_conversation ac- LIMIT 1-),-cte2 AS (- SELECT id, conv_parts- FROM cte1-)-SELECT- LEFT(- COALESCE(- (SELECT string_agg(v::text, ' ')- FROM jsonb_array_elements_text(- jsonb_path_query_array(t.part->'bodyData', 'strict $.**.text', '{}'::jsonb, true)- ) AS v),- '<no text>'- ),- 2000- ) AS content-FROM cte2 c-CROSS JOIN LATERAL jsonb_array_elements(c.conv_parts) WITH ORDINALITY AS t(part, part_ord)-WHERE t.part->>'type' IN ('prompt', 'text');-`,- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- span := mustGetQuerySpan(t, meta, tt.sql)- if len(span.Results) != 1 {- t.Fatalf("got %d results, want 1", len(span.Results))- }- mustHaveExactSource(t, span.Results[0], "ai_conversation", "parts")- })- }-}--func TestAppendQueryDoesNotMutateSharedStack(t *testing.T) {- outer := &catalog.Query{}- parent := &catalog.Query{}- current := &catalog.Query{}- nested := &catalog.Query{}- queryStack := []*catalog.Query{outer, parent, current}-- nextStack := appendQuery(queryStack[:2], nested)-- if queryStack[2] != current {- t.Fatalf("appendQuery mutated caller stack: got %p, want %p", queryStack[2], current)- }- if len(nextStack) != 3 || nextStack[2] != nested {- t.Fatalf("appendQuery returned unexpected stack: %+v", nextStack)- }-}--func TestLoaderIntegration_BuiltinFunctionSelfReferenceDoesNotOverflow(t *testing.T) {- if os.Getenv("BYTEBASE_TEST_FUNCTION_SELF_REFERENCE") == "1" {- debug.SetMaxStack(1 << 20)- varExpr := &catalog.VarExpr{RangeIdx: 0, AttNum: 1}- q := &catalog.Query{- TargetList: []*catalog.TargetEntry{{- Expr: varExpr,- }},- RangeTable: []*catalog.RangeTableEntry{{- Kind: catalog.RTEFunction,- FuncExprs: []catalog.AnalyzedExpr{&catalog.FuncCallExpr{- Args: []catalog.AnalyzedExpr{&catalog.OpExpr{- Left: varExpr,- Right: &catalog.ConstExpr{- Value: "items",- },- }},- }},- }},- }- extractor := newOmniQuerySpanExtractor(loaderTestDB, []string{loaderTestSchema}, base.GetQuerySpanContext{})- extractor.cat = catalog.New()- extractor.walkExpr(q, q.TargetList[0].Expr, make(base.SourceColumnSet))- return- }-- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_BuiltinFunctionSelfReferenceDoesNotOverflow$")- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_FUNCTION_SELF_REFERENCE=1")- if output, err := cmd.CombinedOutput(); err != nil {- t.Fatalf("walkExpr overflowed on self-referential function RTE: %v\n%s", err, output)- }-}--func TestLoaderIntegration_PlainColumnCycleDoesNotOverflow(t *testing.T) {- if os.Getenv("BYTEBASE_TEST_PLAIN_COLUMN_CYCLE") == "1" {- debug.SetMaxStack(1 << 20)- varExpr := &catalog.VarExpr{RangeIdx: 0, AttNum: 1}- q := &catalog.Query{- TargetList: []*catalog.TargetEntry{{- Expr: varExpr,- }},- RangeTable: []*catalog.RangeTableEntry{{- Kind: catalog.RTESubquery,- }},- }- q.RangeTable[0].Subquery = q- _ = isUltimatelyPlainColumn(q, varExpr)- return- }-- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_PlainColumnCycleDoesNotOverflow$")- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_PLAIN_COLUMN_CYCLE=1")- if output, err := cmd.CombinedOutput(); err != nil {- t.Fatalf("isUltimatelyPlainColumn overflowed on cyclic subquery RTE: %v\n%s", err, output)- }-}--func TestLoaderIntegration_PredicateQueryCycleDoesNotOverflow(t *testing.T) {- if os.Getenv("BYTEBASE_TEST_PREDICATE_QUERY_CYCLE") == "1" {- debug.SetMaxStack(1 << 20)- q := &catalog.Query{}- q.CTEList = []*catalog.CommonTableExprQ{{Query: q}}- extractor := newOmniQuerySpanExtractor(loaderTestDB, []string{loaderTestSchema}, base.GetQuerySpanContext{})- analyzer := &plpgsqlAnalyzer{- extractor: extractor,- scope: newVariableScope(nil),- }- analyzer.collectQueryPredicateColumns(q)- return- }-- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_PredicateQueryCycleDoesNotOverflow$")- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_PREDICATE_QUERY_CYCLE=1")- if output, err := cmd.CombinedOutput(); err != nil {- t.Fatalf("collectQueryPredicateColumns overflowed on cyclic query graph: %v\n%s", err, output)- }-}--func TestLoaderIntegration_SetOpQueryCycleDoesNotOverflow(t *testing.T) {- if os.Getenv("BYTEBASE_TEST_SET_OP_QUERY_CYCLE") == "1" {- debug.SetMaxStack(1 << 20)- q := &catalog.Query{SetOp: catalog.SetOpUnion}- q.LArg = q- q.RArg = &catalog.Query{TargetList: []*catalog.TargetEntry{{- Expr: &catalog.ConstExpr{- Value: "1",- },- }}}- selStmt := &ast.SelectStmt{}- selStmt.Larg = selStmt- selStmt.Rarg = &ast.SelectStmt{}- extractor := newOmniQuerySpanExtractor(loaderTestDB, []string{loaderTestSchema}, base.GetQuerySpanContext{})- extractor.extractLineage(q, selStmt)- return- }-- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_SetOpQueryCycleDoesNotOverflow$")- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_SET_OP_QUERY_CYCLE=1")- if output, err := cmd.CombinedOutput(); err != nil {- t.Fatalf("extractLineage overflowed on cyclic set-op query graph: %v\n%s", err, output)- }-}--func TestLoaderIntegration_FallbackCTECycleDoesNotOverflow(t *testing.T) {- if os.Getenv("BYTEBASE_TEST_FALLBACK_CTE_CYCLE") == "1" {- debug.SetMaxStack(1 << 20)- cteSel := &ast.SelectStmt{- TargetList: &ast.List{Items: []ast.Node{&ast.ResTarget{- Name: "x",- Val: &ast.ColumnRef{Fields: &ast.List{Items: []ast.Node{- &ast.String{Str: "c"},- &ast.String{Str: "x"},- }}},- }}},- FromClause: &ast.List{Items: []ast.Node{&ast.RangeVar{Relname: "c"}}},- }- extractor := newOmniQuerySpanExtractor(loaderTestDB, []string{loaderTestSchema}, base.GetQuerySpanContext{})- extractor.cat = catalog.New()- analyzer := &plpgsqlAnalyzer{- extractor: extractor,- scope: newVariableScope(nil),- cteMap: map[string]*ast.SelectStmt{"c": cteSel},- }- analyzer.resolveThroughCTE(cteSel, "x", nil, make(base.SourceColumnSet))- return- }-- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_FallbackCTECycleDoesNotOverflow$")- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_FALLBACK_CTE_CYCLE=1")- if output, err := cmd.CombinedOutput(); err != nil {- t.Fatalf("resolveThroughCTE overflowed on cyclic fallback CTE: %v\n%s", err, output)- }-}--// ---------- helpers ------------// loaderTestDB / loaderTestSchema are the fixed database/schema names-// every catalog-loader integration test uses; keeping them as constants keeps the call-// sites terse and satisfies the unparam linter.-const (- loaderTestDB = "db"- loaderTestSchema = "public"-)--func mustGetQuerySpan(t *testing.T, meta *metadatapb.DatabaseSchemaMetadata, sql string) *base.QuerySpan {- t.Helper()- getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})- span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{- GetDatabaseMetadataFunc: getter,- ListDatabaseNamesFunc: lister,- }, base.Statement{Text: sql}, loaderTestDB, "", false)- if err != nil {- t.Fatalf("GetQuerySpan(%q): %v", sql, err)- }- if span == nil {- t.Fatalf("GetQuerySpan(%q): nil span", sql)- }- return span-}--func mustHaveExactSource(t *testing.T, result base.QuerySpanResult, table, column string) {- t.Helper()- want := base.ColumnResource{- Database: loaderTestDB,- Schema: loaderTestSchema,- Table: table,- Column: column,- }- for src := range result.SourceColumns {- if src == want {- return- }- }- t.Errorf("result %q missing source %+v; have %+v", result.Name, want, result.SourceColumns)-}diff --git a/backend/plugin/parser/pg/query_span_loader_poc_test.go b/backend/plugin/parser/pg/query_span_loader_poc_test.godeleted file mode 100644index 389dc9246f..0000000000--- a/backend/plugin/parser/pg/query_span_loader_poc_test.go+++ /dev/null@@ -1,2527 +0,0 @@-package pg--// v5 PoC — AST-only catalog population.-//-// This PoC demonstrates that we can:-// 1. Construct a *ast.CreateStmt directly in Go code (no SQL text, no parser).-// 2. Call catalog.DefineRelation on it to install a table into an omni catalog.-// 3. Parse a SELECT query against that table.-// 4. Call AnalyzeSelectStmt and get correct column lineage.-//-// The entire "table install" half of the pipeline never touches pgparser.Parse,-// which is the source of BYT-9215 / BYT-9261 class failures. Only the SELECT-// statement itself is parsed.-//-// If this PoC passes, v5 ("lazy per-object AST install") is the recommended-// architecture for PG query span and supersedes v2/v3/v4 plans.--import (- "fmt"- "strings"- "testing"-- "github.com/pkg/errors"-- "github.com/bytebase/omni/pg/ast"- "github.com/bytebase/omni/pg/catalog"-)--// mustTypeName is a test-only shim around the production typeNameFromString.-// It fails the test on any parse error; production code uses typeNameFromString-// directly and handles the error.-func mustTypeName(t *testing.T, typeStr string) *ast.TypeName {- t.Helper()- tn, err := typeNameFromString(typeStr)- if err != nil {- t.Fatalf("typeNameFromString(%q): %v", typeStr, err)- }- return tn-}--// typeName builds an unqualified PG type name AST node.-// For built-in types, use the PG internal name: int -> int4, text -> text,-// bigint -> int8, varchar -> varchar, timestamp -> timestamp.-func typeName(name string) *ast.TypeName {- return &ast.TypeName{- Names: &ast.List{Items: []ast.Node{&ast.String{Str: name}}},- Typemod: -1,- }-}--// columnDef builds a minimal ColumnDef AST node.-func columnDef(name string, typ *ast.TypeName, notNull bool) *ast.ColumnDef {- return &ast.ColumnDef{- Colname: name,- TypeName: typ,- IsNotNull: notNull,- }-}--// buildCreateTableStmt constructs a *ast.CreateStmt for a simple table,-// purely from Go code with no SQL text. This is the shape that a v5-// catalog loader would produce from bytebase metadata.-//-// parameter signals intent and keeps the helper ready for cross-schema tests.-//-//nolint:unparam // schema is always "public" in these PoCs; keeping the-func buildCreateTableStmt(schema, name string, cols []*ast.ColumnDef) *ast.CreateStmt {- nodes := make([]ast.Node, len(cols))- for i, c := range cols {- nodes[i] = c- }- return &ast.CreateStmt{- Relation: &ast.RangeVar{- Schemaname: schema,- Relname: name,- Relpersistence: 'p',- },- TableElts: &ast.List{Items: nodes},- }-}--// TestLoaderPoC_AstOnlyTableInstallAndAnalyze is the main PoC.-//-// Run with:-//-// go test -v -count=1 -run TestLoaderPoC github.com/bytebase/bytebase/backend/plugin/parser/pg-func TestLoaderPoC_AstOnlyTableInstallAndAnalyze(t *testing.T) {- // Step 1: fresh catalog. public / pg_catalog / pg_toast are preloaded.- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Step 2: hand-construct CREATE TABLE public.plan (- // id int NOT NULL,- // title text,- // status text,- // creator_id int- // ) — no SQL string involved.- planStmt := buildCreateTableStmt("public", "plan", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("title", typeName("text"), false),- columnDef("status", typeName("text"), false),- columnDef("creator_id", typeName("int4"), false),- })-- // Step 3: install via DefineRelation. Second arg 'r' = regular table.- if err := cat.DefineRelation(planStmt, 'r'); err != nil {- t.Fatalf("DefineRelation(plan) failed: %v", err)- }-- // Step 4: install a second table to test JOINs too.- principalStmt := buildCreateTableStmt("public", "principal", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("name", typeName("text"), false),- columnDef("email", typeName("text"), false),- })- if err := cat.DefineRelation(principalStmt, 'r'); err != nil {- t.Fatalf("DefineRelation(principal) failed: %v", err)- }-- // Step 5: verify catalog state — both tables resolvable by name.- if rel := cat.GetRelation("public", "plan"); rel == nil {- t.Fatal("GetRelation(public, plan) returned nil after DefineRelation")- } else if len(rel.Columns) != 4 {- t.Errorf("plan column count: got %d, want 4", len(rel.Columns))- }- if rel := cat.GetRelation("public", "principal"); rel == nil {- t.Fatal("GetRelation(public, principal) returned nil after DefineRelation")- }-- // Step 6: parse a SELECT query that touches both tables.- // The SELECT parser is used here — that's expected and safe (it's not- // the DDL parser that breaks in BYT-9215/9261).- stmts, err := ParsePg(`- SELECT p.id, p.title, principal.name- FROM plan p- JOIN principal ON principal.id = p.creator_id- WHERE p.status = 'open'- `)- if err != nil {- t.Fatalf("ParsePg failed: %v", err)- }- if len(stmts) != 1 {- t.Fatalf("expected 1 statement, got %d", len(stmts))- }- selStmt, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- t.Fatalf("expected *ast.SelectStmt, got %T", stmts[0].AST)- }-- // Step 7: THE KEY CALL — analyze the SELECT against the hand-installed- // catalog. If this returns a non-nil Query with a TargetList matching- // the three SELECT columns, v5 architecture is viable.- query, err := cat.AnalyzeSelectStmt(selStmt)- if err != nil {- t.Fatalf("AnalyzeSelectStmt failed: %v", err)- }- if query == nil {- t.Fatal("AnalyzeSelectStmt returned nil query")- }-- // Step 8: assert lineage is sane — 3 target columns.- if got := len(query.TargetList); got != 3 {- t.Errorf("TargetList length: got %d, want 3", got)- for i, te := range query.TargetList {- t.Logf(" [%d] %s", i, te.ResName)- }- }-- // Step 9: assert each target is a VarExpr (plain column reference)- // pointing at the expected column names.- wantNames := []string{"id", "title", "name"}- for i, want := range wantNames {- if i >= len(query.TargetList) {- break- }- te := query.TargetList[i]- if te.ResName != want {- t.Errorf("TargetList[%d].ResName: got %q, want %q", i, te.ResName, want)- }- if _, isVar := te.Expr.(*catalog.VarExpr); !isVar {- t.Errorf("TargetList[%d].Expr: got %T, want *VarExpr", i, te.Expr)- }- }-- // Step 10: verify JoinTree.Quals is analyzed (WHERE was walked).- // We don't assert exact shape here — just that analysis didn't bail.- if query.JoinTree == nil {- t.Error("JoinTree is nil — analyzer did not build a join tree")- } else if query.JoinTree.Quals == nil {- t.Error("JoinTree.Quals is nil — WHERE clause was not analyzed")- }-- t.Logf("v5 PoC passed: hand-constructed CreateStmt → DefineRelation → " +- "AnalyzeSelectStmt pipeline works end-to-end with no DDL text.")-}--// TestLoaderPoC_BrokenAstIsIsolated demonstrates blast-radius containment:-// installing a table with an unresolvable type fails locally, and queries-// against OTHER tables continue to work. This is the core fidelity claim-// v5 makes over the current whole-schema DDL init path.-func TestLoaderPoC_BrokenAstIsIsolated(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Install a good table.- goodStmt := buildCreateTableStmt("public", "good", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("label", typeName("text"), false),- })- if err := cat.DefineRelation(goodStmt, 'r'); err != nil {- t.Fatalf("DefineRelation(good) failed: %v", err)- }-- // Try to install a broken table — column type does not exist.- brokenStmt := buildCreateTableStmt("public", "broken", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("payload", typeName("nonexistent_type_xyz"), false),- })- err := cat.DefineRelation(brokenStmt, 'r')- if err == nil {- t.Log("note: DefineRelation accepted unresolvable type without error")- t.Log(" this is informational — the analyzer may still reject it at query time")- } else {- t.Logf("DefineRelation(broken) rejected as expected: %v", err)- }-- // Key assertion: the good table is still queryable, regardless of- // whether the broken table was accepted.- if rel := cat.GetRelation("public", "good"); rel == nil {- t.Fatal("good table was lost after broken install attempt")- }-- stmts, err := ParsePg(`SELECT id, label FROM good WHERE id = 1`)- if err != nil {- t.Fatalf("ParsePg failed: %v", err)- }- selStmt, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- t.Fatalf("expected *ast.SelectStmt, got %T", stmts[0].AST)- }-- query, err := cat.AnalyzeSelectStmt(selStmt)- if err != nil {- t.Fatalf("AnalyzeSelectStmt on good table failed after broken install: %v", err)- }- if query == nil || len(query.TargetList) != 2 {- t.Errorf("expected 2 target columns on good table query, got %v", query)- }-- t.Log("v5 PoC blast-radius isolation: broken table did not break queries " +- "against unrelated good table.")-}--// ---------- Risk scenario 1: complex typmods (numeric(10,2)) ------------func TestLoaderPoC_ComplexTypmod(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Build a column with numeric(10,2) via the cheat parser.- stmt := buildCreateTableStmt("public", "orders", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),- columnDef("ts", mustTypeName(t, "timestamp(3) with time zone"), false),- })- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation with complex typmods failed: %v", err)- }-- rel := cat.GetRelation("public", "orders")- if rel == nil || len(rel.Columns) != 3 {- t.Fatalf("orders install incomplete: %+v", rel)- }-- // Query the table — analyzer must resolve each column.- q := mustAnalyze(t, cat, `SELECT id, amount, ts FROM orders WHERE amount > 0`)- if len(q.TargetList) != 3 {- t.Errorf("TargetList: got %d, want 3", len(q.TargetList))- }- t.Log("complex typmod PoC: numeric(10,2) and timestamp(3) with time zone resolve via cheat parser")-}--// ---------- Risk scenario 2: array type ------------func TestLoaderPoC_ArrayType(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- stmt := buildCreateTableStmt("public", "tagged", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("tags", mustTypeName(t, "text[]"), false),- columnDef("scores", mustTypeName(t, "int4[]"), false),- })- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation with array columns failed: %v", err)- }-- q := mustAnalyze(t, cat, `SELECT id, tags FROM tagged`)- if len(q.TargetList) != 2 {- t.Errorf("TargetList: got %d, want 2", len(q.TargetList))- }- t.Log("array type PoC: text[] and int4[] resolve via cheat parser")-}--// ---------- Risk scenario 3: SELECT * expansion ------------func TestLoaderPoC_StarExpansion(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- if err := cat.DefineRelation(buildCreateTableStmt("public", "plan", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("title", typeName("text"), false),- columnDef("status", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("install: %v", err)- }-- q := mustAnalyze(t, cat, `SELECT * FROM plan`)- // With a 3-column table, SELECT * should expand to 3 target entries.- if len(q.TargetList) != 3 {- t.Errorf("SELECT * expansion: got %d targets, want 3", len(q.TargetList))- }- wantCols := []string{"id", "title", "status"}- for i, want := range wantCols {- if i >= len(q.TargetList) {- break- }- if q.TargetList[i].ResName != want {- t.Errorf("target[%d]: got %q, want %q", i, q.TargetList[i].ResName, want)- }- }- t.Log("SELECT * expansion PoC: analyzer expands star against hand-installed columns")-}--// ---------- Risk scenario 4: CTE (WITH clause) ------------func TestLoaderPoC_CTE(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- if err := cat.DefineRelation(buildCreateTableStmt("public", "users", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("name", typeName("text"), false),- columnDef("dept", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("install: %v", err)- }-- q := mustAnalyze(t, cat, `- WITH active AS (- SELECT id, name FROM users WHERE dept = 'eng'- )- SELECT id, name FROM active- `)- if len(q.TargetList) != 2 {- t.Errorf("CTE analyze: got %d targets, want 2", len(q.TargetList))- }- t.Log("CTE PoC: WITH clause analyzed against hand-installed table")-}--// ---------- Risk scenario 5: correlated subquery ------------func TestLoaderPoC_CorrelatedSubquery(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- if err := cat.DefineRelation(buildCreateTableStmt("public", "dept", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("name", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("install dept: %v", err)- }- if err := cat.DefineRelation(buildCreateTableStmt("public", "emp", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("dept_id", typeName("int4"), false),- columnDef("salary", typeName("int4"), false),- }), 'r'); err != nil {- t.Fatalf("install emp: %v", err)- }-- q := mustAnalyze(t, cat, `- SELECT d.name,- (SELECT MAX(e.salary) FROM emp e WHERE e.dept_id = d.id) AS max_sal- FROM dept d- `)- if len(q.TargetList) != 2 {- t.Errorf("correlated: got %d targets, want 2", len(q.TargetList))- }- t.Log("correlated subquery PoC: outer scope d.id resolved from inner subquery")-}--// ---------- Risk scenario 6: user enum type in column ------------func TestLoaderPoC_UserEnumType(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Step 1: install the enum via DefineEnum with hand-constructed stmt.- enumStmt := &ast.CreateEnumStmt{- TypeName: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "task_status"},- }},- Vals: &ast.List{Items: []ast.Node{- &ast.String{Str: "pending"},- &ast.String{Str: "running"},- &ast.String{Str: "done"},- }},- }- if err := cat.DefineEnum(enumStmt); err != nil {- t.Fatalf("DefineEnum failed: %v", err)- }-- // Step 2: install a table whose column uses the enum.- // We go through the cheat parser so the TypeName carries the qualified- // name format omni expects for user types.- stmt := buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("status", mustTypeName(t, "public.task_status"), false),- })- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation with enum column failed: %v", err)- }-- q := mustAnalyze(t, cat, `SELECT id, status FROM tasks WHERE status = 'pending'`)- if len(q.TargetList) != 2 {- t.Errorf("enum-col query: got %d targets, want 2", len(q.TargetList))- }- t.Log("user enum PoC: DefineEnum + DefineRelation with enum-typed column + WHERE filter works")-}--// ---------- Risk scenario 7: view via DefineView with parsed body ------------func TestLoaderPoC_DefineViewWithParsedBody(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Install an underlying table.- if err := cat.DefineRelation(buildCreateTableStmt("public", "plan", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("title", typeName("text"), false),- columnDef("creator_id", typeName("int4"), false),- }), 'r'); err != nil {- t.Fatalf("install plan: %v", err)- }-- // View body is stored as text in bytebase metadata. Parse it once.- viewBody := `SELECT id, title FROM plan WHERE creator_id = 1`- stmts, err := ParsePg(viewBody)- if err != nil {- t.Fatalf("parse view body: %v", err)- }- selStmt, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- t.Fatalf("view body: expected SelectStmt, got %T", stmts[0].AST)- }-- // Construct ViewStmt and install via DefineView.- viewStmt := &ast.ViewStmt{- View: &ast.RangeVar{- Schemaname: "public",- Relname: "plan_mine",- Relpersistence: 'p',- },- Query: selStmt,- }- if err := cat.DefineView(viewStmt); err != nil {- t.Fatalf("DefineView failed: %v", err)- }-- rel := cat.GetRelation("public", "plan_mine")- if rel == nil {- t.Fatal("view not installed")- }- if rel.RelKind != 'v' {- t.Errorf("RelKind: got %c, want 'v'", rel.RelKind)- }-- // Query the view. analyzer should be able to expand the view.- q := mustAnalyze(t, cat, `SELECT id, title FROM plan_mine`)- if len(q.TargetList) != 2 {- t.Errorf("query on view: got %d targets, want 2", len(q.TargetList))- }- t.Log("DefineView PoC: parsed-body ViewStmt + DefineView + query on view works")-}--// ---------- Risk scenario 8: nested view (view on view) ------------func TestLoaderPoC_NestedView(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("user_id", typeName("int4"), false),- columnDef("amount", typeName("int4"), false),- }), 'r'); err != nil {- t.Fatalf("install orders: %v", err)- }-- // View 1: orders_by_user- body1 := `SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id`- sel1 := mustParseSelect(t, body1)- if err := cat.DefineView(&ast.ViewStmt{- View: &ast.RangeVar{Schemaname: "public", Relname: "orders_by_user", Relpersistence: 'p'},- Query: sel1,- }); err != nil {- t.Fatalf("DefineView v1: %v", err)- }-- // View 2: top spenders — references view 1.- body2 := `SELECT user_id, total FROM orders_by_user WHERE total > 100`- sel2 := mustParseSelect(t, body2)- if err := cat.DefineView(&ast.ViewStmt{- View: &ast.RangeVar{Schemaname: "public", Relname: "top_spenders", Relpersistence: 'p'},- Query: sel2,- }); err != nil {- t.Fatalf("DefineView v2 (nested): %v", err)- }-- q := mustAnalyze(t, cat, `SELECT user_id, total FROM top_spenders`)- if len(q.TargetList) != 2 {- t.Errorf("nested view query: got %d targets, want 2", len(q.TargetList))- }- t.Log("nested view PoC: view-on-view resolves correctly via DefineView chain")-}--// ---------- helpers ------------func mustAnalyze(t *testing.T, cat *catalog.Catalog, sql string) *catalog.Query {- t.Helper()- stmts, err := ParsePg(sql)- if err != nil {- t.Fatalf("parse %q: %v", sql, err)- }- selStmt, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- t.Fatalf("expected SelectStmt, got %T", stmts[0].AST)- }- q, err := cat.AnalyzeSelectStmt(selStmt)- if err != nil {- t.Fatalf("AnalyzeSelectStmt(%q): %v", sql, err)- }- return q-}--func mustParseSelect(t *testing.T, sql string) *ast.SelectStmt {- t.Helper()- stmts, err := ParsePg(sql)- if err != nil {- t.Fatalf("parse %q: %v", sql, err)- }- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- t.Fatalf("expected SelectStmt, got %T", stmts[0].AST)- }- return sel-}--// ---------- Risk scenario 9: typeNameFromString with quoted/reserved identifier ------------func TestLoaderPoC_QuotedIdentifierType(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Create an enum with a reserved-ish name.- enumStmt := &ast.CreateEnumStmt{- TypeName: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "user"}, // "user" is a reserved word in PG- }},- Vals: &ast.List{Items: []ast.Node{- &ast.String{Str: "admin"},- &ast.String{Str: "member"},- }},- }- if err := cat.DefineEnum(enumStmt); err != nil {- t.Fatalf("DefineEnum for reserved-word enum failed: %v", err)- }-- // The cheat parser should be able to resolve this through SELECT NULL::"public"."user".- // Note: caller is responsible for quoting — this is documented in typeNameFromString.- tn := mustTypeName(t, `"public"."user"`)- if tn == nil {- t.Fatal("typeNameFromString returned nil for quoted reserved-word type")- }-- // And we can use it in a CREATE TABLE.- stmt := buildCreateTableStmt("public", "accts", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("role", tn, false),- })- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation with quoted-reserved-word enum column failed: %v", err)- }-- q := mustAnalyze(t, cat, `SELECT id, role FROM accts`)- if len(q.TargetList) != 2 {- t.Errorf("quoted-identifier enum query: got %d, want 2", len(q.TargetList))- }- t.Log("quoted identifier PoC: cheat parser handles reserved-word-adjacent type names")-}--// ---------- Risk scenario 10: partitioned table ------------func TestLoaderPoC_PartitionedTable(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Construct CREATE TABLE sales (region text, amount int4) PARTITION BY LIST (region).- stmt := &ast.CreateStmt{- Relation: &ast.RangeVar{- Schemaname: "public",- Relname: "sales",- Relpersistence: 'p',- },- TableElts: &ast.List{Items: []ast.Node{- columnDef("region", typeName("text"), false),- columnDef("amount", typeName("int4"), false),- }},- Partspec: &ast.PartitionSpec{- Strategy: "l", // list- PartParams: &ast.List{Items: []ast.Node{- &ast.PartitionElem{Name: "region"},- }},- },- }- // Note: omni test convention passes 'r' even for partitioned tables;- // omni flips relkind internally based on Partspec presence.- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation partitioned failed: %v", err)- }-- rel := cat.GetRelation("public", "sales")- if rel == nil {- t.Fatal("partitioned table not installed")- }-- q := mustAnalyze(t, cat, `SELECT region, amount FROM sales WHERE region = 'us'`)- if len(q.TargetList) != 2 {- t.Errorf("partitioned table query: got %d, want 2", len(q.TargetList))- }- t.Logf("partitioned table PoC: DefineRelation with Partspec works; relation RelKind=%c", rel.RelKind)-}--// ---------- Risk scenario 11: inherited table ------------func TestLoaderPoC_InheritedTable(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Parent with 2 columns.- parentStmt := buildCreateTableStmt("public", "parent", []*ast.ColumnDef{- columnDef("a", typeName("int4"), false),- columnDef("b", typeName("text"), false),- })- if err := cat.DefineRelation(parentStmt, 'r'); err != nil {- t.Fatalf("install parent failed: %v", err)- }-- // Child with 1 locally-defined column, inheriting from parent.- childStmt := &ast.CreateStmt{- Relation: &ast.RangeVar{- Schemaname: "public",- Relname: "child",- Relpersistence: 'p',- },- InhRelations: &ast.List{Items: []ast.Node{- &ast.RangeVar{Schemaname: "public", Relname: "parent"},- }},- TableElts: &ast.List{Items: []ast.Node{- columnDef("c", typeName("int4"), false),- }},- }- if err := cat.DefineRelation(childStmt, 'r'); err != nil {- t.Fatalf("install child with InhRelations failed: %v", err)- }-- child := cat.GetRelation("public", "child")- if child == nil {- t.Fatal("child relation not installed")- }- // Child should have inherited parent's columns plus its own local column.- if len(child.Columns) < 3 {- t.Errorf("child column count: got %d, want >= 3 (a, b inherited + c local)", len(child.Columns))- for _, col := range child.Columns {- t.Logf(" col: %s", col.Name)- }- }-- // Query should resolve all inherited + local columns.- q := mustAnalyze(t, cat, `SELECT a, b, c FROM child`)- if len(q.TargetList) != 3 {- t.Errorf("inherited query: got %d targets, want 3", len(q.TargetList))- }- t.Log("inherited table PoC: InhRelations merges parent columns into child")-}--// ---------- Risk scenario 12: materialized view via ExecCreateTableAs ------------func TestLoaderPoC_MaterializedView(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("val", typeName("int4"), false),- }), 'r'); err != nil {- t.Fatalf("install t: %v", err)- }-- // Parse the matview body — same pattern as DefineView.- body := `SELECT id, val * 2 AS doubled FROM t WHERE val > 0`- selStmt := mustParseSelect(t, body)-- stmt := &ast.CreateTableAsStmt{- Query: selStmt,- Objtype: ast.OBJECT_MATVIEW,- Into: &ast.IntoClause{- Rel: &ast.RangeVar{- Schemaname: "public",- Relname: "t_doubled",- Relpersistence: 'p',- },- },- }- if err := cat.ExecCreateTableAs(stmt); err != nil {- t.Fatalf("ExecCreateTableAs failed: %v", err)- }-- rel := cat.GetRelation("public", "t_doubled")- if rel == nil {- t.Fatal("matview not installed")- }- if rel.RelKind != 'm' {- t.Errorf("matview RelKind: got %c, want 'm'", rel.RelKind)- }-- q := mustAnalyze(t, cat, `SELECT id, doubled FROM t_doubled`)- if len(q.TargetList) != 2 {- t.Errorf("matview query: got %d, want 2", len(q.TargetList))- }- t.Log("matview PoC: ExecCreateTableAs with OBJECT_MATVIEW works; query on matview analyzes")-}--// ---------- Risk scenario 13: function + overload via CreateFunctionStmt ------------func TestLoaderPoC_FunctionOverload(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Helper to build a CreateFunctionStmt — mirrors omni's makeCreateFuncStmt- // but uses explicit AST nodes.- makeFuncStmt := func(schema, name string, argTypeNames []string, retTypeName, language, body string) *ast.CreateFunctionStmt {- var nameItems []ast.Node- if schema != "" {- nameItems = append(nameItems, &ast.String{Str: schema})- }- nameItems = append(nameItems, &ast.String{Str: name})-- var params []ast.Node- for _, argTypeStr := range argTypeNames {- params = append(params, &ast.FunctionParameter{- ArgType: typeName(argTypeStr),- Mode: ast.FUNC_PARAM_IN,- })- }-- stmt := &ast.CreateFunctionStmt{- Funcname: &ast.List{Items: nameItems},- ReturnType: typeName(retTypeName),- }- if len(params) > 0 {- stmt.Parameters = &ast.List{Items: params}- }-- var opts []ast.Node- if language != "" {- opts = append(opts, &ast.DefElem{Defname: "language", Arg: &ast.String{Str: language}})- }- if body != "" {- opts = append(opts, &ast.DefElem{- Defname: "as",- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: body}}},- })- }- if len(opts) > 0 {- stmt.Options = &ast.List{Items: opts}- }- return stmt- }-- // Install overload 1: my_double(int4) returns int4.- err := cat.CreateFunctionStmt(makeFuncStmt("public", "my_double",- []string{"int4"}, "int4", "sql", "SELECT $1 * 2"))- if err != nil {- t.Fatalf("install my_double(int4) failed: %v", err)- }-- // Install overload 2: my_double(int8) returns int8. Different signature,- // must coexist (this is codex's D3 verification — sequential- // CreateFunctionStmt calls install all overloads).- err = cat.CreateFunctionStmt(makeFuncStmt("public", "my_double",- []string{"int8"}, "int8", "sql", "SELECT $1 * 2"))- if err != nil {- t.Fatalf("install my_double(int8) failed: %v", err)- }-- // Both overloads must be visible.- procs := cat.LookupProcByName("my_double")- if len(procs) < 2 {- t.Fatalf("overload count: got %d, want >= 2", len(procs))- }-- // Install a table and query using the function.- if err := cat.DefineRelation(buildCreateTableStmt("public", "nums", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("n", typeName("int4"), false),- }), 'r'); err != nil {- t.Fatalf("install nums: %v", err)- }-- q := mustAnalyze(t, cat, `SELECT id, my_double(n) AS doubled FROM nums`)- if len(q.TargetList) != 2 {- t.Errorf("function query: got %d, want 2", len(q.TargetList))- }- t.Logf("function overload PoC: 2 overloads installed sequentially, analyzer picks correctly. %d procs visible under 'my_double'", len(procs))-}--// ---------- Risk scenario 14: composite type + column referencing it ------------func TestLoaderPoC_CompositeType(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Install a composite type: address (street text, city text, zip text).- compStmt := &ast.CompositeTypeStmt{- Typevar: &ast.RangeVar{- Schemaname: "public",- Relname: "address",- },- Coldeflist: &ast.List{Items: []ast.Node{- columnDef("street", typeName("text"), false),- columnDef("city", typeName("text"), false),- columnDef("zip", typeName("text"), false),- }},- }- if err := cat.DefineCompositeType(compStmt); err != nil {- t.Fatalf("DefineCompositeType failed: %v", err)- }-- // Install a table with a column of the composite type.- stmt := buildCreateTableStmt("public", "customers", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("addr", mustTypeName(t, "public.address"), false),- })- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation with composite column failed: %v", err)- }-- q := mustAnalyze(t, cat, `SELECT id, addr FROM customers`)- if len(q.TargetList) != 2 {- t.Errorf("composite-col query: got %d, want 2", len(q.TargetList))- }- t.Log("composite type PoC: DefineCompositeType + DefineRelation with composite column works")-}--// ---------- Risk scenario 15: domain type + column referencing it ------------func TestLoaderPoC_DomainType(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Install a domain: positive_int, base int4, NOT NULL.- domainStmt := &ast.CreateDomainStmt{- Domainname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "positive_int"},- }},- Typname: typeName("int4"),- Constraints: &ast.List{Items: []ast.Node{- &ast.Constraint{Contype: ast.CONSTR_NOTNULL},- }},- }- if err := cat.DefineDomain(domainStmt); err != nil {- t.Fatalf("DefineDomain failed: %v", err)- }-- // Install a table with a column of the domain type.- stmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("count", mustTypeName(t, "public.positive_int"), true),- })- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation with domain column failed: %v", err)- }-- q := mustAnalyze(t, cat, `SELECT id, count FROM counters WHERE count > 0`)- if len(q.TargetList) != 2 {- t.Errorf("domain-col query: got %d, want 2", len(q.TargetList))- }- t.Log("domain type PoC: DefineDomain + DefineRelation with domain column works")-}--// ---------- Exploration: does query span actually need real types? -----------//-// Hypothesis: for query span purposes, we only need column NAMES, not real-// column TYPES. Lineage is a name-resolution problem, not a type-resolution-// problem. If we install every column as "text" (regardless of real metadata-// type), most queries should still analyze correctly and produce identical-// lineage output.-//-// Why this matters: if true, v5 can skip typeNameFromString entirely, skip-// the user enum / domain / composite problem entirely, and reduce the-// builder function library to a trivial "table with column names" shape.-//-// This test runs a battery of queries against tables where all columns are-// declared as text, and asks: (a) does AnalyzeSelectStmt succeed, (b) is the-// lineage output the same as with correct types? A query that fails here-// would fall through to extractFallbackColumns today (same as the current-// path does on any analyzer error).--func TestLoaderPoC_TypesDontMatterForLineage(t *testing.T) {- // Build two catalogs: one with "honest" types, one with everything as text.- // Run the same queries against both and compare lineage.-- installHonest := func(cat *catalog.Catalog) {- // orders (id int, amount numeric(10,2), status text, placed_at timestamptz, tags text[])- if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),- columnDef("status", typeName("text"), false),- columnDef("placed_at", mustTypeName(t, "timestamp with time zone"), false),- columnDef("tags", mustTypeName(t, "text[]"), false),- }), 'r'); err != nil {- t.Fatalf("honest install: %v", err)- }- // customers (id int, email text)- if err := cat.DefineRelation(buildCreateTableStmt("public", "customers", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("email", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("honest install customers: %v", err)- }- }-- installLying := func(cat *catalog.Catalog) {- // Everything text, everything. Even numeric/timestamp/array columns.- // This is the minimum information the loader would need if we decided- // bytebase metadata should not bother with per-column type strings.- if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{- columnDef("id", typeName("text"), true),- columnDef("amount", typeName("text"), false),- columnDef("status", typeName("text"), false),- columnDef("placed_at", typeName("text"), false),- columnDef("tags", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("lying install: %v", err)- }- if err := cat.DefineRelation(buildCreateTableStmt("public", "customers", []*ast.ColumnDef{- columnDef("id", typeName("text"), true),- columnDef("email", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("lying install customers: %v", err)- }- }-- catHonest := catalog.New()- catHonest.SetSearchPath([]string{"public"})- installHonest(catHonest)-- catLying := catalog.New()- catLying.SetSearchPath([]string{"public"})- installLying(catLying)-- // Each test case: a query, a description, an "honest expected to pass" flag.- // We record (for each catalog) whether AnalyzeSelectStmt succeeded and what- // the lineage looked like, then compare.- cases := []struct {- name string- query string- // expectedHonest is whether we expect the honest catalog to analyze successfully.- // All our cases are expected to succeed on the honest catalog.- }{- // Category 1: Name resolution only — no operator semantics involved- {"plain_select", `SELECT id, status FROM orders`},- {"select_star", `SELECT * FROM orders`},- {"join", `SELECT o.id, c.email FROM orders o JOIN customers c ON c.id = o.id`},- {"alias", `SELECT o.id AS order_id FROM orders o`},-- // Category 2: text-friendly operators (=, LIKE, IS NULL)- {"text_equality", `SELECT id FROM orders WHERE status = 'open'`},- {"text_like", `SELECT id FROM orders WHERE status LIKE 'op%'`},- {"is_null", `SELECT id FROM orders WHERE status IS NULL`},-- // Category 3: numeric operators against text — will omni coerce or fail?- {"numeric_compare", `SELECT id FROM orders WHERE amount > 100`},- {"numeric_arithmetic", `SELECT id, amount + 1 AS amount_plus FROM orders`},- {"numeric_sum", `SELECT SUM(amount) FROM orders GROUP BY status`},-- // Category 4: date/time ops against text- {"date_compare", `SELECT id FROM orders WHERE placed_at > '2024-01-01'::date`},- {"date_extract", `SELECT EXTRACT(year FROM placed_at::timestamp) FROM orders`},-- // Category 5: array ops against text- {"array_contains", `SELECT id FROM orders WHERE tags @> ARRAY['urgent']`},- {"array_unnest", `SELECT id, unnest(tags) FROM orders`},-- // Category 6: subquery + CTE (name resolution heavy)- {"cte", `WITH active AS (SELECT id, status FROM orders WHERE status = 'open') SELECT id FROM active`},- {"subquery_where", `SELECT id FROM orders WHERE id IN (SELECT id FROM customers)`},- {"correlated", `SELECT id, (SELECT email FROM customers c WHERE c.id = orders.id) AS email FROM orders`},-- // Category 7: aggregations- {"count_star", `SELECT COUNT(*) FROM orders`},- {"count_col", `SELECT COUNT(id) FROM orders`},- {"group_by", `SELECT status, COUNT(*) FROM orders GROUP BY status`},- }-- type result struct {- ok bool- err string- nTargets int- names []string- }-- runOne := func(cat *catalog.Catalog, sql string) result {- stmts, err := ParsePg(sql)- if err != nil {- return result{err: "parse: " + err.Error()}- }- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- return result{err: "not a SelectStmt"}- }- q, err := cat.AnalyzeSelectStmt(sel)- if err != nil {- return result{err: "analyze: " + err.Error()}- }- r := result{ok: true, nTargets: len(q.TargetList)}- for _, te := range q.TargetList {- r.names = append(r.names, te.ResName)- }- return r- }-- var bothOk, honestOnlyOk, lyingOnlyOk, bothFail, lineageDiverges int- t.Log("---- query span type-free install exploration ----")- t.Logf("%-22s %-8s %-8s %s", "case", "honest", "lying", "note")- for _, tc := range cases {- h := runOne(catHonest, tc.query)- l := runOne(catLying, tc.query)-- honestMark := "FAIL"- if h.ok {- honestMark = "PASS"- }- lyingMark := "FAIL"- if l.ok {- lyingMark = "PASS"- }-- note := ""- if h.ok && l.ok {- bothOk++- // Compare lineage shape (target count + names).- if h.nTargets != l.nTargets {- note = fmt.Sprintf("TARGETS DIFFER h=%d l=%d", h.nTargets, l.nTargets)- lineageDiverges++- } else {- same := true- for i := range h.names {- if h.names[i] != l.names[i] {- same = false- break- }- }- if !same {- note = fmt.Sprintf("NAMES DIFFER h=%v l=%v", h.names, l.names)- lineageDiverges++- } else {- note = fmt.Sprintf("IDENTICAL (%d cols)", h.nTargets)- }- }- } else if h.ok && !l.ok {- honestOnlyOk++- note = "lying fails: " + truncate(l.err, 60)- } else if !h.ok && l.ok {- lyingOnlyOk++- note = "honest fails: " + truncate(h.err, 60)- } else {- bothFail++- note = "both fail"- }-- t.Logf("%-22s %-8s %-8s %s", tc.name, honestMark, lyingMark, note)- }-- t.Log("")- t.Logf("summary: both_pass=%d honest_only=%d lying_only=%d both_fail=%d diverge=%d",- bothOk, honestOnlyOk, lyingOnlyOk, bothFail, lineageDiverges)- t.Log("")- t.Log("key question: for cases where both paths analyze successfully, does lineage match?")- t.Log("key question: for cases where only honest analyzes, can fallback recover equal-or-better lineage?")-}--func truncate(s string, n int) string {- if len(s) <= n {- return s- }- return s[:n] + "..."-}--// TestLoaderPoC_UnknownPseudoType explores whether PG's 'unknown' pseudo-type-// can be used as a universal column type that accepts all operators.-func TestLoaderPoC_UnknownPseudoType(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- tries := []string{"unknown", "anyelement", "any"}- for _, ty := range tries {- stmt := buildCreateTableStmt("public", "t_"+ty, []*ast.ColumnDef{- {Colname: "id", TypeName: typeName(ty)},- })- err := cat.DefineRelation(stmt, 'r')- if err != nil {- t.Logf("%-12s → DefineRelation rejected: %s", ty, truncate(err.Error(), 80))- } else {- t.Logf("%-12s → DefineRelation accepted", ty)- }- }-}--// TestLoaderPoC_UnknownTypeQueryBattery installs the same queries as-// TestLoaderPoC_TypesDontMatterForLineage but with all columns declared as-// 'unknown' instead of 'text'. The question: does the analyzer's overload-// resolution treat unknown differently from text?-func TestLoaderPoC_UnknownTypeQueryBattery(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- install := func(table string, cols []string) {- colDefs := make([]*ast.ColumnDef, len(cols))- for i, c := range cols {- colDefs[i] = columnDef(c, typeName("unknown"), false)- }- if err := cat.DefineRelation(buildCreateTableStmt("public", table, colDefs), 'r'); err != nil {- t.Fatalf("install %s: %v", table, err)- }- }- install("orders", []string{"id", "amount", "status", "placed_at", "tags", "customer_id"})- install("customers", []string{"id", "email", "name", "region"})-- cases := []struct {- category, name, query string- }{- // ---- Baseline: name resolution only ----- {"name_res", "plain_select", `SELECT id, status FROM orders`},- {"name_res", "select_star", `SELECT * FROM orders`},- {"name_res", "join", `SELECT o.id, c.email FROM orders o JOIN customers c ON c.id = o.customer_id`},- {"name_res", "three_way_join", `SELECT o.id FROM orders o JOIN customers c ON c.id = o.customer_id JOIN orders o2 ON o2.id = o.id`},- {"name_res", "alias_qualified", `SELECT o.id AS order_id, c.name AS customer FROM orders o, customers c WHERE c.id = o.customer_id`},-- // ---- Text-friendly operators ----- {"text_ops", "text_equality", `SELECT id FROM orders WHERE status = 'open'`},- {"text_ops", "text_like", `SELECT id FROM orders WHERE status LIKE 'op%'`},- {"text_ops", "text_ilike", `SELECT id FROM orders WHERE status ILIKE 'OP%'`},- {"text_ops", "text_in_list", `SELECT id FROM orders WHERE status IN ('open', 'closed')`},- {"text_ops", "is_null", `SELECT id FROM orders WHERE status IS NULL`},- {"text_ops", "is_not_null", `SELECT id FROM orders WHERE status IS NOT NULL`},-- // ---- Numeric operators (unknown auto-coerces) ----- {"numeric_ops", "numeric_compare", `SELECT id FROM orders WHERE amount > 100`},- {"numeric_ops", "numeric_arithmetic", `SELECT id, amount + 1 AS p FROM orders`},- {"numeric_ops", "numeric_between", `SELECT id FROM orders WHERE amount BETWEEN 10 AND 100`},- {"numeric_ops", "numeric_negation", `SELECT id, -amount FROM orders`},-- // ---- Date/time ops ----- {"date_ops", "date_compare", `SELECT id FROM orders WHERE placed_at > '2024-01-01'::date`},- {"date_ops", "date_extract", `SELECT EXTRACT(year FROM placed_at::timestamp) FROM orders`},- {"date_ops", "interval_arith", `SELECT id FROM orders WHERE placed_at > now() - interval '7 days'`},-- // ---- Array ops ----- {"array_ops", "array_contains", `SELECT id FROM orders WHERE tags @> ARRAY['urgent']`},- {"array_ops", "array_any", `SELECT id FROM orders WHERE 'urgent' = ANY(tags)`},- {"array_ops", "array_length", `SELECT id, array_length(tags, 1) FROM orders`},- {"array_ops", "array_unnest", `SELECT id, unnest(tags) FROM orders`},-- // ---- Aggregates (the worry zone) ----- {"aggregate", "count_star", `SELECT COUNT(*) FROM orders`},- {"aggregate", "count_col", `SELECT COUNT(id) FROM orders`},- {"aggregate", "count_distinct", `SELECT COUNT(DISTINCT customer_id) FROM orders`},- {"aggregate", "sum", `SELECT SUM(amount) FROM orders`},- {"aggregate", "avg", `SELECT AVG(amount) FROM orders`},- {"aggregate", "min_max", `SELECT MIN(amount), MAX(amount) FROM orders`},- {"aggregate", "string_agg", `SELECT STRING_AGG(status, ',') FROM orders`},- {"aggregate", "array_agg", `SELECT ARRAY_AGG(id) FROM orders`},- {"aggregate", "group_by", `SELECT status, COUNT(*) FROM orders GROUP BY status`},- {"aggregate", "having", `SELECT status, COUNT(*) FROM orders GROUP BY status HAVING COUNT(*) > 5`},-- // ---- Aggregate with explicit cast (workaround) ----- {"cast_workaround", "sum_with_cast", `SELECT SUM(amount::numeric) FROM orders`},- {"cast_workaround", "avg_with_cast", `SELECT AVG(amount::numeric) FROM orders`},- {"cast_workaround", "string_agg_cast", `SELECT STRING_AGG(status::text, ',') FROM orders`},-- // ---- Window functions ----- {"window", "row_number", `SELECT id, ROW_NUMBER() OVER (ORDER BY placed_at) FROM orders`},- {"window", "rank_partition", `SELECT id, RANK() OVER (PARTITION BY status ORDER BY amount) FROM orders`},- {"window", "sum_over", `SELECT id, SUM(amount) OVER (PARTITION BY status) FROM orders`},- {"window", "lag_col", `SELECT id, LAG(status) OVER (ORDER BY placed_at) FROM orders`},-- // ---- Control flow / null handling ----- {"control", "case_when", `SELECT id, CASE WHEN status = 'open' THEN 1 ELSE 0 END FROM orders`},- {"control", "coalesce", `SELECT COALESCE(status, 'unknown') FROM orders`},- {"control", "nullif", `SELECT NULLIF(status, 'open') FROM orders`},-- // ---- Subqueries / set operations ----- {"subquery", "exists", `SELECT id FROM orders o WHERE EXISTS (SELECT 1 FROM customers c WHERE c.id = o.customer_id)`},- {"subquery", "scalar_subquery", `SELECT id, (SELECT MAX(amount) FROM orders) AS max_amt FROM orders`},- {"subquery", "not_in", `SELECT id FROM orders WHERE customer_id NOT IN (SELECT id FROM customers WHERE region = 'us')`},- {"subquery", "union", `SELECT id FROM orders UNION SELECT id FROM customers`},- {"subquery", "intersect", `SELECT id FROM orders INTERSECT SELECT id FROM customers`},- {"subquery", "except", `SELECT id FROM orders EXCEPT SELECT id FROM customers`},-- // ---- CTE + recursion ----- {"cte", "simple_cte", `WITH active AS (SELECT id, status FROM orders WHERE status = 'open') SELECT id FROM active`},- {"cte", "cte_with_agg", `WITH stats AS (SELECT status, COUNT(*) c FROM orders GROUP BY status) SELECT * FROM stats`},-- // ---- Sort / limit ----- {"order_limit", "order_by_col", `SELECT id FROM orders ORDER BY placed_at DESC LIMIT 10`},- {"order_limit", "order_by_expr", `SELECT id FROM orders ORDER BY amount + 1`},- {"order_limit", "distinct_on", `SELECT DISTINCT ON (customer_id) id, customer_id FROM orders ORDER BY customer_id, placed_at DESC`},- }-- type bucket struct{ pass, fail int }- byCategory := map[string]*bucket{}-- pass := 0- fail := 0- var failed []string- for _, tc := range cases {- if byCategory[tc.category] == nil {- byCategory[tc.category] = &bucket{}- }- stmts, err := ParsePg(tc.query)- if err != nil {- t.Logf("%-12s %-22s PARSE FAIL: %v", tc.category, tc.name, err)- fail++- byCategory[tc.category].fail++- failed = append(failed, tc.category+"/"+tc.name+" (parse)")- continue- }- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- t.Logf("%-12s %-22s NOT SELECT", tc.category, tc.name)- fail++- byCategory[tc.category].fail++- continue- }- _, err = cat.AnalyzeSelectStmt(sel)- if err != nil {- t.Logf("%-12s %-22s FAIL: %s", tc.category, tc.name, truncate(err.Error(), 90))- fail++- byCategory[tc.category].fail++- failed = append(failed, tc.category+"/"+tc.name)- continue- }- pass++- byCategory[tc.category].pass++- t.Logf("%-12s %-22s PASS", tc.category, tc.name)- }-- t.Log("")- t.Log("----- unknown-type battery: per-category breakdown -----")- for _, cat := range []string{"name_res", "text_ops", "numeric_ops", "date_ops", "array_ops", "aggregate", "cast_workaround", "window", "control", "subquery", "cte", "order_limit"} {- b := byCategory[cat]- if b == nil {- continue- }- t.Logf(" %-16s %d pass / %d fail", cat, b.pass, b.fail)- }- t.Log("")- t.Logf("TOTAL: %d pass, %d fail (%.0f%% pass rate)", pass, fail, 100.0*float64(pass)/float64(pass+fail))- if len(failed) > 0 {- t.Log("failed:")- for _, f := range failed {- t.Logf(" - %s", f)- }- }-}--// TestLoaderPoC_HybridTypeResolution explores the pragmatic middle ground:-// for each column, try to resolve the metadata type string; if it fails-// (user type not in catalog, malformed type string), fall back to text.-// This gives us "best of both" — real types where available, permissive-// text where not.-func TestLoaderPoC_HybridTypeResolution(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Hybrid column builder: try honest type, fall back to text.- hybridCol := func(name string, typeStr string) *ast.ColumnDef {- tn := mustTypeName(t, typeStr)- // Check if catalog can resolve it at this moment.- // Note: for user types to resolve, they must already be installed.- // We're checking against the catalog AS-IS.- resolved := false- func() {- defer func() { _ = recover() }() // defensive: ResolveType may not panic, but be safe- _, _, err := cat.ResolveType(toCatalogTypeName(tn))- resolved = (err == nil)- }()- if resolved {- return &ast.ColumnDef{Colname: name, TypeName: tn}- }- t.Logf(" hybrid: column %q type %q unresolvable in catalog, falling back to text", name, typeStr)- return &ast.ColumnDef{Colname: name, TypeName: typeName("text")}- }-- // Simulate metadata where some types are resolvable, some aren't.- // 'int4' is a builtin — resolvable.- // 'numeric(10,2)' is a builtin — resolvable.- // 'public.unknown_enum_xyz' is a user type we never installed — should fall back to text.- stmt := buildCreateTableStmt("public", "mix", []*ast.ColumnDef{- hybridCol("id", "int4"),- hybridCol("amount", "numeric(10,2)"),- hybridCol("tag", "public.unknown_enum_xyz"),- })- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("hybrid install failed: %v", err)- }-- // Queries that use the numeric column (known type) should still work- // with real operator resolution.- q1 := mustAnalyze(t, cat, `SELECT id, amount FROM mix WHERE amount > 100`)- if len(q1.TargetList) != 2 {- t.Errorf("q1 targets: got %d, want 2", len(q1.TargetList))- }-- // Queries that touch the tag column only do name resolution — no operator- // type-checking — should work even though tag is text-in-reality-but-- // metadata-said-enum.- q2 := mustAnalyze(t, cat, `SELECT id, tag FROM mix WHERE tag = 'a'`)- if len(q2.TargetList) != 2 {- t.Errorf("q2 targets: got %d, want 2", len(q2.TargetList))- }-- t.Log("hybrid PoC: resolvable types use real TypeName, unresolvable types fall back to text")- t.Log("effect: best of both — operator semantics work where types known, name resolution works where not")-}--// TestLoaderPoC_BvsCDivergence pinpoints the exact behavioral difference between-// Option B (hybrid: builtins honest, user types unknown) and Option C (all-// honest types, user types installed).-//-// Scenario: a table with ONE builtin-int column and ONE user-enum column.-// Run the same queries against two catalogs:-//-// catC: enum installed via DefineEnum, table column is the real enum type-// catB: enum NOT installed, table column is 'unknown'-//-// For each query, record (passes/fails, target count, lineage equivalent).-// The output shows exactly where B and C diverge.-func TestLoaderPoC_BvsCDivergence(t *testing.T) {- // ----- catC: option C (all honest) ------ catC := catalog.New()- catC.SetSearchPath([]string{"public"})-- if err := catC.DefineEnum(&ast.CreateEnumStmt{- TypeName: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "task_status"},- }},- Vals: &ast.List{Items: []ast.Node{- &ast.String{Str: "open"},- &ast.String{Str: "closed"},- }},- }); err != nil {- t.Fatalf("catC install enum: %v", err)- }- if err := catC.DefineRelation(buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("status", mustTypeName(t, "public.task_status"), false),- columnDef("created_at", mustTypeName(t, "timestamptz"), false),- }), 'r'); err != nil {- t.Fatalf("catC install table: %v", err)- }-- // ----- catB: option B (hybrid — enum column as unknown, everything else honest) ------ catB := catalog.New()- catB.SetSearchPath([]string{"public"})- // Note: no DefineEnum call. Enum "doesn't exist" from catB's perspective.- if err := catB.DefineRelation(buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true), // builtin → honest- columnDef("status", typeName("unknown"), false), // user type → unknown- columnDef("created_at", typeName("timestamptz"), false), // builtin → honest- }), 'r'); err != nil {- t.Fatalf("catB install table: %v", err)- }-- cases := []struct {- name, query string- }{- // Queries that should work in BOTH (builtin columns untouched)- {"select_id", `SELECT id FROM tasks`},- {"select_star", `SELECT * FROM tasks`},- {"count_star", `SELECT COUNT(*) FROM tasks`},- {"sum_id", `SELECT SUM(id) FROM tasks`},- {"avg_id", `SELECT AVG(id) FROM tasks`},- {"max_created", `SELECT MAX(created_at) FROM tasks`},-- // Queries touching status column — B gets 'unknown', C gets real enum- {"where_status_eq", `SELECT id FROM tasks WHERE status = 'open'`},- {"where_status_in", `SELECT id FROM tasks WHERE status IN ('open', 'closed')`},- {"select_status", `SELECT status FROM tasks`},- {"group_by_status", `SELECT status, COUNT(*) FROM tasks GROUP BY status`},- {"order_by_status", `SELECT id FROM tasks ORDER BY status`},- {"distinct_status", `SELECT DISTINCT status FROM tasks`},-- // Aggregates directly on the enum column — the real test- {"count_status", `SELECT COUNT(status) FROM tasks`},- {"count_distinct_status", `SELECT COUNT(DISTINCT status) FROM tasks`},- {"min_status", `SELECT MIN(status) FROM tasks`},- {"max_status", `SELECT MAX(status) FROM tasks`},- {"array_agg_status", `SELECT array_agg(status) FROM tasks`},- {"string_agg_status", `SELECT string_agg(status::text, ',') FROM tasks`},-- // Mixed: status in WHERE, sum on builtin- {"mixed_sum", `SELECT SUM(id) FROM tasks WHERE status = 'open'`},- {"mixed_group", `SELECT status, SUM(id) FROM tasks GROUP BY status`},- }-- type res struct {- ok bool- err string- }- runOne := func(cat *catalog.Catalog, sql string) res {- stmts, err := ParsePg(sql)- if err != nil {- return res{err: "parse: " + truncate(err.Error(), 60)}- }- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- return res{err: "not a SelectStmt"}- }- if _, err := cat.AnalyzeSelectStmt(sel); err != nil {- return res{err: truncate(err.Error(), 70)}- }- return res{ok: true}- }-- var bothOk, cOnlyOk, bOnlyOk, bothFail int- t.Logf("%-24s %-6s %-6s %s", "case", "C", "B", "divergence note")- t.Logf("%-24s %-6s %-6s %s", "----", "-", "-", "---------------")- for _, tc := range cases {- c := runOne(catC, tc.query)- b := runOne(catB, tc.query)- cMark := "FAIL"- if c.ok {- cMark = "PASS"- }- bMark := "FAIL"- if b.ok {- bMark = "PASS"- }-- note := ""- switch {- case c.ok && b.ok:- bothOk++- note = "same"- case c.ok && !b.ok:- cOnlyOk++- note = "C only — B: " + b.err- case !c.ok && b.ok:- bOnlyOk++- note = "B only — C: " + c.err- default:- bothFail++- note = "both fail"- }- t.Logf("%-24s %-6s %-6s %s", tc.name, cMark, bMark, note)- }-- t.Log("")- t.Logf("summary: both_ok=%d C_only=%d B_only=%d both_fail=%d", bothOk, cOnlyOk, bOnlyOk, bothFail)- t.Logf("option B loss (queries where C works but B falls back): %d / %d = %.0f%%",- cOnlyOk, len(cases), 100.0*float64(cOnlyOk)/float64(len(cases)))-}--// TestLoaderPoC_BvsCDomainColumn shows the case where B is STRICTLY BETTER than C:-// a table with a domain-typed column where the domain is NOT in bytebase metadata-// (which is the ACTUAL state of bytebase PG sync today — domain/composite/range-// types are not synced). Option C fails at install time and blocks all queries-// on the table. Option B uses unknown and only loses type-sensitive aggregates.-func TestLoaderPoC_BvsCDomainColumn(t *testing.T) {- // Simulate reality: bytebase sync does NOT capture the 'public.positive_int'- // domain. The table metadata says column 'count' has type 'public.positive_int',- // but bytebase has no DomainMetadata for it.-- // ----- catC: option C — try to honest-install everything ------ // Closure expansion would try to load 'public.positive_int' from metadata,- // find nothing, and either fail the table install or skip the column.- // Let's simulate: C tries to install the table with the real type reference,- // fails because the domain doesn't exist in the catalog.- catC := catalog.New()- catC.SetSearchPath([]string{"public"})- cStmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("count", mustTypeName(t, "public.positive_int"), true),- })- cErr := catC.DefineRelation(cStmt, 'r')- if cErr != nil {- t.Logf("[C] DefineRelation failed as expected: %s", truncate(cErr.Error(), 80))- } else {- t.Log("[C] DefineRelation unexpectedly succeeded")- }-- // ----- catB: option B — count column degrades to unknown ------ catB := catalog.New()- catB.SetSearchPath([]string{"public"})- bStmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("count", typeName("unknown"), true),- })- if err := catB.DefineRelation(bStmt, 'r'); err != nil {- t.Fatalf("[B] DefineRelation failed: %v", err)- }- t.Log("[B] DefineRelation succeeded")-- cases := []struct {- name, query string- }{- {"simple_select", `SELECT id, count FROM counters`},- {"where_id", `SELECT id FROM counters WHERE id > 0`},- {"where_count", `SELECT id FROM counters WHERE count > 0`},- {"sum_id", `SELECT SUM(id) FROM counters`},- {"sum_count", `SELECT SUM(count) FROM counters`},- {"sum_id_cast", `SELECT SUM(count::numeric) FROM counters`},- }-- runOne := func(cat *catalog.Catalog, sql string) string {- stmts, err := ParsePg(sql)- if err != nil {- return "parse: " + err.Error()- }- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- return "not a SelectStmt"- }- if _, err := cat.AnalyzeSelectStmt(sel); err != nil {- return truncate(err.Error(), 60)- }- return "PASS"- }-- t.Log("")- t.Log("Query availability on 'counters' table (domain column):")- t.Logf("%-22s %-40s %s", "case", "C result", "B result")- var cWorks, bWorks int- for _, tc := range cases {- var cResult string- if cErr != nil {- cResult = "TABLE NOT INSTALLED"- } else {- cResult = runOne(catC, tc.query)- }- bResult := runOne(catB, tc.query)- if cResult == "PASS" {- cWorks++- }- if bResult == "PASS" {- bWorks++- }- t.Logf("%-22s %-40s %s", tc.name, truncate(cResult, 38), bResult)- }- t.Log("")- t.Logf("domain column query survival: C=%d/%d, B=%d/%d", cWorks, len(cases), bWorks, len(cases))-}--// toCatalogTypeName converts ast.TypeName to catalog.TypeName for ResolveType.-// catalog.TypeName is the lightweight shape used by the public ResolveType API.-func toCatalogTypeName(tn *ast.TypeName) catalog.TypeName {- var schema, name string- if tn.Names != nil && len(tn.Names.Items) > 0 {- items := tn.Names.Items- if len(items) == 1 {- if s, ok := items[0].(*ast.String); ok {- name = s.Str- }- } else if len(items) >= 2 {- if s, ok := items[0].(*ast.String); ok {- schema = s.Str- }- if s, ok := items[len(items)-1].(*ast.String); ok {- name = s.Str- }- }- }- return catalog.TypeName{Schema: schema, Name: name, TypeMod: tn.Typemod}-}--// Format helper for readable output.-var _ = fmt.Sprintf // keep fmt imported when strings.Builder not used--// ---------- Equivalence test: hand-built CreateStmt vs parsed DDL -----------//-// Per codex review section 2B: the PoC must prove that hand-constructed AST-// produces a catalog state equivalent to parsing the equivalent DDL text.-// This test installs the same table two ways on two separate catalogs and-// asserts the resulting Relation objects are structurally identical on all-// fields reachable via public API.--func TestLoaderPoC_HandBuiltVsDDLEquivalence(t *testing.T) {- // Catalog A: install via hand-built CreateStmt.- catA := catalog.New()- catA.SetSearchPath([]string{"public"})- handBuilt := buildCreateTableStmt("public", "orders", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("customer", typeName("text"), false),- columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),- columnDef("placed_at", mustTypeName(t, "timestamp with time zone"), false),- })- if err := catA.DefineRelation(handBuilt, 'r'); err != nil {- t.Fatalf("catA install: %v", err)- }-- // Catalog B: install via parsed DDL text.- catB := catalog.New()- catB.SetSearchPath([]string{"public"})- ddl := `CREATE TABLE public.orders (- id int NOT NULL,- customer text,- amount numeric(10,2),- placed_at timestamp with time zone- )`- if _, err := catB.Exec(ddl, nil); err != nil {- t.Fatalf("catB install via Exec: %v", err)- }-- relA := catA.GetRelation("public", "orders")- relB := catB.GetRelation("public", "orders")- if relA == nil || relB == nil {- t.Fatalf("relation install missing: A=%v B=%v", relA != nil, relB != nil)- }-- // Compare structural fields. OIDs are not expected to match absolutely- // (each catalog has its own oidGen starting point), but relative order- // and per-column shape must match.-- if relA.RelKind != relB.RelKind {- t.Errorf("RelKind differs: A=%c B=%c", relA.RelKind, relB.RelKind)- }- if len(relA.Columns) != len(relB.Columns) {- t.Fatalf("column count: A=%d B=%d", len(relA.Columns), len(relB.Columns))- }- for i := range relA.Columns {- ca, cb := relA.Columns[i], relB.Columns[i]- if ca.Name != cb.Name {- t.Errorf("col[%d].Name: A=%q B=%q", i, ca.Name, cb.Name)- }- if ca.TypeOID != cb.TypeOID {- t.Errorf("col[%d] (%s).TypeOID: A=%d B=%d", i, ca.Name, ca.TypeOID, cb.TypeOID)- }- if ca.NotNull != cb.NotNull {- t.Errorf("col[%d] (%s).NotNull: A=%v B=%v", i, ca.Name, ca.NotNull, cb.NotNull)- }- if ca.AttNum != cb.AttNum {- t.Errorf("col[%d] (%s).AttNum: A=%d B=%d", i, ca.Name, ca.AttNum, cb.AttNum)- }- if ca.TypeMod != cb.TypeMod {- t.Errorf("col[%d] (%s).TypeMod: A=%d B=%d", i, ca.Name, ca.TypeMod, cb.TypeMod)- }- }-- // Functional equivalence: analyze the same SELECT against both catalogs- // and assert the same target list shape.- qA := mustAnalyze(t, catA, `SELECT id, customer, amount, placed_at FROM orders WHERE amount > 0`)- qB := mustAnalyze(t, catB, `SELECT id, customer, amount, placed_at FROM orders WHERE amount > 0`)-- if len(qA.TargetList) != len(qB.TargetList) {- t.Errorf("query target count: A=%d B=%d", len(qA.TargetList), len(qB.TargetList))- }- for i := range qA.TargetList {- if qA.TargetList[i].ResName != qB.TargetList[i].ResName {- t.Errorf("target[%d].ResName: A=%q B=%q",- i, qA.TargetList[i].ResName, qB.TargetList[i].ResName)- }- }-- t.Log("equivalence PoC: hand-built CreateStmt and parsed DDL produce structurally identical Relation + analyze the same way")-}--// ============================================================================-// Loader pseudo-install verification-// ============================================================================-//-// These tests probe whether omni accepts minimal "pseudo" forms of each schema-// object kind, so the catalog loader strategy ("install everything; on failure install a-// pseudo at the same slot to keep downstream installs cascading through") can-// work inline during catalog init rather than as a query-time retry.-//-// Each test exercises one pseudo form in isolation, then the final-// TestLoaderPoC_RootPseudoCascadePrevention combines them end-to-end.--// ---- Pseudo enum: DefineEnum with minimal vals ------func TestLoaderPoC_PseudoEnum_EmptyVals(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Can omni accept an enum with zero values?- stmt := &ast.CreateEnumStmt{- TypeName: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "broken_enum"},- }},- Vals: &ast.List{Items: []ast.Node{}},- }- err := cat.DefineEnum(stmt)- t.Logf("DefineEnum empty vals: err=%v", err)-- if err == nil {- // If it accepted, can a table column reference it and can the query run?- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("status", mustTypeName(t, "public.broken_enum"), false),- })- if err := cat.DefineRelation(tbl, 'r'); err != nil {- t.Logf("DefineRelation with empty-enum column: err=%v", err)- } else {- q, err := analyzeSafe(cat, `SELECT id, status FROM t WHERE status = 'anything'`)- t.Logf("analyze against empty-enum column: err=%v targets=%d", err, targetsOf(q))- }- }-}--func TestLoaderPoC_PseudoEnum_SingleDummyVal(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Fallback: enum with single dummy value.- stmt := &ast.CreateEnumStmt{- TypeName: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "broken_enum"},- }},- Vals: &ast.List{Items: []ast.Node{- &ast.String{Str: "__pseudo__"},- }},- }- if err := cat.DefineEnum(stmt); err != nil {- t.Fatalf("DefineEnum single dummy val failed: %v", err)- }-- // Install a table using the pseudo enum as a column type.- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("status", mustTypeName(t, "public.broken_enum"), false),- })- if err := cat.DefineRelation(tbl, 'r'); err != nil {- t.Fatalf("DefineRelation with pseudo-enum column failed: %v", err)- }-- // Probe queries that would normally hit enum operators.- queries := []string{- `SELECT id, status FROM t`,- `SELECT id FROM t WHERE status = 'running'`,- `SELECT id FROM t WHERE status::text = 'running'`,- `SELECT COUNT(*) FROM t GROUP BY status`,- }- for _, q := range queries {- res, err := analyzeSafe(cat, q)- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))- }-}--// ---- Pseudo domain: DefineDomain with base type text ------func TestLoaderPoC_PseudoDomain_OverText(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- stmt := &ast.CreateDomainStmt{- Domainname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "broken_domain"},- }},- Typname: typeName("text"),- }- if err := cat.DefineDomain(stmt); err != nil {- t.Fatalf("DefineDomain over text failed: %v", err)- }-- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("code", mustTypeName(t, "public.broken_domain"), false),- })- if err := cat.DefineRelation(tbl, 'r'); err != nil {- t.Fatalf("DefineRelation with pseudo-domain column failed: %v", err)- }-- queries := []string{- `SELECT id, code FROM t`,- `SELECT id FROM t WHERE code = 'anything'`,- `SELECT id FROM t WHERE code LIKE 'A%'`,- }- for _, q := range queries {- res, err := analyzeSafe(cat, q)- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))- }-}--// ---- Pseudo composite: DefineCompositeType with metadata field names all text ------func TestLoaderPoC_PseudoComposite_FieldsAllText(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Pretend metadata tells us this composite has fields {street, city, zip}.- // We preserve the names but flatten every field to text.- stmt := &ast.CompositeTypeStmt{- Typevar: &ast.RangeVar{- Schemaname: "public",- Relname: "broken_addr",- },- Coldeflist: &ast.List{Items: []ast.Node{- columnDef("street", typeName("text"), false),- columnDef("city", typeName("text"), false),- columnDef("zip", typeName("text"), false),- }},- }- if err := cat.DefineCompositeType(stmt); err != nil {- t.Fatalf("DefineCompositeType pseudo failed: %v", err)- }-- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("addr", mustTypeName(t, "public.broken_addr"), false),- })- if err := cat.DefineRelation(tbl, 'r'); err != nil {- t.Fatalf("DefineRelation with pseudo-composite column failed: %v", err)- }-- queries := []string{- `SELECT id, addr FROM t`,- `SELECT id, (addr).city FROM t`,- `SELECT id FROM t WHERE (addr).city = 'NYC'`,- }- for _, q := range queries {- res, err := analyzeSafe(cat, q)- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))- }-}--// ---- Pseudo range: DefineRange with subtype text ------func TestLoaderPoC_PseudoRange_SubtypeText(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Try: range with subtype text. PG itself requires subtype to have btree- // ordering, which text does. If omni accepts it, this is a viable pseudo.- stmt := &ast.CreateRangeStmt{- TypeName: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "broken_range"},- }},- Params: &ast.List{Items: []ast.Node{- &ast.DefElem{- Defname: "subtype",- Arg: &ast.TypeName{- Names: &ast.List{Items: []ast.Node{&ast.String{Str: "text"}}},- Typemod: -1,- },- },- }},- }- err := cat.DefineRange(stmt)- t.Logf("DefineRange subtype=text: err=%v", err)- if err != nil {- t.Skip("range install failed; this is fine if bytebase metadata never has ranges")- }-- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("span", mustTypeName(t, "public.broken_range"), false),- })- if err := cat.DefineRelation(tbl, 'r'); err != nil {- t.Logf("DefineRelation with pseudo-range column: err=%v", err)- }-- q, err := analyzeSafe(cat, `SELECT id, span FROM t`)- t.Logf("query on pseudo-range: err=%v targets=%d", err, targetsOf(q))-}--// ---- Pseudo view: constant SELECT body, no FROM ------func TestLoaderPoC_PseudoView_ConstantTargetList(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Pretend metadata tells us view "v" exposes columns {id, title, total},- // but its body is broken/unparseable. Build a pseudo SELECT body with- // NULL::text aliased to each name.- pseudoBody := `SELECT NULL::text AS id, NULL::text AS title, NULL::text AS total`- sel := mustParseSelect(t, pseudoBody)-- stmt := &ast.ViewStmt{- View: &ast.RangeVar{- Schemaname: "public",- Relname: "broken_view",- Relpersistence: 'p',- },- Query: sel,- }- if err := cat.DefineView(stmt); err != nil {- t.Fatalf("DefineView with constant-target body failed: %v", err)- }-- rel := cat.GetRelation("public", "broken_view")- if rel == nil {- t.Fatal("pseudo view not installed")- }- t.Logf("pseudo view installed: RelKind=%c cols=%d", rel.RelKind, len(rel.Columns))- for _, c := range rel.Columns {- t.Logf(" col: %s", c.Name)- }-- queries := []string{- `SELECT id, title, total FROM broken_view`,- `SELECT id FROM broken_view WHERE total = '0'`,- `SELECT COUNT(*) FROM broken_view`,- }- for _, q := range queries {- res, err := analyzeSafe(cat, q)- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))- }-}--// ---- Pseudo function: all params/return as text ------func TestLoaderPoC_PseudoFunction_AllText(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Pretend metadata says the real function was broken_fn(some_broken_type)- // returning some_other_broken_type. We install a pseudo with everything- // collapsed to text.- stmt := &ast.CreateFunctionStmt{- Funcname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "broken_fn"},- }},- Parameters: &ast.List{Items: []ast.Node{- &ast.FunctionParameter{ArgType: typeName("text"), Mode: ast.FUNC_PARAM_IN},- }},- ReturnType: typeName("text"),- Options: &ast.List{Items: []ast.Node{- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},- &ast.DefElem{- Defname: "as",- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},- },- }},- }- if err := cat.CreateFunctionStmt(stmt); err != nil {- t.Fatalf("pseudo function install failed: %v", err)- }-- procs := cat.LookupProcByName("broken_fn")- t.Logf("pseudo function installed: %d procs under 'broken_fn'", len(procs))-- // Install a minimal table to query against.- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("name", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("install t: %v", err)- }-- queries := []string{- `SELECT id, broken_fn(name) AS result FROM t`,- `SELECT id FROM t WHERE broken_fn(name) = 'x'`,- // Analyzer should pick the only overload regardless of actual arg type.- `SELECT broken_fn('literal') FROM t`,- }- for _, q := range queries {- res, err := analyzeSafe(cat, q)- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))- }-}--// ---- End-to-end: root-pseudo prevents cascade ------func TestLoaderPoC_RootPseudoCascadePrevention(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Scenario:- // enum E — real install fails (simulated)- // table T — has column col E- // view V — SELECT col FROM T- //- // Phase 1: for each object in topo order:- // 1. attempt real install- // 2. on failure, install pseudo version in the same slot- //- // Expected: T and V install successfully because they see pseudo E in the- // catalog. Query against V has correct lineage.-- // Step 1: simulate E real-install failure. In the real loader we would try- // DefineEnum first and catch the error. Here we skip straight to the- // pseudo form to simulate "real install failed".- pseudoE := &ast.CreateEnumStmt{- TypeName: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "task_status"},- }},- Vals: &ast.List{Items: []ast.Node{- &ast.String{Str: "__pseudo__"},- }},- }- if err := cat.DefineEnum(pseudoE); err != nil {- t.Fatalf("pseudo enum install failed: %v", err)- }-- // Step 2: install T pointing at the (pseudo) E. This is the real install,- // not a pseudo — T's own metadata is fine.- tStmt := buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("status", mustTypeName(t, "public.task_status"), false),- columnDef("title", typeName("text"), false),- })- if err := cat.DefineRelation(tStmt, 'r'); err != nil {- t.Fatalf("T real install failed (cascade not prevented): %v", err)- }-- // Step 3: install V — real install, view body references T.- viewBody := `SELECT id, title, status FROM tasks WHERE id > 0`- sel := mustParseSelect(t, viewBody)- vStmt := &ast.ViewStmt{- View: &ast.RangeVar{- Schemaname: "public",- Relname: "open_tasks",- Relpersistence: 'p',- },- Query: sel,- }- if err := cat.DefineView(vStmt); err != nil {- t.Fatalf("V real install failed (cascade not prevented): %v", err)- }-- // Step 4: query V — should analyze successfully and return lineage.- q := mustAnalyze(t, cat, `SELECT id, title, status FROM open_tasks`)- if len(q.TargetList) != 3 {- t.Errorf("query on view: got %d targets, want 3", len(q.TargetList))- }-- // Verify lineage reaches base tables, not just view columns.- for i, te := range q.TargetList {- t.Logf(" target[%d]: name=%s expr=%T", i, te.ResName, te.Expr)- }-- // Step 5: try a query that uses enum-typed column with operators that- // would normally need enum resolution.- res, err := analyzeSafe(cat, `SELECT id FROM open_tasks WHERE status = 'pending'`)- t.Logf("query with WHERE on pseudo-enum column: err=%v targets=%d", err, targetsOf(res))-- t.Log("cascade prevention PoC: pseudo-enum at root allowed T and V to install as real objects")-}--// ---- Corner case 1: mixed real + pseudo overloads of the same function name -----//-// Scenario: function fn has two overloads in metadata:-// fn(int4) → int4 (real install succeeds)-// fn(my_broken_type) → my_broken_type (real install fails because type is broken)-//-// catalog loader strategy: keep fn(int4) real, install pseudo fn(text) for the broken one.-// Verify: calls to fn(int_col) still pick fn(int4); calls to fn(text_col) pick-// the pseudo fn(text); calls to fn(literal_string) pick the pseudo.--func TestLoaderPoC_MixedRealAndPseudoOverloads(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- makeFuncStmt := func(name string, argType, retType string) *ast.CreateFunctionStmt {- return &ast.CreateFunctionStmt{- Funcname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: name},- }},- Parameters: &ast.List{Items: []ast.Node{- &ast.FunctionParameter{ArgType: typeName(argType), Mode: ast.FUNC_PARAM_IN},- }},- ReturnType: typeName(retType),- Options: &ast.List{Items: []ast.Node{- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},- &ast.DefElem{- Defname: "as",- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},- },- }},- }- }-- // Install real fn(int4).- if err := cat.CreateFunctionStmt(makeFuncStmt("fn", "int4", "int4")); err != nil {- t.Fatalf("real fn(int4) install failed: %v", err)- }-- // Install pseudo fn(text) — pretending the real fn(broken_type) failed.- if err := cat.CreateFunctionStmt(makeFuncStmt("fn", "text", "text")); err != nil {- t.Fatalf("pseudo fn(text) install failed: %v", err)- }-- procs := cat.LookupProcByName("fn")- t.Logf("fn overloads after mixed install: %d", len(procs))- if len(procs) < 2 {- t.Errorf("expected >= 2 fn overloads, got %d", len(procs))- }-- // Install a table with one int and one text column.- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("n", typeName("int4"), false),- columnDef("s", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("install t: %v", err)- }-- // Each query exercises a different resolution path.- queries := []struct {- name string- sql string- }{- {"fn(int_col) → should pick real fn(int4)", `SELECT id, fn(n) AS r FROM t`},- {"fn(text_col) → should pick pseudo fn(text)", `SELECT id, fn(s) AS r FROM t`},- {"fn(literal_int) → picks fn(int4) or via implicit cast", `SELECT fn(42) FROM t`},- {"fn(literal_text) → picks fn(text)", `SELECT fn('hello') FROM t`},- }- for _, q := range queries {- res, err := analyzeSafe(cat, q.sql)- t.Logf("%s: err=%v targets=%d", q.name, err, targetsOf(res))- }-}--// ---- Corner case 2: pseudo install after a real install has truly failed -----//-// Scenario: real DefineDomain fails because the base type does not exist.-// The loader must then install a pseudo in the same slot. Verify:-// 1. real install actually returns an error (otherwise this test is vacuous)-// 2. catalog does not hold partial state for the failed name-// 3. pseudo install at the same name succeeds-// 4. a table column using that pseudo name resolves correctly--func TestLoaderPoC_PseudoAfterFailedReal(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- // Step 1: try a real DefineDomain with a non-existent base type.- badStmt := &ast.CreateDomainStmt{- Domainname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "status_code"},- }},- Typname: typeName("nonexistent_base_type_xyz"),- }- err := cat.DefineDomain(badStmt)- t.Logf("real DefineDomain with bogus base type: err=%v", err)-- realFailed := err != nil-- // Step 2: check catalog state. If the real install created a partial entry,- // the pseudo install at the same name may conflict.- //- // Probe by trying to look up the name via a constructed type reference.- // There's no direct "is this type defined?" API, but we can try the pseudo- // install and see if it errors with a duplicate.-- // Step 3: install pseudo domain over text at the same name.- pseudoStmt := &ast.CreateDomainStmt{- Domainname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "status_code"},- }},- Typname: typeName("text"),- }- err = cat.DefineDomain(pseudoStmt)- t.Logf("pseudo DefineDomain (base=text) after real failure: err=%v", err)-- if err != nil {- t.Errorf("pseudo install after real failure should succeed; got %v", err)- return- }-- // Step 4: use the pseudo in a table column.- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("code", mustTypeName(t, "public.status_code"), false),- }), 'r'); err != nil {- t.Fatalf("table with pseudo-domain column install failed: %v", err)- }-- q := mustAnalyze(t, cat, `SELECT id, code FROM t WHERE code = 'ok'`)- if len(q.TargetList) != 2 {- t.Errorf("query on pseudo-domain column: got %d targets, want 2", len(q.TargetList))- }-- if !realFailed {- t.Log("note: real DefineDomain with bogus base type DID NOT fail — " +- "omni's validation is looser than expected. Test still exercises " +- "the pseudo install path but the 'recovery after failure' premise " +- "is weaker. Consider whether the loader needs to handle this case.")- } else {- t.Log("pseudo recovery PoC: real install failed cleanly, pseudo at same " +- "name installed successfully, downstream table install works")- }-}--// ---- helpers for loader tests ------// ---- Install cost benchmarks: catalog-loader hand-built vs Exec(DDL text) -----//-// Validates the plan's "5-10× faster than current" claim empirically.-// Generates synthetic schemas of varying size with representative shapes-// (tables with mixed-type columns) and times both install paths.-//-// Run with:-// go test -bench='BenchmarkLoader_Install' -benchmem -run='^$' -count=3 \-// github.com/bytebase/bytebase/backend/plugin/parser/pg--func makeSyntheticTables(n int) []*ast.CreateStmt {- tables := make([]*ast.CreateStmt, n)- for i := range n {- tables[i] = buildCreateTableStmt("public", fmt.Sprintf("t_%d", i), []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("name", typeName("text"), false),- columnDef("created_at", typeName("timestamp"), false),- columnDef("amount", typeName("int8"), false),- columnDef("status", typeName("text"), false),- })- }- return tables-}--func makeSyntheticDDL(n int) string {- var sb strings.Builder- sb.Grow(n * 200)- for i := range n {- fmt.Fprintf(&sb, "CREATE TABLE public.t_%d (id int4 NOT NULL, name text, created_at timestamp, amount int8, status text);\n", i)- }- return sb.String()-}--func benchmarkE3Install(b *testing.B, n int) {- tables := makeSyntheticTables(n)- b.ResetTimer()- for range b.N {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- for _, stmt := range tables {- if err := cat.DefineRelation(stmt, 'r'); err != nil {- b.Fatalf("DefineRelation: %v", err)- }- }- }-}--func benchmarkDDLInstall(b *testing.B, n int) {- ddl := makeSyntheticDDL(n)- b.ResetTimer()- for range b.N {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- if _, err := cat.Exec(ddl, &catalog.ExecOptions{ContinueOnError: true}); err != nil {- b.Fatalf("Exec: %v", err)- }- }-}--func BenchmarkLoader_Install_E3_100(b *testing.B) { benchmarkE3Install(b, 100) }-func BenchmarkLoader_Install_E3_500(b *testing.B) { benchmarkE3Install(b, 500) }-func BenchmarkLoader_Install_E3_2000(b *testing.B) { benchmarkE3Install(b, 2000) }-func BenchmarkLoader_Install_DDL_100(b *testing.B) { benchmarkDDLInstall(b, 100) }-func BenchmarkLoader_Install_DDL_500(b *testing.B) { benchmarkDDLInstall(b, 500) }-func BenchmarkLoader_Install_DDL_2000(b *testing.B) { benchmarkDDLInstall(b, 2000) }--// ---- Overload selection assertions: prove analyzer picks the right overload -----//-// Builds on TestLoaderPoC_MixedRealAndPseudoOverloads. That test logs outcomes-// but does not assert which overload got picked. Here we install two overloads-// with DISTINCT return types (int4 vs text) and assert that the analyzer picks-// the right one by checking the result column's TypeOID.-//-// Well-known PG OIDs: int4=23, text=25 (stable; used throughout omni tests).--func TestLoaderPoC_OverloadSelectionAsserted(t *testing.T) {- const (- oidInt4 uint32 = 23- oidText uint32 = 25- )-- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- mkFn := func(arg, ret string) *ast.CreateFunctionStmt {- return &ast.CreateFunctionStmt{- Funcname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: "fn"},- }},- Parameters: &ast.List{Items: []ast.Node{- &ast.FunctionParameter{ArgType: typeName(arg), Mode: ast.FUNC_PARAM_IN},- }},- ReturnType: typeName(ret),- Options: &ast.List{Items: []ast.Node{- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},- &ast.DefElem{- Defname: "as",- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},- },- }},- }- }-- // Real fn(int4) → int4. Pseudo fn(text) → text (simulated: pretend the- // real fn(broken_type) failed and we installed a text-backed pseudo).- if err := cat.CreateFunctionStmt(mkFn("int4", "int4")); err != nil {- t.Fatalf("fn(int4) install failed: %v", err)- }- if err := cat.CreateFunctionStmt(mkFn("text", "text")); err != nil {- t.Fatalf("fn(text) install failed: %v", err)- }-- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("n", typeName("int4"), false),- columnDef("s", typeName("text"), false),- }), 'r'); err != nil {- t.Fatalf("table install: %v", err)- }-- exprResultType := func(q *catalog.Query, targetIdx int) uint32 {- t.Helper()- if targetIdx >= len(q.TargetList) {- t.Fatalf("target index %d out of range (have %d)", targetIdx, len(q.TargetList))- }- te := q.TargetList[targetIdx]- fce, ok := te.Expr.(*catalog.FuncCallExpr)- if !ok {- t.Fatalf("target[%d] expected FuncCallExpr, got %T", targetIdx, te.Expr)- }- return fce.ResultType- }-- // Case 1: fn(int_col) must pick fn(int4) → returns int4- q := mustAnalyze(t, cat, `SELECT fn(n) FROM t`)- if got := exprResultType(q, 0); got != oidInt4 {- t.Errorf("fn(int_col): expected ResultType=%d (int4), got %d", oidInt4, got)- }-- // Case 2: fn(text_col) must pick fn(text) → returns text- q = mustAnalyze(t, cat, `SELECT fn(s) FROM t`)- if got := exprResultType(q, 0); got != oidText {- t.Errorf("fn(text_col): expected ResultType=%d (text), got %d", oidText, got)- }-- // Case 3: fn(42) — integer literal — must pick fn(int4)- q = mustAnalyze(t, cat, `SELECT fn(42) FROM t`)- if got := exprResultType(q, 0); got != oidInt4 {- t.Errorf("fn(literal_int): expected ResultType=%d (int4), got %d", oidInt4, got)- }-- // Case 4: fn('hello') — string literal — must pick fn(text)- q = mustAnalyze(t, cat, `SELECT fn('hello') FROM t`)- if got := exprResultType(q, 0); got != oidText {- t.Errorf("fn(literal_text): expected ResultType=%d (text), got %d", oidText, got)- }-- t.Log("overload selection asserted: analyzer picks the right overload by arg type; " +- "mixed real+pseudo overloads resolve correctly")-}--// ---- REAL cascade prevention: force a real install failure, then install pseudo -----//-// The previous cascade test simulated failure by skipping the real install-// entirely. This test actually runs a real install that fails at the omni-// level, then proves the loader pattern "catch error → install pseudo at-// same slot" lets downstream installs use real AST.-//-// Scenario:-// Domain D (base=nonexistent_type_xyz) — real DefineDomain ERRORS-// Table T (col D, title text) — real DefineRelation must succeed against pseudo D-// View V (SELECT col, title FROM T) — real DefineView must succeed against real T-// Query SELECT col, title FROM V — must analyze with lineage to T.{col, title}--func TestLoaderPoC_RealCascadeWithGenuineFailure(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})-- installOne := func(kind, name string, installReal func() error, installPseudo func() error) (degraded bool) {- if err := installReal(); err != nil {- t.Logf("[%s %q] real install failed: %v", kind, name, err)- if pErr := installPseudo(); pErr != nil {- t.Fatalf("[%s %q] pseudo install also failed: %v", kind, name, pErr)- }- t.Logf("[%s %q] pseudo installed at same slot", kind, name)- return true- }- t.Logf("[%s %q] real install succeeded", kind, name)- return false- }-- // Step 1: install D. Real fails (nonexistent base), pseudo succeeds.- dName := "d_status"- dDegraded := installOne("domain", dName,- func() error {- return cat.DefineDomain(&ast.CreateDomainStmt{- Domainname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: dName},- }},- Typname: typeName("nonexistent_base_type_for_test_xyz"),- })- },- func() error {- return cat.DefineDomain(&ast.CreateDomainStmt{- Domainname: &ast.List{Items: []ast.Node{- &ast.String{Str: "public"},- &ast.String{Str: dName},- }},- Typname: typeName("text"),- })- },- )- if !dDegraded {- t.Fatal("expected D real install to fail (test premise broken)")- }-- // Step 2: install T. This is a REAL install (T's metadata is fine).- // The column references the pseudo D, so DefineRelation must find it.- tName := "records"- tDegraded := installOne("table", tName,- func() error {- return cat.DefineRelation(buildCreateTableStmt("public", tName, []*ast.ColumnDef{- columnDef("id", typeName("int4"), true),- columnDef("col", mustTypeName(t, "public."+dName), false),- columnDef("title", typeName("text"), false),- }), 'r')- },- func() error {- return errors.Errorf("should not reach pseudo for table %s", tName)- },- )- if tDegraded {- t.Fatal("T should install as REAL against pseudo D — cascade prevention violated")- }-- // Step 3: install V. REAL install. View body references T (which is real).- vName := "records_view"- vDegraded := installOne("view", vName,- func() error {- sel := mustParseSelect(t, `SELECT id, col, title FROM records WHERE id > 0`)- return cat.DefineView(&ast.ViewStmt{- View: &ast.RangeVar{- Schemaname: "public",- Relname: vName,- Relpersistence: 'p',- },- Query: sel,- })- },- func() error {- return errors.Errorf("should not reach pseudo for view %s", vName)- },- )- if vDegraded {- t.Fatal("V should install as REAL against real T — cascade prevention violated")- }-- // Step 4: query V. Verify lineage makes it all the way to base table columns.- q := mustAnalyze(t, cat, `SELECT id, col, title FROM records_view`)- if len(q.TargetList) != 3 {- t.Fatalf("view query: got %d targets, want 3", len(q.TargetList))- }- for i, te := range q.TargetList {- if _, isVar := te.Expr.(*catalog.VarExpr); !isVar {- t.Errorf("target[%d] (%s): expected VarExpr (resolved lineage), got %T",- i, te.ResName, te.Expr)- }- }-- // Step 5: query with WHERE on the pseudo-backed column — text semantics- // should let this analyze successfully.- q2, err := analyzeSafe(cat, `SELECT id FROM records_view WHERE col = 'active'`)- if err != nil {- t.Errorf("WHERE on pseudo-domain column should analyze; got err=%v", err)- } else if len(q2.TargetList) != 1 {- t.Errorf("WHERE query: got %d targets, want 1", len(q2.TargetList))- }-- t.Log("REAL cascade PoC: genuine DefineDomain failure → pseudo at same slot → " +- "downstream table and view install as real → query lineage intact")-}--// analyzeSafe runs AnalyzeSelectStmt but returns (nil, err) instead of failing-// the test — probe tests need to record whether analyze succeeded.-func analyzeSafe(cat *catalog.Catalog, sql string) (*catalog.Query, error) {- stmts, err := ParsePg(sql)- if err != nil {- return nil, errors.Wrap(err, "parse")- }- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("not a SelectStmt: %T", stmts[0].AST)- }- return cat.AnalyzeSelectStmt(sel)-}--func targetsOf(q *catalog.Query) int {- if q == nil {- return -1- }- return len(q.TargetList)-}diff --git a/backend/plugin/parser/pg/query_span_loader_pseudo.go b/backend/plugin/parser/pg/query_span_loader_pseudo.godeleted file mode 100644index 17d4d4109f..0000000000--- a/backend/plugin/parser/pg/query_span_loader_pseudo.go+++ /dev/null@@ -1,273 +0,0 @@-package pg--import (- "strings"-- "github.com/pkg/errors"-- "github.com/bytebase/omni/pg/ast"-)--// This file builds "pseudo" AST nodes used by the Catalog loader when a real-// install fails. Every pseudo form is backed by built-in types only (text,-// primarily) so pseudo install itself cannot cascade into further failures.-//-// Pseudo forms preserve object names and — where metadata allows — column-// names. Type fidelity is intentionally dropped. For query span this is-// sufficient in the vast majority of cases: lineage is a name-resolution-// problem, not a type-resolution problem.-//-// Every form in this file corresponds to a PoC test in-// query_span_v5_poc_test.go (TestLoaderPoC_Pseudo*).--// pseudoTextTypeName is the single pseudo column/parameter/return type omni-// accepts uniformly. Every pseudo form ends up routing to this shape.-func pseudoTextTypeName() *ast.TypeName {- return &ast.TypeName{- Names: &ast.List{Items: []ast.Node{&ast.String{Str: "text"}}},- Typemod: -1,- }-}--// pseudoCreateEnumStmt builds a minimal enum with zero values. omni's-// DefineEnum accepts an empty Vals list.-func pseudoCreateEnumStmt(schema, name string) *ast.CreateEnumStmt {- return &ast.CreateEnumStmt{- TypeName: qualifiedNameList(schema, name),- Vals: &ast.List{Items: []ast.Node{}},- }-}--// pseudoCreateDomainStmt builds a domain over text with no constraints.-// Query span does not inspect domain constraints, so dropping them is safe.-func pseudoCreateDomainStmt(schema, name string) *ast.CreateDomainStmt {- return &ast.CreateDomainStmt{- Domainname: qualifiedNameList(schema, name),- Typname: pseudoTextTypeName(),- }-}--// pseudoCompositeTypeStmt builds a composite type with a single text field-// named "_broken". It is the fallback when a composite type's real-// definition is absent from metadata or fails to install, so (col).field-// access against a pseudo composite will fall through to-// extractFallbackColumns.-func pseudoCompositeTypeStmt(schema, name string) *ast.CompositeTypeStmt {- return &ast.CompositeTypeStmt{- Typevar: &ast.RangeVar{- Schemaname: schema,- Relname: name,- },- Coldeflist: &ast.List{Items: []ast.Node{- &ast.ColumnDef{- Colname: "_broken",- TypeName: pseudoTextTypeName(),- },- }},- }-}--// pseudoCreateRangeStmt builds a range over text. text has btree ordering so-// omni accepts it as a valid subtype.-func pseudoCreateRangeStmt(schema, name string) *ast.CreateRangeStmt {- return &ast.CreateRangeStmt{- TypeName: qualifiedNameList(schema, name),- Params: &ast.List{Items: []ast.Node{- &ast.DefElem{- Defname: "subtype",- Arg: pseudoTextTypeName(),- },- }},- }-}--// pseudoCreateTableStmt builds a table whose columns are all text-typed but-// retain their metadata names. This is the fallback used when real install-// fails because of unresolved column types (broken enum, unknown composite,-// etc.).-func pseudoCreateTableStmt(schema, name string, columnNames []string) *ast.CreateStmt {- items := make([]ast.Node, 0, len(columnNames))- for _, col := range columnNames {- if col == "" {- continue- }- items = append(items, &ast.ColumnDef{- Colname: col,- TypeName: pseudoTextTypeName(),- })- }- return &ast.CreateStmt{- Relation: &ast.RangeVar{- Schemaname: schema,- Relname: name,- Relpersistence: 'p',- },- TableElts: &ast.List{Items: items},- }-}--// pseudoViewStmt builds a view whose body is a constant SELECT target list-// exposing each metadata column name as a nullable text expression. The body-// has no FROM clause.-//-// Returns an error if the body cannot be parsed; callers should treat this-// as a "truly broken" object.-func pseudoViewStmt(schema, name string, columnNames []string) (*ast.ViewStmt, error) {- sel, err := pseudoConstantSelect(columnNames)- if err != nil {- return nil, err- }- return &ast.ViewStmt{- View: &ast.RangeVar{- Schemaname: schema,- Relname: name,- Relpersistence: 'p',- },- Query: sel,- }, nil-}--// pseudoCreateTableAsStmt builds a pseudo materialized view using the same-// constant-target-list body as pseudoViewStmt.-func pseudoCreateTableAsStmt(schema, name string, columnNames []string) (*ast.CreateTableAsStmt, error) {- sel, err := pseudoConstantSelect(columnNames)- if err != nil {- return nil, err- }- return &ast.CreateTableAsStmt{- Query: sel,- Objtype: ast.OBJECT_MATVIEW,- Into: &ast.IntoClause{- Rel: &ast.RangeVar{- Schemaname: schema,- Relname: name,- Relpersistence: 'p',- },- },- }, nil-}--// pseudoCreateFunctionStmt builds a function whose signature collapses every-// parameter to text and whose return type is text. argCount is taken from the-// real function's signature so analyzer overload resolution by arity still-// matches against the pseudo.-//-// The function body is `SELECT $1` when argCount > 0, `SELECT NULL::text`-// otherwise — always a valid SQL-language body.-func pseudoCreateFunctionStmt(schema, name string, argCount int) *ast.CreateFunctionStmt {- stmt := &ast.CreateFunctionStmt{- Funcname: qualifiedNameList(schema, name),- ReturnType: pseudoTextTypeName(),- }- if argCount > 0 {- params := make([]ast.Node, argCount)- for i := range argCount {- params[i] = &ast.FunctionParameter{- ArgType: pseudoTextTypeName(),- Mode: ast.FUNC_PARAM_IN,- }- }- stmt.Parameters = &ast.List{Items: params}- }- body := "SELECT NULL::text"- if argCount > 0 {- body = "SELECT $1::text"- }- stmt.Options = &ast.List{Items: []ast.Node{- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},- &ast.DefElem{- Defname: "as",- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: body}}},- },- }}- return stmt-}--// pseudoConstantSelect parses "SELECT NULL::text AS col1, NULL::text AS col2 ..."-// and returns the resulting *ast.SelectStmt. Columns with empty names are-// skipped; if the resulting target list is empty, a single anonymous column-// is emitted so the SELECT is well-formed.-func pseudoConstantSelect(columnNames []string) (*ast.SelectStmt, error) {- var targets []string- for _, col := range columnNames {- if col == "" {- continue- }- targets = append(targets, "NULL::text AS "+quoteIdent(col))- }- if len(targets) == 0 {- targets = []string{"NULL::text"}- }- sql := "SELECT " + strings.Join(targets, ", ")- stmts, err := ParsePg(sql)- if err != nil {- return nil, errors.Wrap(err, "parse pseudo constant select")- }- if len(stmts) != 1 {- return nil, errors.Errorf("pseudo constant select: expected 1 statement, got %d", len(stmts))- }- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("pseudo constant select: expected SelectStmt, got %T", stmts[0].AST)- }- return sel, nil-}--// functionArgCountFromSignature parses a signature string like-// "my_func(integer, text)" and returns the number of arguments. A signature-// with no parentheses or an empty argument list returns 0.-func functionArgCountFromSignature(signature string) int {- openIdx := strings.Index(signature, "(")- if openIdx < 0 {- return 0- }- closeIdx := strings.LastIndex(signature, ")")- if closeIdx < 0 || closeIdx <= openIdx {- return 0- }- inner := strings.TrimSpace(signature[openIdx+1 : closeIdx])- if inner == "" {- return 0- }- // Split on top-level commas. Type names inside parens (rare, e.g.- // numeric(10,2)) must not count as separators.- depth := 0- count := 1- for _, r := range inner {- switch r {- case '(':- depth++- case ')':- if depth > 0 {- depth--- }- case ',':- if depth == 0 {- count++- }- default:- // other runes contribute nothing to arg-count accounting- }- }- return count-}--// ---------------- small AST helpers shared with builders.go ------------------// qualifiedNameList builds an omni List of String nodes representing a-// schema-qualified object name. An empty schema emits only the name.-func qualifiedNameList(schema, name string) *ast.List {- items := make([]ast.Node, 0, 2)- if schema != "" {- items = append(items, &ast.String{Str: schema})- }- items = append(items, &ast.String{Str: name})- return &ast.List{Items: items}-}--// quoteIdent wraps an identifier in double quotes, doubling any internal-// double quotes. Used when a metadata column name may contain special chars-// or collide with SQL reserved words.-func quoteIdent(name string) string {- return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`-}diff --git a/backend/plugin/parser/pg/query_span_loader_test.go b/backend/plugin/parser/pg/query_span_loader_test.godeleted file mode 100644index ce0d2a8773..0000000000--- a/backend/plugin/parser/pg/query_span_loader_test.go+++ /dev/null@@ -1,911 +0,0 @@-package pg--import (- "context"- "strings"- "testing"-- "github.com/pkg/errors"-- metadatapb "github.com/bytebase/omni/metadata"- "github.com/bytebase/omni/pg/ast"- "github.com/bytebase/omni/pg/catalog"-)--// ---------------- typeNameFromString ------------------func TestLoaderTypeNameFromString(t *testing.T) {- cases := []struct {- in string- wantErr bool- }{- {"int4", false},- {"text", false},- {"numeric(10,2)", false},- {"timestamp(3) with time zone", false},- {"character varying(255)", false},- {"text[]", false},- {"public.task_status", false},- {`"public"."task_status"`, false},- {"", true}, // empty cannot parse as SELECT NULL::- }- for _, c := range cases {- tn, err := typeNameFromString(c.in)- if c.wantErr {- if err == nil {- t.Errorf("typeNameFromString(%q): expected error, got tn=%+v", c.in, tn)- }- continue- }- if err != nil {- t.Errorf("typeNameFromString(%q): unexpected error: %v", c.in, err)- continue- }- if tn == nil {- t.Errorf("typeNameFromString(%q): nil TypeName with no error", c.in)- }- }-}--// ---------------- extractUserTypeRefs ------------------func TestLoaderExtractUserTypeRefs(t *testing.T) {- cases := []struct {- in string- want []UserTypeRef- }{- // Built-in scalars.- {"integer", nil},- {"int4", nil},- {"text", nil},- {"bigint", nil},- {"boolean", nil},- {"json", nil},- {"jsonb", nil},- {"uuid", nil},- {"date", nil},- {"interval", nil},-- // Built-in with length / precision / timezone.- {"numeric(10,2)", nil},- {"numeric(8)", nil},- {"decimal(10,2)", nil},- {"character varying(255)", nil},- {"character(10)", nil},- {"varchar(255)", nil},- {"bit(8)", nil},- {"bit varying(8)", nil},- {"timestamp(3) with time zone", nil},- {"time(6) without time zone", nil},- {"timestamp without time zone", nil},-- // USER-DEFINED.- {"public.task_status", []UserTypeRef{{Schema: "public", Name: "task_status"}}},- {"myschema.my_domain", []UserTypeRef{{Schema: "myschema", Name: "my_domain"}}},-- // Quoted qualified name.- {`"MySchema"."MyType"`, []UserTypeRef{{Schema: "MySchema", Name: "MyType"}}},-- // PG internal array form.- {"_text", nil},- {"_int4", nil},- {"_task_status", nil}, // sync.go:834 drops the schema; we do not topo-sort it.-- // System schemas.- {"pg_catalog.int4", nil},- {"information_schema.cardinal_number", nil},- {"pg_toast.pg_toast_2619", nil},-- // Empty.- {"", nil},-- // Whitespace.- {" text ", nil},- }- for _, c := range cases {- got := extractUserTypeRefs(c.in)- if !userTypeRefsEqual(got, c.want) {- t.Errorf("extractUserTypeRefs(%q): got %+v, want %+v", c.in, got, c.want)- }- }-}--func userTypeRefsEqual(a, b []UserTypeRef) bool {- if len(a) != len(b) {- return false- }- for i := range a {- if a[i] != b[i] {- return false- }- }- return true-}--// ---------------- helpers ------------------func TestLoaderStripTypeModifiers(t *testing.T) {- cases := []struct {- in, want string- }{- {"numeric(10,2)", "numeric"},- {"timestamp(3) with time zone", "timestamp"},- {"timestamp without time zone", "timestamp"},- {"character varying(255)", "character varying"},- {" text ", "text"},- {"public.task_status", "public.task_status"},- {"", ""},- }- for _, c := range cases {- if got := stripTypeModifiers(c.in); got != c.want {- t.Errorf("stripTypeModifiers(%q): got %q, want %q", c.in, got, c.want)- }- }-}--func TestLoaderSplitQualifiedName(t *testing.T) {- cases := []struct {- in string- wantSchema string- wantName string- wantOk bool- }{- {"public.foo", "public", "foo", true},- {`"weird name"."another"`, "weird name", "another", true},- {"foo", "", "", false},- {"", "", "", false},- {"a.b.c", "", "", false}, // too many parts- }- for _, c := range cases {- s, n, ok := splitQualifiedName(c.in)- if s != c.wantSchema || n != c.wantName || ok != c.wantOk {- t.Errorf("splitQualifiedName(%q): got (%q,%q,%v), want (%q,%q,%v)",- c.in, s, n, ok, c.wantSchema, c.wantName, c.wantOk)- }- }-}--func TestLoaderIsSystemSchema(t *testing.T) {- // Sanity check that extractUserTypeRefs routes system schemas to nil via- // IsSystemSchema. The underlying predicate is tested in system_objects.go.- cases := map[string]bool{- "pg_catalog.foo": true,- "information_schema.bar": true,- "pg_toast.baz": true,- "pg_temp_1.qux": true,- "public.foo": false,- "myschema.bar": false,- }- for in, wantFiltered := range cases {- got := extractUserTypeRefs(in)- if wantFiltered && got != nil {- t.Errorf("extractUserTypeRefs(%q): expected filtered, got %+v", in, got)- }- if !wantFiltered && got == nil {- t.Errorf("extractUserTypeRefs(%q): expected non-nil, got nil", in)- }- }-}--// ---------------- classifyAnalyzeError ------------------func TestLoaderClassifyAnalyzeError(t *testing.T) {- cases := []struct {- name string- err error- want fallbackReason- }{- {"nil", nil, reasonNone},- {- "undefined function",- &catalog.Error{Code: catalog.CodeUndefinedFunction, Message: "function fn(text) does not exist"},- reasonExpectedPseudoSemantic,- },- {- "ambiguous function",- &catalog.Error{Code: catalog.CodeAmbiguousFunction, Message: "function fn is not unique"},- reasonExpectedPseudoSemantic,- },- {- "datatype mismatch",- &catalog.Error{Code: catalog.CodeDatatypeMismatch, Message: "UNION types text and integer cannot be matched"},- reasonExpectedPseudoSemantic,- },- {- "feature not supported",- &catalog.Error{Code: catalog.CodeFeatureNotSupported, Message: "not supported"},- reasonExpectedPseudoSemantic,- },- {- "ambiguous column",- &catalog.Error{Code: catalog.CodeAmbiguousColumn, Message: "column x is ambiguous"},- reasonExpectedPseudoSemantic,- },- {- "undefined table",- &catalog.Error{Code: catalog.CodeUndefinedTable, Message: `relation "foo" does not exist`},- reasonUndefinedReference,- },- {- "undefined column",- &catalog.Error{Code: catalog.CodeUndefinedColumn, Message: `column "x" does not exist`},- reasonUndefinedReference,- },- {- "undefined object",- &catalog.Error{Code: catalog.CodeUndefinedObject, Message: `type "t" does not exist`},- reasonUndefinedReference,- },- {- "undefined schema",- &catalog.Error{Code: catalog.CodeUndefinedSchema, Message: `schema "s" does not exist`},- reasonUndefinedReference,- },- {- "other catalog error",- &catalog.Error{Code: catalog.CodeDuplicateTable, Message: "duplicate"},- reasonAnalyzerUnsupported,- },- {- "plain error",- errors.New("unsupported node type"),- reasonAnalyzerUnsupported,- },- {- "wrapped catalog error",- errors.Wrap(&catalog.Error{Code: catalog.CodeDatatypeMismatch, Message: "mismatch"}, "ctx"),- reasonExpectedPseudoSemantic,- },- }- for _, c := range cases {- if got := classifyAnalyzeError(c.err); got != c.want {- t.Errorf("%s: classifyAnalyzeError got %v, want %v", c.name, got, c.want)- }- }-}--// ---------------- pseudo builders ------------------func TestLoaderPseudoEnum(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- if err := cat.DefineEnum(pseudoCreateEnumStmt("public", "broken")); err != nil {- t.Fatalf("DefineEnum: %v", err)- }- // Reference it from a table column and analyze.- tn, err := typeNameFromString("public.broken")- if err != nil {- t.Fatalf("typeNameFromString: %v", err)- }- tbl := &ast.CreateStmt{- Relation: &ast.RangeVar{Schemaname: "public", Relname: "t", Relpersistence: 'p'},- TableElts: &ast.List{Items: []ast.Node{- &ast.ColumnDef{Colname: "s", TypeName: tn},- }},- }- if err := cat.DefineRelation(tbl, 'r'); err != nil {- t.Fatalf("DefineRelation with pseudo enum column: %v", err)- }-}--func TestLoaderPseudoDomain(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- if err := cat.DefineDomain(pseudoCreateDomainStmt("public", "d")); err != nil {- t.Fatalf("DefineDomain: %v", err)- }-}--func TestLoaderPseudoComposite(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- if err := cat.DefineCompositeType(pseudoCompositeTypeStmt("public", "c")); err != nil {- t.Fatalf("DefineCompositeType: %v", err)- }-}--func TestLoaderPseudoRange(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- if err := cat.DefineRange(pseudoCreateRangeStmt("public", "r")); err != nil {- t.Fatalf("DefineRange: %v", err)- }-}--func TestLoaderPseudoTable(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- stmt := pseudoCreateTableStmt("public", "t", []string{"id", "name", "email"})- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation: %v", err)- }- rel := cat.GetRelation("public", "t")- if rel == nil {- t.Fatal("relation not installed")- }- if len(rel.Columns) != 3 {- t.Errorf("column count: got %d, want 3", len(rel.Columns))- }-}--func TestLoaderPseudoView(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- stmt, err := pseudoViewStmt("public", "v", []string{"id", "label"})- if err != nil {- t.Fatalf("build pseudo view: %v", err)- }- if err := cat.DefineView(stmt); err != nil {- t.Fatalf("DefineView: %v", err)- }- rel := cat.GetRelation("public", "v")- if rel == nil {- t.Fatal("view not installed")- }- if len(rel.Columns) != 2 {- t.Errorf("view column count: got %d, want 2", len(rel.Columns))- }-}--func TestLoaderPseudoMatView(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- stmt, err := pseudoCreateTableAsStmt("public", "m", []string{"x", "y"})- if err != nil {- t.Fatalf("build pseudo matview: %v", err)- }- if err := cat.ExecCreateTableAs(stmt); err != nil {- t.Fatalf("ExecCreateTableAs: %v", err)- }-}--func TestLoaderPseudoFunction(t *testing.T) {- cases := []struct {- name string- argCount int- }{- {"zero args", 0},- {"one arg", 1},- {"three args", 3},- }- for _, c := range cases {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- stmt := pseudoCreateFunctionStmt("public", "fn", c.argCount)- if err := cat.CreateFunctionStmt(stmt); err != nil {- t.Errorf("%s: CreateFunctionStmt: %v", c.name, err)- continue- }- procs := cat.LookupProcByName("fn")- if len(procs) == 0 {- t.Errorf("%s: no procs installed", c.name)- }- }-}--func TestLoaderFunctionArgCountFromSignature(t *testing.T) {- cases := map[string]int{- "": 0,- "fn()": 0,- "fn(integer)": 1,- "fn(integer, text)": 2,- "fn(integer, text, uuid)": 3,- "fn(numeric(10,2))": 1, // inner paren must not split- "fn(numeric(10,2), int)": 2,- }- for in, want := range cases {- if got := functionArgCountFromSignature(in); got != want {- t.Errorf("functionArgCountFromSignature(%q): got %d, want %d", in, got, want)- }- }-}--func TestLoaderQualifiedNameList(t *testing.T) {- list := qualifiedNameList("public", "foo")- if list == nil || len(list.Items) != 2 {- t.Fatalf("qualifiedNameList(public, foo): got %+v, want 2 items", list)- }- bare := qualifiedNameList("", "foo")- if bare == nil || len(bare.Items) != 1 {- t.Fatalf("qualifiedNameList(\"\", foo): got %+v, want 1 item", bare)- }-}--// ---------------- real builders ------------------func TestLoaderBuildCreateEnumStmt(t *testing.T) {- stmt := buildCreateEnumStmt("public", &metadatapb.EnumTypeMetadata{- Name: "task_status",- Values: []string{"pending", "running", "done"},- })- if stmt == nil || stmt.Vals == nil || len(stmt.Vals.Items) != 3 {- t.Fatalf("unexpected: %+v", stmt)- }- cat := catalog.New()- if err := cat.DefineEnum(stmt); err != nil {- t.Fatalf("DefineEnum: %v", err)- }-}--func TestLoaderBuildCreateStmt(t *testing.T) {- stmt, err := buildCreateStmt("public", &metadatapb.TableMetadata{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4", Nullable: false},- {Name: "name", Type: "text", Nullable: true},- {Name: "amount", Type: "numeric(10,2)", Nullable: true},- },- })- if err != nil {- t.Fatalf("buildCreateStmt: %v", err)- }- if stmt == nil || stmt.TableElts == nil || len(stmt.TableElts.Items) != 3 {- t.Fatalf("unexpected: %+v", stmt)- }- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- if err := cat.DefineRelation(stmt, 'r'); err != nil {- t.Fatalf("DefineRelation: %v", err)- }- rel := cat.GetRelation("public", "t")- if rel == nil || len(rel.Columns) != 3 {- t.Fatalf("rel: %+v", rel)- }-}--func TestLoaderBuildCreateStmt_BadTypeErrors(t *testing.T) {- _, err := buildCreateStmt("public", &metadatapb.TableMetadata{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "bad", Type: "this is not a type at all;;"},- },- })- if err == nil {- t.Fatal("expected error for bad column type")- }-}--func TestLoaderBuildCreateStmt_EmptyTypeErrors(t *testing.T) {- _, err := buildCreateStmt("public", &metadatapb.TableMetadata{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{{Name: "x", Type: ""}},- })- if err == nil {- t.Fatal("expected error for empty column type")- }-}--func TestLoaderBuildViewStmt(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- base, err := buildCreateStmt("public", &metadatapb.TableMetadata{- Name: "orders",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- {Name: "total", Type: "int4"},- },- })- if err != nil {- t.Fatalf("build base: %v", err)- }- if err := cat.DefineRelation(base, 'r'); err != nil {- t.Fatalf("install base: %v", err)- }- stmt, err := buildViewStmt("public", &metadatapb.ViewMetadata{- Name: "v",- Definition: "SELECT id, total FROM orders",- })- if err != nil {- t.Fatalf("buildViewStmt: %v", err)- }- if err := cat.DefineView(stmt); err != nil {- t.Fatalf("DefineView: %v", err)- }-}--func TestLoaderBuildViewStmt_EmptyDefinition(t *testing.T) {- _, err := buildViewStmt("public", &metadatapb.ViewMetadata{Name: "v"})- if err == nil {- t.Fatal("expected error for empty definition")- }-}--func TestLoaderBuildCreateTableAsStmt(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- base, err := buildCreateStmt("public", &metadatapb.TableMetadata{- Name: "orders",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- },- })- if err != nil {- t.Fatalf("build base: %v", err)- }- if err := cat.DefineRelation(base, 'r'); err != nil {- t.Fatalf("install base: %v", err)- }- stmt, err := buildCreateTableAsStmt("public", &metadatapb.MaterializedViewMetadata{- Name: "m",- Definition: "SELECT id FROM orders",- })- if err != nil {- t.Fatalf("buildCreateTableAsStmt: %v", err)- }- if err := cat.ExecCreateTableAs(stmt); err != nil {- t.Fatalf("ExecCreateTableAs: %v", err)- }-}--func TestLoaderBuildCreateFunctionStmt(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- stmt, err := buildCreateFunctionStmt("public", &metadatapb.FunctionMetadata{- Name: "fn",- Signature: "fn(integer, text)",- })- if err != nil {- t.Fatalf("buildCreateFunctionStmt: %v", err)- }- if stmt.Parameters == nil || len(stmt.Parameters.Items) != 2 {- t.Fatalf("expected 2 params, got %+v", stmt.Parameters)- }- if err := cat.CreateFunctionStmt(stmt); err != nil {- t.Fatalf("CreateFunctionStmt: %v", err)- }-}--func TestLoaderBuildCreateFunctionStmt_ZeroArgs(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- stmt, err := buildCreateFunctionStmt("public", &metadatapb.FunctionMetadata{- Name: "fn",- Signature: "fn()",- })- if err != nil {- t.Fatalf("buildCreateFunctionStmt: %v", err)- }- if err := cat.CreateFunctionStmt(stmt); err != nil {- t.Fatalf("CreateFunctionStmt: %v", err)- }-}--func TestLoaderParseFunctionSignatureArgTypes(t *testing.T) {- cases := map[string][]string{- "fn()": nil,- "fn(integer)": {"integer"},- "fn(integer, text)": {"integer", "text"},- "fn(numeric(10,2), text)": {"numeric(10,2)", "text"},- "fn( int4 , text )": {"int4", "text"},- "fn": nil,- }- for in, want := range cases {- got, err := parseFunctionSignatureArgTypes(in)- if err != nil {- t.Errorf("%q: unexpected error %v", in, err)- continue- }- if !stringSlicesEqual(got, want) {- t.Errorf("%q: got %v, want %v", in, got, want)- }- }-}--func stringSlicesEqual(a, b []string) bool {- if len(a) != len(b) {- return false- }- for i := range a {- if a[i] != b[i] {- return false- }- }- return true-}--func TestLoaderParseSelectBody(t *testing.T) {- sel, err := parseSelectBody("SELECT 1 AS x")- if err != nil {- t.Fatalf("parseSelectBody: %v", err)- }- if sel == nil {- t.Fatal("nil sel")- }- if _, err := parseSelectBody(""); err == nil {- t.Error("expected error for empty body")- }- if _, err := parseSelectBody("SELECT 1; SELECT 2"); err == nil {- t.Error("expected error for multi-statement body")- }-}--// ---------------- loader ------------------func TestLoader_HappyPath(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- EnumTypes: []*metadatapb.EnumTypeMetadata{- {Name: "task_status", Values: []string{"pending", "running"}},- },- Tables: []*metadatapb.TableMetadata{{- Name: "tasks",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- {Name: "status", Type: "public.task_status"},- {Name: "title", Type: "text"},- },- }},- Views: []*metadatapb.ViewMetadata{{- Name: "open_tasks",- Definition: "SELECT id, title, status FROM tasks",- DependencyColumns: []*metadatapb.DependencyColumn{- {Schema: "public", Table: "tasks", Column: "id"},- {Schema: "public", Table: "tasks", Column: "title"},- {Schema: "public", Table: "tasks", Column: "status"},- },- }},- }},- }- loader := newCatalogLoader(cat, meta)- if err := loader.Load(context.Background()); err != nil {- t.Fatalf("Load: %v", err)- }- if len(loader.degraded) != 0 {- t.Errorf("expected zero degraded, got %+v", loader.degraded)- }- if len(loader.trulyBroken) != 0 {- t.Errorf("expected zero truly broken, got %+v", loader.trulyBroken)- }- if cat.GetRelation("public", "tasks") == nil {- t.Error("tasks not installed")- }- if cat.GetRelation("public", "open_tasks") == nil {- t.Error("open_tasks not installed")- }-}--func TestLoader_BrokenEnumCascadesToPseudo(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Tables: []*metadatapb.TableMetadata{{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- // Reference a user type that is not in metadata. The- // real install will fail; pseudo should take over.- {Name: "status", Type: "public.nonexistent_enum"},- },- }},- }},- }- loader := newCatalogLoader(cat, meta)- if err := loader.Load(context.Background()); err != nil {- t.Fatalf("Load: %v", err)- }- key := "rel:public.t"- if _, ok := loader.degraded[key]; !ok {- t.Errorf("expected public.t to be degraded, got %+v", loader.degraded)- }- if _, ok := loader.trulyBroken[key]; ok {- t.Errorf("unexpected truly broken: %+v", loader.trulyBroken)- }- rel := cat.GetRelation("public", "t")- if rel == nil {- t.Fatal("pseudo t not installed")- }- if len(rel.Columns) != 2 {- t.Errorf("pseudo t: got %d cols, want 2", len(rel.Columns))- }-}--func TestLoader_TopoOrder_DependencyBeforeUse(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- // View is listed BEFORE its base table in metadata. The loader- // must reorder so the table is installed first; otherwise the- // view body analyzes against an empty catalog.- Views: []*metadatapb.ViewMetadata{{- Name: "v",- Definition: "SELECT id FROM base",- DependencyColumns: []*metadatapb.DependencyColumn{- {Schema: "public", Table: "base", Column: "id"},- },- }},- Tables: []*metadatapb.TableMetadata{{- Name: "base",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int4"},- },- }},- }},- }- loader := newCatalogLoader(cat, meta)- if err := loader.Load(context.Background()); err != nil {- t.Fatalf("Load: %v", err)- }- if loader.degraded["rel:public.v"] != nil {- t.Errorf("view should install real, got degraded: %v", loader.degraded["rel:public.v"])- }- if loader.degraded["rel:public.base"] != nil {- t.Errorf("base should install real, got degraded: %v", loader.degraded["rel:public.base"])- }-}--func TestLoader_CycleBreaking(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- // Two views referencing each other — a metadata-level cycle.- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Views: []*metadatapb.ViewMetadata{- {- Name: "v_alpha",- Definition: "SELECT id FROM v_beta",- DependencyColumns: []*metadatapb.DependencyColumn{- {Schema: "public", Table: "v_beta", Column: "id"},- },- },- {- Name: "v_beta",- Definition: "SELECT id FROM v_alpha",- DependencyColumns: []*metadatapb.DependencyColumn{- {Schema: "public", Table: "v_alpha", Column: "id"},- },- },- },- }},- }- loader := newCatalogLoader(cat, meta)- if err := loader.Load(context.Background()); err != nil {- t.Fatalf("Load: %v", err)- }- // Both views should be in the catalog (real or pseudo). At least one- // should be degraded (the lex-first member of the SCC).- if cat.GetRelation("public", "v_alpha") == nil {- t.Error("v_alpha missing")- }- if cat.GetRelation("public", "v_beta") == nil {- t.Error("v_beta missing")- }- if len(loader.degraded) == 0 {- t.Error("expected at least one degraded in SCC")- }- // v_alpha is lex-first by sortKey, so it should be the pseudo root.- if _, ok := loader.degraded["rel:public.v_alpha"]; !ok {- t.Errorf("lex-first v_alpha should be degraded, got %+v", loader.degraded)- }-}--func TestLoader_LoaderObjectsTracksEverything(t *testing.T) {- cat := catalog.New()- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- EnumTypes: []*metadatapb.EnumTypeMetadata{- {Name: "e", Values: []string{"a"}},- },- Tables: []*metadatapb.TableMetadata{{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{{Name: "id", Type: "int4"}},- }},- }},- }- loader := newCatalogLoader(cat, meta)- _ = loader.Load(context.Background())- if !loader.loaderObjects["schema:public"] {- t.Error("schema:public not tracked")- }- if !loader.loaderObjects["type:public.e"] {- t.Error("type:public.e not tracked")- }- if !loader.loaderObjects["rel:public.t"] {- t.Error("rel:public.t not tracked")- }-}--func TestLoader_NilMetaIsNoop(t *testing.T) {- cat := catalog.New()- loader := newCatalogLoader(cat, nil)- if err := loader.Load(context.Background()); err != nil {- t.Errorf("Load(nil meta): %v", err)- }-}--func TestLoader_NilCatalogErrors(t *testing.T) {- loader := newCatalogLoader(nil, &metadatapb.DatabaseSchemaMetadata{})- if err := loader.Load(context.Background()); err == nil {- t.Error("expected error for nil catalog")- }-}--// TestLoader_FunctionOverloadsAllInstalled guards against a regression where-// multiple FunctionMetadata entries sharing a name collapsed into a single-// loader object because the loader key didn't include the signature. Both-// overloads must end up in the catalog under the same name.-func TestLoader_FunctionOverloadsAllInstalled(t *testing.T) {- cat := catalog.New()- cat.SetSearchPath([]string{"public"})- meta := &metadatapb.DatabaseSchemaMetadata{- Schemas: []*metadatapb.SchemaMetadata{{- Name: "public",- Functions: []*metadatapb.FunctionMetadata{- {- Name: "fn",- Signature: "fn(integer)",- Definition: "CREATE OR REPLACE FUNCTION public.fn(x int) RETURNS int LANGUAGE sql AS $$ SELECT $1 $$;",- },- {- Name: "fn",- Signature: "fn(text)",- Definition: "CREATE OR REPLACE FUNCTION public.fn(x text) RETURNS text LANGUAGE sql AS $$ SELECT $1 $$;",- },- },- }},- }- loader := newCatalogLoader(cat, meta)- if err := loader.Load(context.Background()); err != nil {- t.Fatalf("Load: %v", err)- }- if len(loader.degraded) != 0 {- t.Errorf("expected zero degraded, got %+v", loader.degraded)- }- procs := cat.LookupProcByName("fn")- if len(procs) < 2 {- t.Errorf("expected both overloads installed, got %d procs", len(procs))- }- fnEntries := 0- for k := range loader.loaderObjects {- if strings.HasPrefix(k, "func:public.fn|") {- fnEntries++- }- }- if fnEntries != 2 {- t.Errorf("expected 2 distinct fn loader-object keys, got %d: %+v",- fnEntries, loader.loaderObjects)- }-}--func TestLoaderTopoSort_Determinism(t *testing.T) {- // Build a few fixtures and ensure topoSortObjects returns the same order- // for the same input.- objects := []*objectEntry{- {kind: kindSchema, schema: "public", name: "public"},- {kind: kindTable, schema: "public", name: "b", tableMeta: &metadatapb.TableMetadata{Name: "b"}},- {kind: kindTable, schema: "public", name: "a", tableMeta: &metadatapb.TableMetadata{Name: "a"}},- {kind: kindEnum, schema: "public", name: "e", enumMeta: &metadatapb.EnumTypeMetadata{Name: "e"}},- }- first := topoSortObjects(objects)- second := topoSortObjects(objects)- if len(first) != len(second) {- t.Fatal("length mismatch")- }- for i := range first {- if first[i].key() != second[i].key() {- t.Errorf("non-deterministic: [%d] %s vs %s", i, first[i].key(), second[i].key())- }- }-}--func TestLoaderFallbackReasonString(t *testing.T) {- cases := map[fallbackReason]string{- reasonNone: "none",- reasonExpectedPseudoSemantic: "expected_pseudo_semantic",- reasonUndefinedReference: "undefined_reference",- reasonAnalyzerUnsupported: "analyzer_unsupported",- }- for r, want := range cases {- if got := r.String(); got != want {- t.Errorf("fallbackReason(%d).String(): got %q, want %q", r, got, want)- }- }-}diff --git a/backend/plugin/parser/pg/query_span_loader_type_name.go b/backend/plugin/parser/pg/query_span_loader_type_name.godeleted file mode 100644index 191502df5a..0000000000--- a/backend/plugin/parser/pg/query_span_loader_type_name.go+++ /dev/null@@ -1,285 +0,0 @@-package pg--import (- "strings"-- "github.com/pkg/errors"-- "github.com/bytebase/omni/pg/ast"-)--// UserTypeRef names a user-defined type referenced by a column, function-// parameter, or function return type. Extracted from type strings produced by-// the PG sync layer (sync.go:820-872).-type UserTypeRef struct {- Schema string- Name string-}--// typeNameFromString parses a PG type string into an *ast.TypeName by running-// it through omni's SELECT parser via the "SELECT NULL::<type>" trick. This-// reuses the stable SELECT parse path and avoids the DDL parse path that-// breaks on BYT-9215 / BYT-9261 class inputs.-//-// Callers must only pass values originating from bytebase sync metadata. The-// guardrails (single-statement, single-target) reject most malformed inputs-// but are not a sanitization layer for arbitrary user input.-func typeNameFromString(typeStr string) (*ast.TypeName, error) {- stmts, err := ParsePg("SELECT NULL::" + typeStr)- if err != nil {- return nil, errors.Wrapf(err, "parse type %q", typeStr)- }- if len(stmts) != 1 {- return nil, errors.Errorf("type %q: expected 1 statement, got %d", typeStr, len(stmts))- }- sel, ok := stmts[0].AST.(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("type %q: expected SelectStmt, got %T", typeStr, stmts[0].AST)- }- if sel.TargetList == nil || len(sel.TargetList.Items) != 1 {- return nil, errors.Errorf("type %q: expected 1 target", typeStr)- }- rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)- if !ok {- return nil, errors.Errorf("type %q: expected ResTarget, got %T", typeStr, sel.TargetList.Items[0])- }- cast, ok := rt.Val.(*ast.TypeCast)- if !ok {- return nil, errors.Errorf("type %q: expected TypeCast, got %T", typeStr, rt.Val)- }- return cast.TypeName, nil-}--// extractUserTypeRefs returns the user-defined type references embedded in a-// PG type string. Built-in types, PG internal array forms (_name), and-// system-schema-qualified types return nil.-//-// Soundness rule (hard contract C5): false negatives are acceptable (the-// loader's pseudo fallback catches them); false positives are not, because-// they invent edges in the dependency graph.-func extractUserTypeRefs(typeStr string) []UserTypeRef {- if typeStr == "" {- return nil- }- base := stripTypeModifiers(typeStr)- if base == "" || isBuiltinType(base) {- return nil- }- if strings.HasPrefix(base, "_") {- // PG internal array form. The element may be a user type but the- // schema is not recoverable from this representation — see the- // sync.go:834 note in the plan.- return nil- }- schema, name, ok := splitQualifiedName(base)- if !ok {- return nil- }- if IsSystemSchema(schema) {- return nil- }- return []UserTypeRef{{Schema: schema, Name: name}}-}--// stripTypeModifiers normalizes a PG type string to its bare base-type form-// for allow-list lookup. It strips:-// - parenthesized type modifiers: "numeric(10,2)" -> "numeric"-// - trailing time-zone suffixes: "timestamp(3) with time zone" -> "timestamp"-// - surrounding whitespace-//-// The result may still be a qualified name like "public.task_status"; that is-// the caller's job to split.-func stripTypeModifiers(typeStr string) string {- s := strings.TrimSpace(typeStr)- // Array suffixes wrap the element type: "public.addr[]" refers to- // public.addr. Quoted identifiers end with a quote, so a genuine name- // containing "[]" is never clipped.- for strings.HasSuffix(s, "[]") {- s = strings.TrimSpace(strings.TrimSuffix(s, "[]"))- }- s = stripTimeZoneSuffix(s)- s = stripParens(s)- return strings.TrimSpace(s)-}--// stripTimeZoneSuffix removes a trailing "with time zone" / "without time-// zone" clause. Idempotent on inputs that do not carry the suffix.-func stripTimeZoneSuffix(s string) string {- lower := strings.ToLower(s)- for _, suffix := range []string{" with time zone", " without time zone"} {- if strings.HasSuffix(lower, suffix) {- return s[:len(s)-len(suffix)]- }- }- return s-}--// stripParens removes the first parenthesized chunk from a type string.-// "numeric(10,2)" -> "numeric"-// "timestamp(3)" -> "timestamp"-// "character varying(255)" -> "character varying"-func stripParens(s string) string {- before, _, found := strings.Cut(s, "(")- if !found {- return s- }- return strings.TrimSpace(before)-}--// splitQualifiedName splits a dotted type name into (schema, name). Quoted-// identifiers are supported.-// "public.task_status" -> ("public", "task_status", true)-// "myschema.my_domain" -> ("myschema", "my_domain", true)-// `"MySchema"."MyType"` -> ("MySchema", "MyType", true)-// "task_status" -> ("", "", false)-func splitQualifiedName(s string) (schema, name string, ok bool) {- parts, err := splitDottedIdentifier(s)- if err != nil || len(parts) != 2 {- return "", "", false- }- return parts[0], parts[1], true-}--// splitDottedIdentifier splits "a.b.c" into ["a", "b", "c"], handling-// double-quoted identifiers like `"weird name"."another"`.-func splitDottedIdentifier(s string) ([]string, error) {- var parts []string- var cur strings.Builder- inQuote := false- for i := 0; i < len(s); i++ {- c := s[i]- switch {- case c == '"':- inQuote = !inQuote- case c == '.' && !inQuote:- parts = append(parts, cur.String())- cur.Reset()- default:- cur.WriteByte(c)- }- }- if inQuote {- return nil, errors.Errorf("unbalanced quote in %q", s)- }- parts = append(parts, cur.String())- return parts, nil-}--// isBuiltinType reports whether a bare (modifier-stripped) type name is a-// PostgreSQL built-in type. Membership is exact, case-insensitive.-//-// The list is hand-maintained. New entries must come with a golden test in-// query_span_e3_test.go.-func isBuiltinType(s string) bool {- _, ok := pgBuiltinTypes[strings.ToLower(s)]- return ok-}--// pgBuiltinTypes is the allow-list of PostgreSQL built-in type base names-// (without type modifiers, time-zone suffixes, or array brackets).-//-// Sources: PostgreSQL data types reference + the transforms in-// backend/plugin/db/pg/sync.go:820-872 for the specific surface bytebase-// currently emits.-var pgBuiltinTypes = map[string]struct{}{- // Numeric.- "smallint": {},- "integer": {},- "int": {},- "int2": {},- "int4": {},- "int8": {},- "bigint": {},- "decimal": {},- "numeric": {},- "real": {},- "double precision": {},- "float4": {},- "float8": {},- "smallserial": {},- "serial": {},- "bigserial": {},- "money": {},-- // Character / binary.- "character": {},- "character varying": {},- "char": {},- "varchar": {},- "text": {},- "bytea": {},- "bpchar": {},- "name": {},-- // Bit.- "bit": {},- "bit varying": {},- "varbit": {},-- // Date / time.- "date": {},- "time": {},- "timetz": {},- "timestamp": {},- "timestamptz": {},- "interval": {},-- // Boolean and friends.- "boolean": {},- "bool": {},-- // Structured.- "json": {},- "jsonb": {},- "xml": {},- "uuid": {},-- // Geometric.- "point": {},- "line": {},- "lseg": {},- "box": {},- "path": {},- "polygon": {},- "circle": {},-- // Network.- "cidr": {},- "inet": {},- "macaddr": {},- "macaddr8": {},-- // Text search.- "tsvector": {},- "tsquery": {},-- // PG pseudo / system.- "void": {},- "record": {},- "anyelement": {},- "anyarray": {},- "anynonarray": {},- "anyenum": {},- "anyrange": {},- "any": {},- "trigger": {},- "event_trigger": {},- "cstring": {},- "internal": {},- "language_handler": {},- "fdw_handler": {},- "index_am_handler": {},- "tsm_handler": {},- "pg_lsn": {},- "oid": {},- "regclass": {},- "regproc": {},- "regprocedure": {},- "regoper": {},- "regoperator": {},- "regtype": {},- "regconfig": {},- "regdictionary": {},- "regnamespace": {},- "regrole": {},-}diff --git a/backend/plugin/parser/pg/query_span_plpgsql.go b/backend/plugin/parser/pg/query_span_plpgsql.goindex 1ed87bdfe3..34ed565fcf 100644--- a/backend/plugin/parser/pg/query_span_plpgsql.go+++ b/backend/plugin/parser/pg/query_span_plpgsql.go@@ -160,10 +160,6 @@ func (e *omniQuerySpanExtractor) analyzeSQLBody(body string) ([]base.SourceColum}query, err := e.cat.AnalyzeSelectStmt(selStmt)if err != nil {- // Record the classifier verdict for test inspection; fallback- // below always runs.- e.lastFallbackReason = classifyAnalyzeError(err)-// Fall back to parse-tree extraction when analysis fails// (e.g., nested function calls, parameter references).fallbackResults := e.extractFallbackColumns(selStmt)@@ -499,10 +495,6 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedSQL(sql string) []base.SourceColumnSetquery, err := a.extractor.cat.AnalyzeSelectStmt(selStmt)if err != nil {- // Record the classifier verdict for test inspection; fallback below- // always runs.- a.extractor.lastFallbackReason = classifyAnalyzeError(err)-// When full analysis fails (e.g., type mismatches with aggregate functions),// fall back to extracting column references from the parse tree and resolving// them against known tables in the catalog.diff --git a/backend/plugin/parser/pg/query_span_test.go b/backend/plugin/parser/pg/query_span_test.goindex 5285f85042..cdfde7d715 100644--- a/backend/plugin/parser/pg/query_span_test.go+++ b/backend/plugin/parser/pg/query_span_test.go@@ -4,9 +4,14 @@ import ("context""io""os"+ "os/exec"+ "regexp"+ "runtime/debug""testing"metadatapb "github.com/bytebase/omni/metadata"+ "github.com/bytebase/omni/pg/ast"+ "github.com/bytebase/omni/pg/catalog""github.com/pkg/errors""github.com/stretchr/testify/require""gopkg.in/yaml.v3"@@ -104,3 +109,686 @@ func buildMockDatabaseMetadataGetter(databaseMetadata []*metadatapb.DatabaseSchereturn names, nil}}++func TestQuerySpanSurvivesABadQuotedIdentifier(t *testing.T) {+ // A table name carrying a character sequence that a deparse-and-reparse+ // loop chokes on (BYT-9215). Loading the snapshot never renders DDL, so the+ // table installs and unrelated queries succeed either way.+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{+ // A quoted identifier containing an apostrophe — the kind of+ // input BYT-9215 reported failing under Exec(ddl).+ {+ Name: "'weird'table",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ },+ },+ // An unrelated healthy table.+ {+ Name: "accounts",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "email", Type: "text"},+ },+ },+ },+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, email FROM accounts`)+ if len(span.Results) != 2 {+ t.Fatalf("accounts query: got %d results, want 2", len(span.Results))+ }+ for _, r := range span.Results {+ if len(r.SourceColumns) == 0 {+ t.Errorf("result %q has empty sources (lineage lost)", r.Name)+ }+ }+}++func TestGetQuerySpanWithSelectedSchemaFallsBackToPublic(t *testing.T) {+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{+ {+ Name: "app",+ },+ {+ Name: "public",+ Tables: []*metadatapb.TableMetadata{{+ Name: "customer",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "ssn", Type: "text"},+ },+ }},+ },+ },+ }+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})+ span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ }, base.Statement{Text: `SELECT ssn FROM customer`}, "db", "app", false)+ if err != nil {+ t.Fatalf("GetQuerySpan: %v", err)+ }+ if len(span.Results) != 1 {+ t.Fatalf("got %d results, want 1", len(span.Results))+ }+ want := base.ColumnResource{Database: "db", Schema: "public", Table: "customer", Column: "ssn"}+ if _, ok := span.Results[0].SourceColumns[want]; !ok {+ t.Fatalf("result %q missing public fallback source %+v; have %+v", span.Results[0].Name, want, span.Results[0].SourceColumns)+ }+ wantAccess := base.ColumnResource{Database: "db", Schema: "public", Table: "customer"}+ if _, ok := span.SourceColumns[wantAccess]; !ok {+ t.Fatalf("span missing public fallback access source %+v; have %+v", wantAccess, span.SourceColumns)+ }+}++func TestGetQuerySpanSelectedSchemaTakesPrecedenceOverPublic(t *testing.T) {+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{+ {+ Name: "app",+ Tables: []*metadatapb.TableMetadata{{+ Name: "customer",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "email", Type: "text"},+ },+ }},+ },+ {+ Name: "public",+ Tables: []*metadatapb.TableMetadata{{+ Name: "customer",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "ssn", Type: "text"},+ },+ }},+ },+ },+ }+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})+ span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ }, base.Statement{Text: `SELECT email FROM customer`}, "db", "app", false)+ if err != nil {+ t.Fatalf("GetQuerySpan: %v", err)+ }+ if len(span.Results) != 1 {+ t.Fatalf("got %d results, want 1", len(span.Results))+ }+ want := base.ColumnResource{Database: "db", Schema: "app", Table: "customer", Column: "email"}+ if _, ok := span.Results[0].SourceColumns[want]; !ok {+ t.Fatalf("result %q missing selected-schema source %+v; have %+v", span.Results[0].Name, want, span.Results[0].SourceColumns)+ }+}++func TestQuerySpanSurvivesABrokenEnum(t *testing.T) {+ // The table references a user-defined type the snapshot never declares, so+ // it installs as a stand-in with text columns. A query against it still+ // returns the column names the snapshot records.+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{{+ Name: "tasks",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ // References an enum that is NOT declared in metadata —+ // buildCreateStmt will succeed (typeNameFromString works)+ // but DefineRelation fails (enum not in catalog).+ {Name: "status", Type: "public.nonexistent_enum"},+ {Name: "title", Type: "text"},+ },+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, status, title FROM tasks`)+ if len(span.Results) != 3 {+ t.Fatalf("tasks query: got %d results, want 3", len(span.Results))+ }+ wantNames := []string{"id", "status", "title"}+ for i, want := range wantNames {+ if i >= len(span.Results) {+ break+ }+ if span.Results[i].Name != want {+ t.Errorf("result[%d]: got %q, want %q", i, span.Results[i].Name, want)+ }+ if len(span.Results[i].SourceColumns) == 0 {+ t.Errorf("result[%d] (%s): empty sources", i, span.Results[i].Name)+ }+ }+}++func TestQuerySpanKeepsAHealthyNeighborOfABrokenTable(t *testing.T) {+ // A query against a healthy table must succeed even when an unrelated+ // table in the same schema references a broken type. This is the core+ // blast-radius claim: one bad object does not poison all queries.+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{+ {+ Name: "broken",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "bad", Type: "public.nonexistent_type"},+ },+ },+ {+ Name: "healthy",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "label", Type: "text"},+ },+ },+ },+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, label FROM healthy`)+ if len(span.Results) != 2 {+ t.Fatalf("healthy query: got %d results, want 2", len(span.Results))+ }+ for _, r := range span.Results {+ if len(r.SourceColumns) == 0 {+ t.Errorf("result %q lost lineage because of unrelated broken table", r.Name)+ }+ }+}++func TestQuerySpanResolvesAViewOverABrokenTable(t *testing.T) {+ // Chain: enum missing → table T references it, so T stands in → view V over+ // T installs against that stand-in → a query on V still resolves.+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{{+ Name: "records",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "status", Type: "public.nonexistent_enum"},+ {Name: "title", Type: "text"},+ },+ }},+ Views: []*metadatapb.ViewMetadata{{+ Name: "records_view",+ Definition: "SELECT id, title, status FROM records",+ DependencyColumns: []*metadatapb.DependencyColumn{+ {Schema: "public", Table: "records", Column: "id"},+ {Schema: "public", Table: "records", Column: "title"},+ {Schema: "public", Table: "records", Column: "status"},+ },+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, title, status FROM records_view`)+ if len(span.Results) != 3 {+ t.Fatalf("view query: got %d results, want 3", len(span.Results))+ }+ for i, r := range span.Results {+ if len(r.SourceColumns) == 0 {+ t.Errorf("result[%d] (%s) lost lineage", i, r.Name)+ }+ }+}++func TestQuerySpanOnAHealthySchema(t *testing.T) {+ // Baseline: a clean schema produces exact lineage down to+ // (schema, table, column).+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{{+ Name: "users",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "email", Type: "text"},+ {Name: "created_at", Type: "timestamp with time zone"},+ },+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, email FROM users WHERE id > 0`)+ if len(span.Results) != 2 {+ t.Fatalf("got %d results, want 2", len(span.Results))+ }+ mustHaveExactSource(t, span.Results[0], "users", "id")+ mustHaveExactSource(t, span.Results[1], "users", "email")+}++func TestQuerySpanKeepsLineageThroughAPartition(t *testing.T) {+ // Masking resolves a partition to its parent table, which needs the read to+ // trace to the partition's columns.+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{{+ Name: "orders",+ Columns: []*metadatapb.ColumnMetadata{{Name: "id", Type: "int4"}, {Name: "ssn", Type: "text"}},+ Partitions: []*metadatapb.TablePartitionMetadata{{Name: "orders_2024"}},+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT ssn FROM orders_2024`)+ if len(span.Results) != 1 {+ t.Fatalf("got %d results, want 1", len(span.Results))+ }+ mustHaveExactSource(t, span.Results[0], "orders_2024", "ssn")+}++func TestQuerySpanResolvesADeclaredEnum(t *testing.T) {+ // When an enum IS declared in metadata, the table installs as real and+ // enum-typed columns resolve through their real type. This is the+ // positive control for TestQuerySpanSurvivesABrokenEnum.+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ EnumTypes: []*metadatapb.EnumTypeMetadata{{+ Name: "task_status",+ Values: []string{"pending", "running", "done"},+ }},+ Tables: []*metadatapb.TableMetadata{{+ Name: "tasks",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "status", Type: "public.task_status"},+ },+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, status FROM tasks WHERE status = 'pending'`)+ if len(span.Results) != 2 {+ t.Fatalf("got %d results, want 2", len(span.Results))+ }+ mustHaveExactSource(t, span.Results[0], "tasks", "id")+ mustHaveExactSource(t, span.Results[1], "tasks", "status")+}++func TestQuerySpanOnACorrelatedRangeFunctionSubquery(t *testing.T) {+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{+ {+ Name: "compliance_case_record_audits",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "case_id", Type: "text"},+ {Name: "entity_id", Type: "text"},+ {Name: "reason", Type: "text"},+ {Name: "remark", Type: "text"},+ {Name: "case_record", Type: "jsonb"},+ {Name: "reviewer_by", Type: "text"},+ {Name: "deleted_at", Type: "timestamptz"},+ },+ },+ {+ Name: "compliance_cases",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "case_id", Type: "text"},+ {Name: "case_info", Type: "jsonb"},+ {Name: "deleted_at", Type: "timestamptz"},+ },+ },+ },+ }},+ }+ sql := `+select+ a.case_id,+ a.entity_id as uid,+ a.reason,+ a.remark,+ (+ select elem->>'matchedDateTimeValue'+ from jsonb_array_elements(a.case_record::jsonb->'secondaryFieldResults') elem+ where elem->>'typeId' = '******'+ limit 1+ ) as wc_dob,+ (+ select elem->>'matchedValue'+ from jsonb_array_elements(a.case_record::jsonb->'secondaryFieldResults') elem+ where elem->>'typeId' = '******'+ limit 1+ ) as wc_citizenship,+ c.case_info->>'birth_date' as kyc_dob,+ c.case_info->>'nationality' as kyc_citizenship,+ a.reviewer_by as review_by+from compliance_case_record_audits a+inner join compliance_cases c+ on c.case_id = a.case_id and c.deleted_at is null+where a.reviewer_by in ('****** ', '******')+ and a.deleted_at is null;+`+ span := mustGetQuerySpan(t, meta, sql)+ if len(span.Results) != 9 {+ t.Fatalf("got %d results, want 9", len(span.Results))+ }+ mustHaveExactSource(t, span.Results[4], "compliance_case_record_audits", "case_record")+ mustHaveExactSource(t, span.Results[5], "compliance_case_record_audits", "case_record")+ mustHaveExactSource(t, span.Results[6], "compliance_cases", "case_info")+ mustHaveExactSource(t, span.Results[7], "compliance_cases", "case_info")+}++func TestQuerySpanKeepsJSONBLineageThroughCTEsAndALateralJoin(t *testing.T) {+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{{+ Name: "ai_conversation",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "text"},+ {Name: "parts", Type: "jsonb"},+ },+ }},+ }},+ }++ tests := []struct {+ name string+ sql string+ }{+ {+ name: "single_cte",+ sql: `+WITH convs AS (+ SELECT ac.id, ac.parts AS conv_parts+ FROM ai_conversation ac+ LIMIT 1+)+SELECT+ LEFT(+ COALESCE(+ (SELECT string_agg(v::text, ' ')+ FROM jsonb_array_elements_text(+ jsonb_path_query_array(t.part->'bodyData', 'strict $.**.text', '{}'::jsonb, true)+ ) AS v),+ '<no text>'+ ),+ 2000+ ) AS content+FROM convs c+CROSS JOIN LATERAL jsonb_array_elements(c.conv_parts) WITH ORDINALITY AS t(part, part_ord)+WHERE t.part->>'type' IN ('prompt', 'text');+`,+ },+ {+ name: "multiple_ctes",+ sql: `+WITH cte1 AS (+ SELECT ac.id, ac.parts AS conv_parts+ FROM ai_conversation ac+ LIMIT 1+),+cte2 AS (+ SELECT id, conv_parts+ FROM cte1+)+SELECT+ LEFT(+ COALESCE(+ (SELECT string_agg(v::text, ' ')+ FROM jsonb_array_elements_text(+ jsonb_path_query_array(t.part->'bodyData', 'strict $.**.text', '{}'::jsonb, true)+ ) AS v),+ '<no text>'+ ),+ 2000+ ) AS content+FROM cte2 c+CROSS JOIN LATERAL jsonb_array_elements(c.conv_parts) WITH ORDINALITY AS t(part, part_ord)+WHERE t.part->>'type' IN ('prompt', 'text');+`,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ span := mustGetQuerySpan(t, meta, tt.sql)+ if len(span.Results) != 1 {+ t.Fatalf("got %d results, want 1", len(span.Results))+ }+ mustHaveExactSource(t, span.Results[0], "ai_conversation", "parts")+ })+ }+}++func TestAppendQueryDoesNotMutateSharedStack(t *testing.T) {+ outer := &catalog.Query{}+ parent := &catalog.Query{}+ current := &catalog.Query{}+ nested := &catalog.Query{}+ queryStack := []*catalog.Query{outer, parent, current}++ nextStack := appendQuery(queryStack[:2], nested)++ if queryStack[2] != current {+ t.Fatalf("appendQuery mutated caller stack: got %p, want %p", queryStack[2], current)+ }+ if len(nextStack) != 3 || nextStack[2] != nested {+ t.Fatalf("appendQuery returned unexpected stack: %+v", nextStack)+ }+}++func TestWalkExprStopsOnSelfReferentialFunction(t *testing.T) {+ mustNotOverflow(t, func() {+ varExpr := &catalog.VarExpr{RangeIdx: 0, AttNum: 1}+ q := &catalog.Query{+ TargetList: []*catalog.TargetEntry{{+ Expr: varExpr,+ }},+ RangeTable: []*catalog.RangeTableEntry{{+ Kind: catalog.RTEFunction,+ FuncExprs: []catalog.AnalyzedExpr{&catalog.FuncCallExpr{+ Args: []catalog.AnalyzedExpr{&catalog.OpExpr{+ Left: varExpr,+ Right: &catalog.ConstExpr{+ Value: "items",+ },+ }},+ }},+ }},+ }+ extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})+ extractor.cat = catalog.New()+ extractor.walkExpr(q, q.TargetList[0].Expr, make(base.SourceColumnSet))+ })+}++func TestIsUltimatelyPlainColumnStopsOnCyclicSubquery(t *testing.T) {+ mustNotOverflow(t, func() {+ varExpr := &catalog.VarExpr{RangeIdx: 0, AttNum: 1}+ q := &catalog.Query{+ TargetList: []*catalog.TargetEntry{{+ Expr: varExpr,+ }},+ RangeTable: []*catalog.RangeTableEntry{{+ Kind: catalog.RTESubquery,+ }},+ }+ q.RangeTable[0].Subquery = q+ _ = isUltimatelyPlainColumn(q, varExpr)+ })+}++func TestCollectQueryPredicateColumnsStopsOnCyclicQuery(t *testing.T) {+ mustNotOverflow(t, func() {+ q := &catalog.Query{}+ q.CTEList = []*catalog.CommonTableExprQ{{Query: q}}+ extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})+ analyzer := &plpgsqlAnalyzer{+ extractor: extractor,+ scope: newVariableScope(nil),+ }+ analyzer.collectQueryPredicateColumns(q)+ })+}++func TestExtractLineageStopsOnCyclicSetOp(t *testing.T) {+ mustNotOverflow(t, func() {+ q := &catalog.Query{SetOp: catalog.SetOpUnion}+ q.LArg = q+ q.RArg = &catalog.Query{TargetList: []*catalog.TargetEntry{{+ Expr: &catalog.ConstExpr{+ Value: "1",+ },+ }}}+ selStmt := &ast.SelectStmt{}+ selStmt.Larg = selStmt+ selStmt.Rarg = &ast.SelectStmt{}+ extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})+ extractor.extractLineage(q, selStmt)+ })+}++func TestResolveThroughCTEStopsOnCyclicCTE(t *testing.T) {+ mustNotOverflow(t, func() {+ cteSel := &ast.SelectStmt{+ TargetList: &ast.List{Items: []ast.Node{&ast.ResTarget{+ Name: "x",+ Val: &ast.ColumnRef{Fields: &ast.List{Items: []ast.Node{+ &ast.String{Str: "c"},+ &ast.String{Str: "x"},+ }}},+ }}},+ FromClause: &ast.List{Items: []ast.Node{&ast.RangeVar{Relname: "c"}}},+ }+ extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})+ extractor.cat = catalog.New()+ analyzer := &plpgsqlAnalyzer{+ extractor: extractor,+ scope: newVariableScope(nil),+ cteMap: map[string]*ast.SelectStmt{"c": cteSel},+ }+ analyzer.resolveThroughCTE(cteSel, "x", nil, make(base.SourceColumnSet))+ })+}++// ---------- helpers ----------++// testDB / testSchema are the fixed database and schema names these tests use;+// as constants they keep the call sites terse and satisfy the unparam linter.+const (+ testDB = "db"+ testSchema = "public"+)++// noOverflowEnv marks the subprocess mustNotOverflow spawns.+const noOverflowEnv = "BYTEBASE_TEST_NO_OVERFLOW"++// mustNotOverflow runs fn in a subprocess with a 1 MB stack, so recursion that+// does not terminate fails this test instead of taking the whole run down. The+// subprocess re-runs this same test, which then takes the fn branch.+func mustNotOverflow(t *testing.T, fn func()) {+ t.Helper()+ if os.Getenv(noOverflowEnv) == "1" {+ debug.SetMaxStack(1 << 20)+ fn()+ return+ }+ cmd := exec.Command(os.Args[0], "-test.run=^"+regexp.QuoteMeta(t.Name())+"$")+ cmd.Env = append(os.Environ(), noOverflowEnv+"=1")+ if output, err := cmd.CombinedOutput(); err != nil {+ t.Fatalf("recursion did not terminate: %v\n%s", err, output)+ }+}++func mustGetQuerySpan(t *testing.T, meta *metadatapb.DatabaseSchemaMetadata, sql string) *base.QuerySpan {+ t.Helper()+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})+ span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ }, base.Statement{Text: sql}, testDB, "", false)+ if err != nil {+ t.Fatalf("GetQuerySpan(%q): %v", sql, err)+ }+ if span == nil {+ t.Fatalf("GetQuerySpan(%q): nil span", sql)+ }+ return span+}++func mustHaveExactSource(t *testing.T, result base.QuerySpanResult, table, column string) {+ t.Helper()+ want := base.ColumnResource{+ Database: testDB,+ Schema: testSchema,+ Table: table,+ Column: column,+ }+ for src := range result.SourceColumns {+ if src == want {+ return+ }+ }+ t.Errorf("result %q missing source %+v; have %+v", result.Name, want, result.SourceColumns)+}++// TestRangeVarFallbackSkipsCompositeTypes proves the query-span fallback does+// not expand a standalone composite type as a FROM source — PostgreSQL+// rejects composite types as table sources.+func TestRangeVarFallbackSkipsCompositeTypes(t *testing.T) {+ meta := &metadatapb.DatabaseSchemaMetadata{+ Schemas: []*metadatapb.SchemaMetadata{+ {+ Name: "public",+ CompositeTypes: []*metadatapb.CompositeTypeMetadata{+ {+ Name: "addr",+ Attributes: []*metadatapb.CompositeTypeAttribute{+ {Name: "street", Type: "text"},+ },+ },+ },+ Tables: []*metadatapb.TableMetadata{+ {+ Name: "t",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "integer"},+ },+ },+ },+ },+ },+ }++ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ if _, err := cat.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{}); err != nil {+ t.Fatalf("LoadMetadata: %v", err)+ }++ e := &omniQuerySpanExtractor{+ cat: cat,+ searchPath: []string{"public"},+ defaultDatabase: "db",+ }+ if results := e.extractColumnsFromRangeVar(&ast.RangeVar{Schemaname: "public", Relname: "addr"}); results != nil {+ t.Errorf("composite type must not expand as a FROM source, got %d columns", len(results))+ }+ if results := e.extractColumnsFromRangeVar(&ast.RangeVar{Schemaname: "public", Relname: "t"}); len(results) != 1 {+ t.Errorf("real table must still expand, got %d columns", len(results))+ }+}diff --git a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go b/backend/plugin/parser/pg/query_span_unresolved_columns_test.goindex 2987300059..28adc2e193 100644--- a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go+++ b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go@@ -42,15 +42,11 @@ func healthySchema() *metadatapb.DatabaseSchemaMetadata {// A table with no columns models a snapshot from a sync before #20581.func degradedSchema() *metadatapb.DatabaseSchemaMetadata {- return degradedSchemaNamed("t")-}--func degradedSchemaNamed(table string) *metadatapb.DatabaseSchemaMetadata {return &metadatapb.DatabaseSchemaMetadata{Name: "db",Schemas: []*metadatapb.SchemaMetadata{{Name: "public",- Tables: []*metadatapb.TableMetadata{{Name: table}},+ Tables: []*metadatapb.TableMetadata{{Name: "t"}},}},}}@@ -100,6 +96,8 @@ func TestUnresolvedColumnsSignalFiresOnDegradedSnapshot(t *testing.T) {"WITH t AS (SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t)) AS c FROM (SELECT 1) x) SELECT * FROM t","WITH t AS (SELECT string_agg('x', ',' ORDER BY (SELECT count(*) FROM t)) AS c FROM (SELECT 1) x) SELECT * FROM t",`WITH "T" AS (SELECT 1 AS n) SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t)) FROM "T"`,+ "SELECT * FROM generate_series(1, (SELECT count(*)::int FROM public.t))",+ "SELECT * FROM (VALUES ((SELECT count(*) FROM public.t))) v(x)",}for _, statement := range statements {t.Run(statement, func(t *testing.T) {@@ -243,20 +241,3 @@ func TestUnresolvedColumnsSignalResolvesRelationsNotRoutines(t *testing.T) {require.Nil(t, span.UnresolvedColumnsError,"the sequence in a is the relation this query reads, and a sequence carries no column list to judge")}--// BYT-10076 tracks reads omitted from the access set; these cases record that gap.-func TestUnresolvedColumnsSignalNotCoveredShapes(t *testing.T) {- notCovered := map[string]string{- "subquery in a FROM-clause function argument": "SELECT * FROM generate_series(1, (SELECT count(*)::int FROM public.d))",- "subquery inside a VALUES list": "SELECT * FROM (VALUES ((SELECT count(*) FROM public.d))) v(x)",- }- for name, statement := range notCovered {- t.Run(name, func(t *testing.T) {- span := spanFor(t, statement, degradedSchemaNamed("d"))- require.Empty(t, span.SourceColumns,- "the premise of this gap is that the access set is empty; if this fails the gap may have been closed upstream")- require.Nil(t, span.UnresolvedColumnsError,- "documented gap: no access reported, so nothing to check (BYT-10076)")- })- }-}diff --git a/backend/plugin/parser/redshift/completion.go b/backend/plugin/parser/redshift/completion.goindex 1c7051caee..5eaa935c68 100644--- a/backend/plugin/parser/redshift/completion.go+++ b/backend/plugin/parser/redshift/completion.go@@ -2,13 +2,11 @@ package redshiftimport ("context"- "fmt""slices""strings""unicode""unicode/utf8"- metadatapb "github.com/bytebase/omni/metadata"redshiftcatalog "github.com/bytebase/omni/redshift/catalog"redshiftcompletion "github.com/bytebase/omni/redshift/completion"@@ -21,9 +19,9 @@ func init() {}// Completion returns Redshift auto-completion candidates backed by omni's-// parser-native completer. Bytebase metadata is copied into an omni Redshift-// catalog with minimal DDL so the completer can resolve schemas, relations,-// columns, sequences, and Redshift-specific grammar slots.+// parser-native completer. Bytebase metadata is loaded into an omni Redshift+// catalog so the completer can resolve schemas, relations, columns, sequences,+// and Redshift-specific grammar slots.func Completion(ctx context.Context, cCtx base.CompletionContext, statement string, caretLine int, caretOffset int) ([]base.Candidate, error) {cat := buildCompletionCatalog(ctx, cCtx)@@ -102,48 +100,7 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext) *r}cat := redshiftcatalog.New()- for _, schemaName := range metadata.ListSchemaNames() {- schemaMeta := metadata.GetSchemaMetadata(schemaName)- if schemaMeta == nil {- continue- }- execCompletionDDL(cat, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s;", quoteIdent(schemaName)))-- for _, tableName := range schemaMeta.ListTableNames() {- tableMeta := schemaMeta.GetTable(tableName)- if tableMeta == nil {- continue- }- execCompletionDDL(cat, createTableDDL(schemaName, tableName, tableMeta.GetProto().GetColumns()))- }- for _, tableName := range schemaMeta.ListForeignTableNames() {- tableMeta := schemaMeta.GetExternalTable(tableName)- if tableMeta == nil {- continue- }- execCompletionDDL(cat, createTableDDL(schemaName, tableName, tableMeta.GetProto().GetColumns()))- }- for _, viewName := range schemaMeta.ListViewNames() {- viewMeta := schemaMeta.GetView(viewName)- if viewMeta == nil {- continue- }- execCompletionDDL(cat, createViewDDL("VIEW", schemaName, viewName, viewMeta.GetColumns()))- }- for _, viewName := range schemaMeta.ListMaterializedViewNames() {- viewMeta := schemaMeta.GetMaterializedView(viewName)- if viewMeta == nil {- continue- }- cat.SetSearchPath([]string{schemaName, "public"})- if viewMeta.GetDefinition() == "" || !execCompletionDDL(cat, createMaterializedViewDDL(schemaName, viewName, viewMeta.GetDefinition())) {- execCompletionDDL(cat, createViewDDL("MATERIALIZED VIEW", schemaName, viewName, nil))- }- }- for _, sequenceName := range schemaMeta.ListSequenceNames() {- execCompletionDDL(cat, fmt.Sprintf("CREATE SEQUENCE %s.%s;", quoteIdent(schemaName), quoteIdent(sequenceName)))- }- }+ cat.LoadMetadata(metadata.GetProto())searchPath := []string{"public"}if cCtx.DefaultSchema != "" {@@ -153,100 +110,6 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext) *rreturn cat}-func execCompletionDDL(cat *redshiftcatalog.Catalog, sql string) bool {- _, err := cat.Exec(sql, nil)- return err == nil-}--func createTableDDL(schemaName, tableName string, columns []*metadatapb.ColumnMetadata) string {- return fmt.Sprintf(- "CREATE TABLE %s.%s (%s);",- quoteIdent(schemaName),- quoteIdent(tableName),- columnListDDL(columns),- )-}--func createViewDDL(kind, schemaName, viewName string, columns []*metadatapb.ColumnMetadata) string {- selectItems := make([]string, 0, len(columns))- for _, column := range columns {- if column == nil || column.GetName() == "" {- continue- }- selectItems = append(selectItems, fmt.Sprintf(- "CAST(NULL AS %s) AS %s",- normalizeCompletionType(column.GetType()),- quoteIdent(column.GetName()),- ))- }- if len(selectItems) == 0 {- selectItems = append(selectItems, "1 AS __bytebase_completion_placeholder")- }- return fmt.Sprintf(- "CREATE %s %s.%s AS SELECT %s;",- kind,- quoteIdent(schemaName),- quoteIdent(viewName),- strings.Join(selectItems, ", "),- )-}--func createMaterializedViewDDL(schemaName, viewName, definition string) string {- definition = strings.TrimSpace(definition)- definition = strings.TrimSuffix(definition, ";")- if strings.HasPrefix(strings.ToUpper(definition), "CREATE ") {- return definition + ";"- }- return fmt.Sprintf(- "CREATE MATERIALIZED VIEW %s.%s AS %s;",- quoteIdent(schemaName),- quoteIdent(viewName),- definition,- )-}--func columnListDDL(columns []*metadatapb.ColumnMetadata) string {- columnDefs := make([]string, 0, len(columns))- for _, column := range columns {- if column == nil || column.GetName() == "" {- continue- }- columnDefs = append(columnDefs, fmt.Sprintf("%s %s", quoteIdent(column.GetName()), normalizeCompletionType(column.GetType())))- }- if len(columnDefs) == 0 {- columnDefs = append(columnDefs, "__bytebase_completion_placeholder text")- }- return strings.Join(columnDefs, ", ")-}--func normalizeCompletionType(typ string) string {- lower := strings.ToLower(strings.TrimSpace(typ))- switch {- case lower == "":- return "text"- case strings.Contains(lower, "bigint"):- return "bigint"- case strings.Contains(lower, "int"):- return "integer"- case strings.Contains(lower, "bool"):- return "boolean"- case strings.Contains(lower, "char"), strings.Contains(lower, "text"), strings.Contains(lower, "string"):- return "text"- case strings.Contains(lower, "numeric"), strings.Contains(lower, "decimal"):- return "numeric"- case strings.Contains(lower, "double"):- return "double precision"- case strings.Contains(lower, "float"), strings.Contains(lower, "real"):- return "real"- case strings.Contains(lower, "date"):- return "date"- case strings.Contains(lower, "time"):- return "timestamp"- default:- return "text"- }-}-func quoteIdent(name string) string {return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`}diff --git a/backend/plugin/parser/redshift/query_span_extractor.go b/backend/plugin/parser/redshift/query_span_extractor.goindex 5b157e6a21..d4b5c4bdcb 100644--- a/backend/plugin/parser/redshift/query_span_extractor.go+++ b/backend/plugin/parser/redshift/query_span_extractor.go@@ -2,7 +2,6 @@ package redshiftimport ("context"- "slices""strings"metadatapb "github.com/bytebase/omni/metadata"@@ -12,7 +11,6 @@ import ("github.com/pkg/errors""github.com/bytebase/bytebase/backend/plugin/parser/base"- "github.com/bytebase/bytebase/backend/store/model")const redshiftGetDatabaseMetadataError = "failed to get database metadata for database: %s"@@ -87,9 +85,6 @@ func (q *omniQuerySpanExtractor) getOmniQuerySpanWithFunctionStack(ctx context.Cif err != nil {return nil, err}- if err := q.installOmniQuerySpanFunctions(ctx, cat); err != nil {- return nil, err- }omniSpan, err := redshiftanalysis.GetQuerySpan(cat, statement)if err != nil {if isOmniQuerySpanNotFound(err) {@@ -211,37 +206,10 @@ func (q *omniQuerySpanExtractor) newOmniQuerySpanCatalog(ctx context.Context) (*}cat := redshiftcatalog.New()cat.SetSearchPath(q.searchPath)- cat.SetRelationResolver(&redshiftQuerySpanRelationResolver{- metadata: metadata,- })+ cat.UseMetadata(metadata.GetProto())return cat, nil}-func (q *omniQuerySpanExtractor) installOmniQuerySpanFunctions(ctx context.Context, cat *redshiftcatalog.Catalog) error {- if q.gCtx.GetDatabaseMetadataFunc == nil {- return nil- }- _, metadata, err := q.gCtx.GetDatabaseMetadataFunc(ctx, q.gCtx.InstanceID, q.defaultDatabase)- if err != nil {- return errors.Wrapf(err, redshiftGetDatabaseMetadataError, q.defaultDatabase)- }- if metadata == nil {- return nil- }- for _, schema := range metadata.GetProto().GetSchemas() {- for _, function := range schema.GetFunctions() {- definition := strings.TrimSpace(function.GetDefinition())- if definition == "" {- continue- }- if _, err := cat.Exec(definition, nil); err != nil {- continue- }- }- }- return nil-}-type omniFunctionBodySources struct {accessTables base.SourceColumnSetresults []base.QuerySpanResult@@ -269,256 +237,6 @@ type omniFunctionSourceProjection struct {type omniFunctionSourceScope map[string]omniFunctionSourceProjection-type redshiftQuerySpanRelationResolver struct {- metadata *model.DatabaseMetadata-}--func (r *redshiftQuerySpanRelationResolver) ResolveRelation(schemaName, relationName string, searchPath []string) (*redshiftcatalog.RelationSpec, error) {- if r.metadata == nil {- return nil, nil- }-- resolvedSchemaName, resolvedRelationName := schemaName, relationName- if resolvedSchemaName == "" {- resolvedSchemaName, resolvedRelationName = r.metadata.SearchRelation(searchPath, relationName)- if resolvedSchemaName == "" && resolvedRelationName == "" {- return nil, nil- }- }-- schema := r.metadata.GetSchemaMetadata(resolvedSchemaName)- if schema == nil {- return nil, nil- }- return redshiftRelationSpecFromMetadata(r.metadata, schema, resolvedSchemaName, resolvedRelationName), nil-}--func redshiftRelationSpecFromMetadata(metadata *model.DatabaseMetadata, schema *model.SchemaMetadata, schemaName, relationName string) *redshiftcatalog.RelationSpec {- if table := schema.GetTable(relationName); table != nil {- return redshiftTableRelationSpec(schemaName, table.GetProto().GetName(), table.GetProto().GetColumns())- }- if table := schema.GetExternalTable(relationName); table != nil {- return redshiftTableRelationSpec(schemaName, table.GetProto().GetName(), table.GetProto().GetColumns())- }- if view := schema.GetView(relationName); view != nil {- if definition := strings.TrimSpace(view.GetDefinition()); definition != "" {- return &redshiftcatalog.RelationSpec{- SchemaName: schemaName,- Name: view.GetName(),- Kind: redshiftcatalog.RelationKindView,- Definition: qualifyRedshiftViewDefinition(definition, schemaName, metadata),- }- }- return redshiftTableRelationSpec(schemaName, view.GetName(), view.GetColumns())- }- if view := schema.GetMaterializedView(relationName); view != nil {- if definition := strings.TrimSpace(view.GetDefinition()); definition != "" {- return &redshiftcatalog.RelationSpec{- SchemaName: schemaName,- Name: view.GetName(),- Kind: redshiftcatalog.RelationKindMaterializedView,- Definition: qualifyRedshiftViewDefinition(definition, schemaName, metadata),- }- }- return redshiftTableRelationSpec(schemaName, view.GetName(), nil)- }- if sequence := schema.GetSequence(relationName); sequence != nil {- return redshiftSequenceRelationSpec(schemaName, sequence.GetName())- }- return nil-}--func redshiftTableRelationSpec(schemaName, tableName string, columns []*metadatapb.ColumnMetadata) *redshiftcatalog.RelationSpec {- return &redshiftcatalog.RelationSpec{- SchemaName: schemaName,- Name: tableName,- Kind: redshiftcatalog.RelationKindTable,- Columns: redshiftRelationColumnSpecs(columns),- }-}--func redshiftRelationColumnSpecs(columns []*metadatapb.ColumnMetadata) []redshiftcatalog.RelationColumnSpec {- result := make([]redshiftcatalog.RelationColumnSpec, 0, len(columns))- for _, column := range columns {- if column == nil || column.GetName() == "" {- continue- }- result = append(result, redshiftcatalog.RelationColumnSpec{- Name: column.GetName(),- Type: normalizeCompletionType(column.GetType()),- })- }- if len(result) == 0 {- result = append(result, redshiftcatalog.RelationColumnSpec{- Name: "__bytebase_query_span_placeholder",- Type: "text",- })- }- return result-}--func redshiftSequenceRelationSpec(schemaName, sequenceName string) *redshiftcatalog.RelationSpec {- return &redshiftcatalog.RelationSpec{- SchemaName: schemaName,- Name: sequenceName,- Kind: redshiftcatalog.RelationKindTable,- Columns: []redshiftcatalog.RelationColumnSpec{- {Name: "last_value", Type: "bigint"},- {Name: "log_cnt", Type: "bigint"},- {Name: "is_called", Type: "boolean"},- },- }-}--type redshiftSchemaQualificationEdit struct {- offset int- schema string-}--func qualifyRedshiftViewDefinition(definition, schemaName string, metadata *model.DatabaseMetadata) string {- stmts, err := ParseRedshift(definition)- if err != nil || len(stmts) != 1 {- return definition- }-- var query redshiftast.Node- switch stmt := stmts[0].AST.(type) {- case *redshiftast.ViewStmt:- query = stmt.Query- case *redshiftast.CreateTableAsStmt:- query = stmt.Query- case *redshiftast.SelectStmt:- query = stmt- default:- return definition- }- if query == nil {- return definition- }-- searchPath := normalizeOmniSearchPath([]string{schemaName, "public"})- var edits []redshiftSchemaQualificationEdit- collectRedshiftSchemaQualificationEdits(metadata, searchPath, query, map[string]bool{}, &edits)- if len(edits) == 0 {- return definition- }-- slices.SortFunc(edits, func(a, b redshiftSchemaQualificationEdit) int {- return b.offset - a.offset- })- result := definition- for _, edit := range edits {- if edit.offset < 0 || edit.offset > len(result) {- continue- }- result = result[:edit.offset] + quoteIdent(edit.schema) + "." + result[edit.offset:]- }- return result-}--func collectRedshiftSchemaQualificationEdits(metadata *model.DatabaseMetadata, searchPath []string, node redshiftast.Node, ctes map[string]bool, edits *[]redshiftSchemaQualificationEdit) {- if node == nil {- return- }- if selectStmt, ok := node.(*redshiftast.SelectStmt); ok {- collectRedshiftSchemaQualificationEditsFromSelect(metadata, searchPath, selectStmt, ctes, edits)- return- }-- redshiftast.Inspect(node, func(n redshiftast.Node) bool {- if n != node {- if selectStmt, ok := n.(*redshiftast.SelectStmt); ok {- collectRedshiftSchemaQualificationEditsFromSelect(metadata, searchPath, selectStmt, ctes, edits)- return false- }- }- rangeVar, ok := n.(*redshiftast.RangeVar)- if !ok || rangeVar == nil || rangeVar.Relname == "" || rangeVar.Schemaname != "" || rangeVar.Catalogname != "" {- return true- }- if isOmniCTEReference(rangeVar, ctes) || rangeVar.Loc.Start < 0 {- return true- }- schemaName, _ := metadata.SearchRelation(searchPath, rangeVar.Relname)- if schemaName == "" {- return true- }- *edits = append(*edits, redshiftSchemaQualificationEdit{- offset: rangeVar.Loc.Start,- schema: schemaName,- })- return true- })-}--func collectRedshiftSchemaQualificationEditsFromSelect(metadata *model.DatabaseMetadata, searchPath []string, selectStmt *redshiftast.SelectStmt, ctes map[string]bool, edits *[]redshiftSchemaQualificationEdit) {- localCTEs := cloneOmniCTENameSet(ctes)- if selectStmt.WithClause != nil && selectStmt.WithClause.Ctes != nil {- cteScope := cloneOmniCTENameSet(ctes)- if selectStmt.WithClause.Recursive {- for _, item := range selectStmt.WithClause.Ctes.Items {- cte, ok := item.(*redshiftast.CommonTableExpr)- if ok && cte.Ctename != "" {- cteScope[strings.ToLower(cte.Ctename)] = true- }- }- }- for _, item := range selectStmt.WithClause.Ctes.Items {- cte, ok := item.(*redshiftast.CommonTableExpr)- if !ok || cte.Ctename == "" {- continue- }- collectRedshiftSchemaQualificationEdits(metadata, searchPath, cte.Ctequery, cteScope, edits)- name := strings.ToLower(cte.Ctename)- cteScope[name] = true- localCTEs[name] = true- }- }-- for _, node := range []redshiftast.Node{- selectStmt.WhereClause,- selectStmt.HavingClause,- selectStmt.QualifyClause,- selectStmt.LimitOffset,- selectStmt.LimitCount,- } {- if node != nil {- collectRedshiftSchemaQualificationEdits(metadata, searchPath, node, localCTEs, edits)- }- }- if selectStmt.IntoClause != nil {- collectRedshiftSchemaQualificationEdits(metadata, searchPath, selectStmt.IntoClause, localCTEs, edits)- }- if selectStmt.Larg != nil {- collectRedshiftSchemaQualificationEdits(metadata, searchPath, selectStmt.Larg, localCTEs, edits)- }- if selectStmt.Rarg != nil {- collectRedshiftSchemaQualificationEdits(metadata, searchPath, selectStmt.Rarg, localCTEs, edits)- }- for _, list := range []*redshiftast.List{- selectStmt.DistinctClause,- selectStmt.TargetList,- selectStmt.FromClause,- selectStmt.GroupClause,- selectStmt.WindowClause,- selectStmt.ExcludeList,- selectStmt.ValuesLists,- selectStmt.SortClause,- selectStmt.LockingClause,- } {- collectRedshiftSchemaQualificationEditsFromList(metadata, searchPath, list, localCTEs, edits)- }-}--func collectRedshiftSchemaQualificationEditsFromList(metadata *model.DatabaseMetadata, searchPath []string, list *redshiftast.List, ctes map[string]bool, edits *[]redshiftSchemaQualificationEdit) {- if list == nil {- return- }- for _, item := range list.Items {- collectRedshiftSchemaQualificationEdits(metadata, searchPath, item, ctes, edits)- }-}-func (q *omniQuerySpanExtractor) collectOmniAccessTables(ctx context.Context, node redshiftast.Node) (base.SourceColumnSet, error) {result := make(base.SourceColumnSet)if err := q.collectOmniAccessTablesFromNode(ctx, node, map[string]bool{}, result); err != nil {diff --git a/backend/plugin/parser/redshift/query_span_test.go b/backend/plugin/parser/redshift/query_span_test.goindex 37e3e56fe5..c24f0cd265 100644--- a/backend/plugin/parser/redshift/query_span_test.go+++ b/backend/plugin/parser/redshift/query_span_test.go@@ -108,15 +108,12 @@ func TestRedshiftQuerySpanEntrypointDoesNotDependOnANTLR(t *testing.T) {}}-func TestRedshiftOmniQuerySpanUsesLazyRelationResolver(t *testing.T) {+// Query span must resolve relations lazily through UseMetadata rather than+// installing the whole snapshot up front.+func TestRedshiftOmniQuerySpanUsesUseMetadata(t *testing.T) {content, err := os.ReadFile("query_span_extractor.go")require.NoError(t, err)- source := string(content)- require.Contains(t, source, "SetRelationResolver")- require.NotContains(t, source, "pendingOmniCatalogView")- require.NotContains(t, source, "buildOmniQuerySpanCatalog")- require.NotContains(t, source, "orderPendingOmniCatalogViews")- require.NotContains(t, source, "createQuerySpanViewDDL")+ require.Contains(t, string(content), "UseMetadata")}func TestRedshiftOmniQuerySpanNonSelectTypes(t *testing.T) {diff --git a/backend/plugin/parser/tidb/completion.go b/backend/plugin/parser/tidb/completion.goindex 533368873f..38b6c070c0 100644--- a/backend/plugin/parser/tidb/completion.go+++ b/backend/plugin/parser/tidb/completion.go@@ -2,13 +2,11 @@ package tidbimport ("context"- "fmt""slices""strings""unicode""unicode/utf8"- metadatapb "github.com/bytebase/omni/metadata""github.com/bytebase/omni/tidb/catalog"omnicompletion "github.com/bytebase/omni/tidb/completion"tidbparser "github.com/bytebase/omni/tidb/parser"@@ -17,11 +15,6 @@ import ("github.com/bytebase/bytebase/backend/plugin/parser/base")-// genericColumnType is a syntactically valid fallback type used when a column's-// real type is empty or fails to parse, so the column name still surfaces as a-// completion candidate.-const genericColumnType = "int"-// writeStatementKeywords are statement-initiating keywords for write, DDL,// transaction-control, and admin statements. They are dropped from completion// in the read-only query scene, leaving only read statements (SELECT, WITH,@@ -112,13 +105,11 @@ func Completion(ctx context.Context, cCtx base.CompletionContext, statement strireturn result, nil}-// buildCatalog constructs an omni TiDB catalog from Bytebase metadata by-// replaying minimal DDL. It fully loads the default database plus any database-// referenced as a qualifier in the statement (so cross-database qualified-// completion like `other_db.tbl.col` resolves), and registers every other known-// database name so database-name candidates still surface. Every identifier is-// backticked so reserved words and special characters parse; tables are created-// one at a time so a single unparseable column type cannot empty the catalog.+// buildCatalog constructs an omni TiDB catalog from Bytebase metadata. It+// fully loads the default database plus any database referenced as a qualifier+// in the statement (so cross-database qualified completion like+// `other_db.tbl.col` resolves), and registers every other known database name+// so database-name candidates still surface.func buildCatalog(ctx context.Context, cCtx base.CompletionContext, statement string) *catalog.Catalog {cat := catalog.New()allNames := listAllDatabaseNames(ctx, cCtx)@@ -166,33 +157,15 @@ func buildCatalog(ctx context.Context, cCtx base.CompletionContext, statement streturn cat}-// loadDatabaseObjects fully loads one database's tables and views into the-// catalog, under that database's namespace.+// loadDatabaseObjects loads one database's tables and views into the catalog.func loadDatabaseObjects(ctx context.Context, cCtx base.CompletionContext, cat *catalog.Catalog, dbName string) {- db := backtickIdentifier(dbName)_, dbMeta, err := cCtx.Metadata(ctx, cCtx.InstanceID, dbName)if err != nil || dbMeta == nil {// Still register the name so it can be a database candidate.- _, _ = cat.Exec("CREATE DATABASE "+db+";", &catalog.ExecOptions{ContinueOnError: true})- return- }- if _, err := cat.Exec("CREATE DATABASE "+db+"; USE "+db+";", &catalog.ExecOptions{ContinueOnError: true}); err != nil {- return- }- schema := dbMeta.GetSchemaMetadata("")- if schema == nil {+ _, _ = cat.Exec("CREATE DATABASE "+backtickIdentifier(dbName)+";", &catalog.ExecOptions{ContinueOnError: true})return}- for _, tableName := range schema.ListTableNames() {- if table := schema.GetTable(tableName); table != nil {- defineTable(cat, tableName, table.GetProto().GetColumns())- }- }- for _, viewName := range schema.ListViewNames() {- if view := schema.GetView(viewName); view != nil {- defineView(cat, viewName, view.GetDefinition())- }- }+ _, _ = cat.LoadMetadata(ctx, dbMeta.GetProto())}// listAllDatabaseNames returns the instance's database names, or nil if the@@ -252,76 +225,6 @@ func isIdentByte(b byte) bool {(b >= '0' && b <= '9')}-// defineTable installs a table in the catalog, retrying with generic column-// types if the real types fail to parse so that column names still surface.-func defineTable(cat *catalog.Catalog, name string, columns []*metadatapb.ColumnMetadata) {- if len(columns) == 0 {- return- }- if execTableDDL(cat, name, columns, false) {- return- }- execTableDDL(cat, name, columns, true)-}--func execTableDDL(cat *catalog.Catalog, name string, columns []*metadatapb.ColumnMetadata, generic bool) bool {- defs := make([]string, 0, len(columns))- for _, col := range columns {- colType := col.GetType()- if generic || colType == "" {- colType = genericColumnType- }- defs = append(defs, backtickIdentifier(col.GetName())+" "+colType)- }- ddl := fmt.Sprintf("CREATE TABLE %s (%s);", backtickIdentifier(name), strings.Join(defs, ", "))- return execOK(cat, ddl)-}--// defineView installs a view in the catalog. Unlike tables, the only structured-// input is the view's definition SQL, whose exact shape varies (full-// "CREATE VIEW ..." vs. a bare SELECT). We try the definition as-is, then-// wrapped in CREATE VIEW, and finally fall back to a trivial view so the view-// name still surfaces as a candidate (matching mysql, which reads view names-// straight from metadata) even when the definition cannot be parsed.-func defineView(cat *catalog.Catalog, name, definition string) {- var attempts []string- if definition != "" {- attempts = append(attempts,- definition,- fmt.Sprintf("CREATE VIEW %s AS %s", backtickIdentifier(name), definition),- )- }- attempts = append(attempts, fmt.Sprintf("CREATE VIEW %s AS SELECT 1", backtickIdentifier(name)))- for _, ddl := range attempts {- if execOK(cat, ddl) {- return- }- }-}--// execOK runs DDL against the catalog and reports whether it actually installed-// a schema object: every statement parsed without error AND at least one was a-// non-DML (utility) statement that mutated the catalog. A definition that-// reduces to only skipped DML — e.g. a bare SELECT view body, which TiDB sync-// stores in information_schema.VIEWS.VIEW_DEFINITION — is NOT a success, so the-// caller falls through to the wrapped CREATE VIEW form.-func execOK(cat *catalog.Catalog, ddl string) bool {- results, err := cat.Exec(ddl, &catalog.ExecOptions{ContinueOnError: true})- if err != nil {- return false- }- applied := false- for _, r := range results {- if r.Error != nil {- return false- }- if !r.Skipped {- applied = true- }- }- return applied-}-// backtickIdentifier quotes an identifier for TiDB DDL, escaping embedded// backticks by doubling them.func backtickIdentifier(name string) string {diff --git a/backend/plugin/parser/tidb/completion_test.go b/backend/plugin/parser/tidb/completion_test.goindex 5f4335e041..f075ea67c7 100644--- a/backend/plugin/parser/tidb/completion_test.go+++ b/backend/plugin/parser/tidb/completion_test.go@@ -128,10 +128,10 @@ func TestCompletion_ReservedWordIdentifiersSurface(t *testing.T) {}// A column whose stored type cannot be parsed must not silently vanish:-// buildCatalog retries the whole table with a generic column type, so every+// LoadMetadata stands the table in with text columns, so every// column name still surfaces. Without that retry the failing CREATE TABLE would// drop the entire table from the catalog.-func TestCompletion_UnparseableColumnTypeFallsBackToGeneric(t *testing.T) {+func TestCompletion_UnparseableColumnTypeFallsBackToStandIn(t *testing.T) {meta := metadataFunc(&metadatapb.DatabaseSchemaMetadata{Name: "db",Schemas: []*metadatapb.SchemaMetadata{@@ -140,7 +140,7 @@ func TestCompletion_UnparseableColumnTypeFallsBackToGeneric(t *testing.T) {Tables: []*metadatapb.TableMetadata{{Name: "t", Columns: []*metadatapb.ColumnMetadata{{Name: "good", Type: "int"},- {Name: "weird", Type: ")"}, // unparseable type — forces the generic retry+ {Name: "weird", Type: ")"}, // unparseable type — forces the stand-in}},},},@@ -154,7 +154,7 @@ func TestCompletion_UnparseableColumnTypeFallsBackToGeneric(t *testing.T) {require.True(t, hasCandidate(got, base.CandidateTypeColumn, "good"),"column 'good' should surface; got %v", got)require.True(t, hasCandidate(got, base.CandidateTypeColumn, "weird"),- "column 'weird' with an unparseable type should still surface via the generic-type retry; got %v", got)+ "column 'weird' with an unparseable type should still surface via the stand-in; got %v", got)}// In the read-only query scene, write statements (DML/DDL incl. TiDB BATCH)diff --git a/backend/plugin/parser/trino/completion.go b/backend/plugin/parser/trino/completion.goindex 116afedd49..c843435bbc 100644--- a/backend/plugin/parser/trino/completion.go+++ b/backend/plugin/parser/trino/completion.go@@ -11,7 +11,6 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"- "github.com/bytebase/bytebase/backend/store/model")func init() {@@ -90,7 +89,7 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext, stif err != nil || meta == nil {continue}- loadCatalogMetadata(cat, norm, meta)+ cat.LoadMetadata(meta.GetProto())}if cCtx.DefaultDatabase != "" {@@ -102,36 +101,14 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext, streturn cat}-// loadCatalogMetadata copies one database's schemas, tables and views (with-// their column lists and view definitions) into the omni catalog under the-// given normalized catalog name, returning the view definitions it loaded.-// Views carry their defining query so omni's analysis can resolve lineage-// through them (GetQuerySpanWithCatalog); an empty definition leaves the view-// opaque. Shared by the completion and query-span catalog builders.-func loadCatalogMetadata(cat *catalog.Catalog, norm string, meta *model.DatabaseMetadata) []string {+// viewDefinitions returns the defining query of every view in the snapshot.+// Query span feeds them back as a corpus so it can tell which catalogs a view+// reaches into; an empty definition leaves the view opaque.+func viewDefinitions(meta *metadatapb.DatabaseSchemaMetadata) []string {var definitions []string- database := cat.EnsureCatalog(norm)- for _, schemaName := range meta.ListSchemaNames() {- schemaMeta := meta.GetSchemaMetadata(schemaName)- if schemaMeta == nil {- continue- }- sc := database.EnsureSchema(catalog.Normalize(schemaName))- for _, tableName := range schemaMeta.ListTableNames() {- tableMeta := schemaMeta.GetTable(tableName)- if tableMeta == nil {- continue- }- sc.AddTable(catalog.Normalize(tableName), columnsOf(tableMeta.GetProto().GetColumns())...)- }- for _, viewName := range schemaMeta.ListViewNames() {- viewMeta := schemaMeta.GetView(viewName)- if viewMeta == nil {- continue- }- v := sc.AddView(catalog.Normalize(viewName), columnsOf(viewMeta.GetColumns())...)- v.Definition = viewMeta.GetDefinition()- if def := viewMeta.GetDefinition(); def != "" {+ for _, schema := range meta.GetSchemas() {+ for _, view := range schema.GetViews() {+ if def := view.GetDefinition(); def != "" {definitions = append(definitions, def)}}@@ -164,19 +141,6 @@ func catalogNeeded(normName, defaultDB, lowerStmt string) bool {return strings.Contains(lowerStmt, strings.ToLower(normName))}-// columnsOf converts storepb columns into omni catalog columns (names-// normalized).-func columnsOf(columns []*metadatapb.ColumnMetadata) []*catalog.Column {- out := make([]*catalog.Column, 0, len(columns))- for _, c := range columns {- if c == nil {- continue- }- out = append(out, catalog.NewColumn(catalog.Normalize(c.Name), c.Type, c.Nullable))- }- return out-}-// convertCandidateType maps an omni completion candidate type onto the bytebase// base.CandidateType. A Trino catalog corresponds to a bytebase DATABASE.func convertCandidateType(t completion.CandidateType) base.CandidateType {diff --git a/backend/plugin/parser/trino/query_span_extractor.go b/backend/plugin/parser/trino/query_span_extractor.goindex 3c57fcc3c8..5a9fa69d32 100644--- a/backend/plugin/parser/trino/query_span_extractor.go+++ b/backend/plugin/parser/trino/query_span_extractor.go@@ -220,7 +220,8 @@ func (q *querySpanExtractor) buildSpanCatalog(ctx context.Context, statement strif err != nil || meta == nil {continue}- for _, def := range loadCatalogMetadata(cat, norm, meta) {+ cat.LoadMetadata(meta.GetProto())+ for _, def := range viewDefinitions(meta.GetProto()) {corpus += "\n" + strings.ToLower(def)}}diff --git a/backend/plugin/schema/mysql/walk_through.go b/backend/plugin/schema/mysql/walk_through.goindex bf459e6599..fabde580fe 100644--- a/backend/plugin/schema/mysql/walk_through.go+++ b/backend/plugin/schema/mysql/walk_through.go@@ -27,27 +27,26 @@ func init() {// WalkThroughWithContext performs DDL simulation using the omni MySQL catalog.// Flow:-// 1. Create the catalog and select the target database.-// 2. loadWalkThroughCatalog: install each object individually with per-object-// pseudo fallback, so one broken CREATE TABLE can't disable the whole-// simulation.+// 1. Create the catalog.+// 2. catalog.LoadMetadata: install the snapshot, standing in for what fails.// 3. catalog.Exec(userSQL) → execute user DDL// 4. Map errors → *storepb.Advice// 5. Convert updated catalog → DatabaseMetadata (for downstream rules)-func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMetadata, asts []base.AST) *storepb.Advice {- if ctx.RawSQL == "" {+func WalkThroughWithContext(ctx context.Context, wtCtx schema.WalkThroughContext, d *model.DatabaseMetadata, asts []base.AST) *storepb.Advice {+ if wtCtx.RawSQL == "" {return nil}dbName := d.GetProto().GetName()precheck := firstMySQLWalkThroughAdvice(precheckMySQLOmniWalkThrough(d, asts),- precheckMySQLRawWalkThrough(d, ctx.RawSQL, asts),+ precheckMySQLRawWalkThrough(d, wtCtx.RawSQL, asts),)- // Step 1: Create the catalog and the target database.+ // Step 1: Create the catalog. LoadMetadata creates and selects the target+ // database itself; only a RENAME destination needs one made up front.c := catalog.New()- initSQL := fmt.Sprintf("SET foreign_key_checks = 0;\nCREATE DATABASE IF NOT EXISTS %s;\nUSE %s;", mysqlQuoteIdentifier(dbName), mysqlQuoteIdentifier(dbName))+ initSQL := "SET foreign_key_checks = 0;"for _, targetDB := range mysqlRenameTargetDatabases(d, asts) {initSQL += fmt.Sprintf("\nCREATE DATABASE IF NOT EXISTS %s;", mysqlQuoteIdentifier(targetDB))}@@ -61,10 +60,8 @@ func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMeta}}- // Step 2: Install every schema object individually with pseudo fallback.- // TODO: thread a real context.Context through WalkThroughContext; for now the- // loader only uses it for early cancellation during catalog bulk-load.- if err := loadWalkThroughCatalog(context.Background(), c, dbName, d.GetProto()); err != nil {+ // Step 2: Install every schema object, with stand-ins for the ones that fail.+ if _, err := c.LoadMetadata(ctx, d.GetProto()); err != nil {return &storepb.Advice{Status: storepb.Advice_ERROR,Code: code.DDLSimulationFailed.Int32(),@@ -75,7 +72,7 @@ func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMeta}// Step 3: Execute user SQL.- results, execErr := c.Exec(ctx.RawSQL, &catalog.ExecOptions{ContinueOnError: true})+ results, execErr := c.Exec(wtCtx.RawSQL, &catalog.ExecOptions{ContinueOnError: true})if execErr != nil {errCode := code.DDLSimulationFailedcontent := execErr.Error()@@ -780,7 +777,11 @@ func tableToProto(t *catalog.Table) *metadatapb.TableMetadata {for _, idx := range t.Indexes {indexType := strings.ToUpper(idx.IndexType)if indexType == "" {+ // A key without USING gets the engine's default access method.indexType = "BTREE"+ if strings.EqualFold(t.Engine, "MEMORY") || strings.EqualFold(t.Engine, "HEAP") {+ indexType = "HASH"+ }}idxMeta := &metadatapb.IndexMetadata{Name: idx.Name,diff --git a/backend/plugin/schema/mysql/walk_through_loader.go b/backend/plugin/schema/mysql/walk_through_loader.godeleted file mode 100644index cbbd4116e3..0000000000--- a/backend/plugin/schema/mysql/walk_through_loader.go+++ /dev/null@@ -1,1083 +0,0 @@-package mysql--import (- "cmp"- "context"- "slices"- "strings"-- "github.com/pkg/errors"-- metadatapb "github.com/bytebase/omni/metadata"- "github.com/bytebase/omni/mysql/ast"- "github.com/bytebase/omni/mysql/catalog"- mysqlparser "github.com/bytebase/omni/mysql/parser"-)--// autoIncrementSentinel matches the value that MySQL sync writes into-// ColumnMetadata.Default when the column is AUTO_INCREMENT. See-// get_database_definition.go:isAutoIncrement.-const autoIncrementSentinel = "AUTO_INCREMENT"--// loadWalkThroughCatalog installs every schema object from DatabaseSchemaMetadata-// into the omni MySQL catalog with per-object isolation and pseudo fallback.-//-// Rationale: the prior implementation emitted a single concatenated DDL and-// executed it as one Exec call. A single failing CREATE TABLE would leave the-// catalog missing that table, and every downstream view / user query that-// referenced it would also fail. This loader restricts each failure to its-// owning object: if a table cannot be installed as-is, a pseudo table with-// the right column names (but TEXT types, no constraints) is installed in its-// place so that references still resolve.-//-// Preconditions: the caller must already have created and selected the target-// database. This loader mutates foreign_key_checks to allow forward FK-// references during bulk load.-func loadWalkThroughCatalog(ctx context.Context, cat *catalog.Catalog, dbName string, meta *metadatapb.DatabaseSchemaMetadata) error {- if cat == nil {- return errors.New("loadWalkThroughCatalog: nil catalog")- }- if meta == nil {- return nil- }-- // Disable FK checks for the duration of the bulk load. MySQL tolerates- // forward FK references only when this flag is off.- prevFKChecks := cat.ForeignKeyChecks()- cat.SetForeignKeyChecks(false)- defer cat.SetForeignKeyChecks(prevFKChecks)-- cat.SetCurrentDatabase(dbName)-- objects := wtCollectObjects(dbName, meta)- sorted := wtTopoSort(objects)- for _, obj := range sorted {- if err := ctx.Err(); err != nil {- return err- }- wtInstallOne(cat, dbName, obj)- }- return nil-}--// wtObjectKind identifies the kind of a walk-through loader object.-type wtObjectKind int--const (- kindWTTable wtObjectKind = iota- kindWTView- kindWTFunction- kindWTProcedure- kindWTTrigger- kindWTEvent-)--// wtObjectEntry is one unit of work for the walk-through loader.-type wtObjectEntry struct {- kind wtObjectKind- name string- // parentTable is set for kindWTTrigger — triggers must attach to an- // existing table in the catalog, so we carry the owning table name- // separately from the trigger name.- parentTable string-- // Exactly one of the following is set based on kind.- tableMeta *metadatapb.TableMetadata- viewMeta *metadatapb.ViewMetadata- funcMeta *metadatapb.FunctionMetadata- procMeta *metadatapb.ProcedureMetadata- triggerMeta *metadatapb.TriggerMetadata- eventMeta *metadatapb.EventMetadata-}--func (e *wtObjectEntry) key() string {- switch e.kind {- case kindWTTable:- return "table:" + e.name- case kindWTView:- return "view:" + e.name- case kindWTFunction:- return "func:" + e.name- case kindWTProcedure:- return "proc:" + e.name- case kindWTTrigger:- return "trigger:" + e.parentTable + "." + e.name- case kindWTEvent:- return "event:" + e.name- }- return "unknown:" + e.name-}--func (e *wtObjectEntry) sortKey() string {- base := wtKindLabel(e.kind) + "\x00" + e.name- if e.kind == kindWTTrigger {- // Keep triggers grouped by their parent table so stable ordering- // within a file doesn't depend on trigger name alone.- base += "\x00" + e.parentTable- }- return base-}--func wtKindLabel(k wtObjectKind) string {- switch k {- case kindWTTable:- return "1table"- case kindWTView:- return "2view"- case kindWTFunction:- return "3function"- case kindWTProcedure:- return "4procedure"- case kindWTTrigger:- // Triggers must install after their target table. The ordering here- // alone does the trick because kindWTTable sorts before kindWTTrigger.- return "5trigger"- case kindWTEvent:- return "6event"- }- return "9unknown"-}--// wtCollectObjects flattens DatabaseSchemaMetadata into wtObjectEntry values.-// MySQL metadata uses a single empty-named schema, so we look only at the first.-func wtCollectObjects(_ string, meta *metadatapb.DatabaseSchemaMetadata) []*wtObjectEntry {- var out []*wtObjectEntry- for _, sm := range meta.Schemas {- for _, tbl := range sm.Tables {- if tbl == nil || tbl.Name == "" {- continue- }- out = append(out, &wtObjectEntry{- kind: kindWTTable,- name: tbl.Name,- tableMeta: tbl,- })- for _, trg := range tbl.Triggers {- if trg == nil || trg.Name == "" {- continue- }- out = append(out, &wtObjectEntry{- kind: kindWTTrigger,- name: trg.Name,- parentTable: tbl.Name,- triggerMeta: trg,- })- }- }- for _, view := range sm.Views {- if view == nil || view.Name == "" {- continue- }- out = append(out, &wtObjectEntry{- kind: kindWTView,- name: view.Name,- viewMeta: view,- })- }- for _, fn := range sm.Functions {- if fn == nil || fn.Name == "" {- continue- }- out = append(out, &wtObjectEntry{- kind: kindWTFunction,- name: fn.Name,- funcMeta: fn,- })- }- for _, proc := range sm.Procedures {- if proc == nil || proc.Name == "" {- continue- }- out = append(out, &wtObjectEntry{- kind: kindWTProcedure,- name: proc.Name,- procMeta: proc,- })- }- for _, ev := range sm.Events {- if ev == nil || ev.Name == "" {- continue- }- out = append(out, &wtObjectEntry{- kind: kindWTEvent,- name: ev.Name,- eventMeta: ev,- })- }- }- return out-}--// wtTopoSort orders objects by dependency using Tarjan SCC.-//-// For MySQL the dependency graph is extremely thin: tables have no cross-edges-// (FKs are tolerated by SetForeignKeyChecks(false)), views tolerate forward-// refs via DefineView semantics, and functions/procedures reference objects-// only in their body text (opaque to us). So the graph reduces to a fixed-// per-kind ordering: tables → views → functions → procedures. We still run-// Tarjan so the shape of this file mirrors the pg walk-through loader, and so-// the ordering remains deterministic if dependency edges are added later.-func wtTopoSort(objects []*wtObjectEntry) []*wtObjectEntry {- if len(objects) == 0 {- return nil- }- index := make(map[string]*wtObjectEntry, len(objects))- for _, obj := range objects {- index[obj.key()] = obj- }-- edges := wtBuildEdges(objects, index)- sccs := wtTarjanSCC(objects, edges)-- sccOf := make(map[string]int, len(objects))- for i, scc := range sccs {- for _, obj := range scc {- sccOf[obj.key()] = i- }- }-- condensedEdges := make([][]int, len(sccs))- inDegree := make([]int, len(sccs))- seenEdge := make(map[[2]int]bool)- for src, dsts := range edges {- srcSCC, ok := sccOf[src]- if !ok {- continue- }- for _, dst := range dsts {- dstSCC, ok2 := sccOf[dst]- if !ok2 || dstSCC == srcSCC {- continue- }- edge := [2]int{dstSCC, srcSCC}- if seenEdge[edge] {- continue- }- seenEdge[edge] = true- condensedEdges[dstSCC] = append(condensedEdges[dstSCC], srcSCC)- inDegree[srcSCC]++- }- }-- ready := make([]int, 0)- for i := range sccs {- if inDegree[i] == 0 {- ready = append(ready, i)- }- }- wtSortSCCsByMin(ready, sccs)-- var flat []*wtObjectEntry- for len(ready) > 0 {- next := ready[0]- ready = ready[1:]- flat = append(flat, wtSortedSCCMembers(sccs[next])...)- for _, nb := range condensedEdges[next] {- inDegree[nb]--- if inDegree[nb] == 0 {- ready = append(ready, nb)- }- }- wtSortSCCsByMin(ready, sccs)- }-- if len(flat) != len(objects) {- emitted := make(map[string]bool, len(flat))- for _, o := range flat {- emitted[o.key()] = true- }- var missed []*wtObjectEntry- for _, o := range objects {- if !emitted[o.key()] {- missed = append(missed, o)- }- }- slices.SortStableFunc(missed, func(a, b *wtObjectEntry) int {- return cmp.Compare(a.sortKey(), b.sortKey())- })- flat = append(flat, missed...)- }- return flat-}--// wtBuildEdges is intentionally thin today. See wtTopoSort for the rationale.-func wtBuildEdges(_ []*wtObjectEntry, _ map[string]*wtObjectEntry) map[string][]string {- return map[string][]string{}-}--func wtTarjanSCC(objects []*wtObjectEntry, edges map[string][]string) [][]*wtObjectEntry {- type state struct {- index, low int- onStack bool- }- st := make(map[string]*state, len(objects))- byKey := make(map[string]*wtObjectEntry, len(objects))- for _, obj := range objects {- byKey[obj.key()] = obj- }-- keys := make([]string, 0, len(objects))- for _, obj := range objects {- keys = append(keys, obj.key())- }- slices.Sort(keys)-- var (- idx int- stack []string- result [][]*wtObjectEntry- )-- var strongconnect func(v string)- strongconnect = func(v string) {- st[v] = &state{index: idx, low: idx, onStack: true}- idx++- stack = append(stack, v)- for _, w := range edges[v] {- if _, ok := byKey[w]; !ok {- continue- }- if _, seen := st[w]; !seen {- strongconnect(w)- st[v].low = min(st[v].low, st[w].low)- } else if st[w].onStack {- st[v].low = min(st[v].low, st[w].index)- }- }- if st[v].low == st[v].index {- var scc []*wtObjectEntry- for {- w := stack[len(stack)-1]- stack = stack[:len(stack)-1]- st[w].onStack = false- scc = append(scc, byKey[w])- if w == v {- break- }- }- result = append(result, scc)- }- }-- for _, k := range keys {- if _, seen := st[k]; !seen {- strongconnect(k)- }- }- return result-}--func wtSortedSCCMembers(scc []*wtObjectEntry) []*wtObjectEntry {- out := make([]*wtObjectEntry, len(scc))- copy(out, scc)- slices.SortStableFunc(out, func(a, b *wtObjectEntry) int {- return cmp.Compare(a.sortKey(), b.sortKey())- })- return out-}--func wtSortSCCsByMin(indices []int, sccs [][]*wtObjectEntry) {- slices.SortStableFunc(indices, func(a, b int) int {- return cmp.Compare(wtMinSortKey(sccs[a]), wtMinSortKey(sccs[b]))- })-}--func wtMinSortKey(scc []*wtObjectEntry) string {- if len(scc) == 0 {- return ""- }- best := scc[0].sortKey()- for _, o := range scc[1:] {- if k := o.sortKey(); k < best {- best = k- }- }- return best-}--// wtInstallOne attempts a real install and falls back to pseudo for objects-// that have a pseudo form. For objects without one (functions, procedures)-// failure silently drops the object from the catalog.-func wtInstallOne(cat *catalog.Catalog, dbName string, obj *wtObjectEntry) {- if err := wtInstallReal(cat, obj); err == nil {- return- }- switch obj.kind {- case kindWTTable:- _ = wtInstallPseudoTable(cat, dbName, obj.tableMeta)- case kindWTView:- _ = wtInstallPseudoView(cat, dbName, obj.viewMeta)- case kindWTTrigger:- _ = wtInstallPseudoTrigger(cat, obj.triggerMeta, obj.parentTable)- case kindWTEvent:- _ = wtInstallPseudoEvent(cat, obj.eventMeta)- default:- // Functions, procedures: no pseudo form. Leave uninstalled.- }-}--// wtInstallReal builds the object's AST directly from metadata and calls the-// corresponding catalog Define* API. This bypasses the bytebase deparser and-// the omni parser on the critical path — both of which can have bugs that-// would otherwise cause an object to fail real install when the metadata-// itself is fine.-//-// Parser use is deliberately confined to cases where the input string comes-// from MySQL itself (COLUMN_TYPE, VIEW_DEFINITION, SHOW CREATE output,-// information_schema expression columns). Parsing those is parsing reality,-// not parsing our own deparse, and that's the distinction that matters.-func wtInstallReal(cat *catalog.Catalog, obj *wtObjectEntry) error {- switch obj.kind {- case kindWTTable:- stmt, err := wtBuildCreateTableStmt(obj.tableMeta)- if err != nil {- return err- }- return cat.DefineTable(stmt)- case kindWTView:- stmt, err := wtBuildCreateViewStmt(obj.viewMeta)- if err != nil {- return err- }- return cat.DefineView(stmt)- case kindWTFunction:- stmt, err := wtParseCreateRoutineStmt(obj.funcMeta.Definition, false)- if err != nil {- return err- }- return cat.DefineFunction(stmt)- case kindWTProcedure:- stmt, err := wtParseCreateRoutineStmt(obj.procMeta.Definition, true)- if err != nil {- return err- }- return cat.DefineProcedure(stmt)- case kindWTTrigger:- tm := obj.triggerMeta- if tm == nil || tm.Timing == "" || tm.Event == "" {- // Missing routing metadata — let pseudo fallback synthesize- // defaults rather than storing a semantically broken trigger.- return errors.New("trigger: missing Timing or Event")- }- return cat.DefineTrigger(wtBuildCreateTriggerStmt(tm, obj.parentTable))- case kindWTEvent:- stmt, err := wtParseCreateEventStmt(obj.eventMeta.Definition)- if err != nil {- return err- }- return cat.DefineEvent(stmt)- }- return errors.Errorf("wtInstallReal: unknown kind %d", obj.kind)-}--// wtBuildCreateTableStmt maps TableMetadata directly onto *ast.CreateTableStmt.-// Expression-bearing fields (DEFAULT, ON UPDATE, GENERATED, CHECK) go through-// wtParseExpr, which tolerates parse failures by silently dropping the-// affected feature rather than failing the whole table.-func wtBuildCreateTableStmt(tbl *metadatapb.TableMetadata) (*ast.CreateTableStmt, error) {- if tbl == nil || tbl.Name == "" {- return nil, errors.New("wtBuildCreateTableStmt: empty table")- }- stmt := &ast.CreateTableStmt{- Table: &ast.TableRef{Name: tbl.Name},- }-- for _, col := range tbl.Columns {- if col == nil || col.Name == "" {- continue- }- def, err := wtBuildColumnDef(col)- if err != nil {- return nil, errors.Wrapf(err, "column %q", col.Name)- }- stmt.Columns = append(stmt.Columns, def)- }-- for _, idx := range tbl.Indexes {- if idx == nil || len(idx.Expressions) == 0 {- continue- }- if c := wtBuildIndexConstraint(idx); c != nil {- stmt.Constraints = append(stmt.Constraints, c)- }- }-- for _, fk := range tbl.ForeignKeys {- if fk == nil {- continue- }- stmt.Constraints = append(stmt.Constraints, wtBuildFKConstraint(fk))- }-- for _, chk := range tbl.CheckConstraints {- if chk == nil {- continue- }- if c := wtBuildCheckConstraint(chk); c != nil {- stmt.Constraints = append(stmt.Constraints, c)- }- }-- if tbl.Engine != "" {- stmt.Options = append(stmt.Options, &ast.TableOption{Name: "ENGINE", Value: tbl.Engine})- }- if tbl.Charset != "" {- stmt.Options = append(stmt.Options, &ast.TableOption{Name: "CHARSET", Value: tbl.Charset})- }- if tbl.Collation != "" {- stmt.Options = append(stmt.Options, &ast.TableOption{Name: "COLLATE", Value: tbl.Collation})- }- if tbl.Comment != "" {- stmt.Options = append(stmt.Options, &ast.TableOption{Name: "COMMENT", Value: tbl.Comment})- }-- // Partitions deliberately skipped: PartitionClause requires parsed- // expression trees that we don't have metadata for in decomposed form.- // Losing partition info does not affect column resolution or query shape.-- return stmt, nil-}--func wtBuildColumnDef(col *metadatapb.ColumnMetadata) (*ast.ColumnDef, error) {- typeName, err := wtParseTypeName(col.Type)- if err != nil {- return nil, err- }- // Column-level charset/collation can come in via the type string or via- // separate metadata fields. Prefer type-embedded if present, else take- // the separate fields.- if typeName.Charset == "" && col.CharacterSet != "" {- typeName.Charset = col.CharacterSet- }- if typeName.Collate == "" && col.Collation != "" {- typeName.Collate = col.Collation- }- // Spatial SRID (MySQL 8.0): carry the explicit SRID by presence (including the valid- // SRID 0) so a pre-synced `POINT SRID n` column seeds the catalog with its SRID and- // tableToProto reads it back unchanged (no phantom SRID diff).- if col.Srid != nil {- typeName.SRID = int(col.GetSrid())- typeName.HasSRID = true- }-- def := &ast.ColumnDef{- Name: col.Name,- TypeName: typeName,- Comment: col.Comment,- }- if !col.Nullable {- def.Constraints = append(def.Constraints, &ast.ColumnConstraint{Type: ast.ColConstrNotNull})- }- // INVISIBLE column (MySQL 8.0.23+): seed the catalog as invisible so a pre-synced- // INVISIBLE column round-trips (tableToProto re-reads it) instead of surfacing as a- // phantom visibility change.- if col.IsInvisible {- def.Constraints = append(def.Constraints, &ast.ColumnConstraint{Type: ast.ColConstrInvisible})- }-- // MySQL sync encodes AUTO_INCREMENT as the literal string "AUTO_INCREMENT"- // in ColumnMetadata.Default. Detect it here and set AutoIncrement instead- // of treating it as a real default expression.- isAutoInc := strings.EqualFold(col.Default, autoIncrementSentinel)- if isAutoInc {- def.AutoIncrement = true- }-- if !isAutoInc && col.Default != "" && col.Generation == nil {- // MySQL sync fills "NULL" as the default for all nullable columns,- // including TEXT/BLOB/JSON/GEOMETRY — types that MySQL's grammar- // does NOT accept a DEFAULT clause on. Emitting `DEFAULT NULL` for- // these would cause DefineTable to reject the whole table and force- // pseudo fallback. Mirror the filter in- // get_database_definition.go:typeSupportsDefaultValue.- if !strings.EqualFold(col.Default, "NULL") || wtTypeSupportsDefault(col.Type) {- if expr, err := wtParseExpr(col.Default); err == nil {- def.DefaultValue = expr- }- // Silent drop on parse failure — a missing DEFAULT is a closer- // approximation of reality than a whole-column failure.- }- }- if col.OnUpdate != "" {- if expr, err := wtParseExpr(col.OnUpdate); err == nil {- def.OnUpdate = expr- }- }- if col.Generation != nil && col.Generation.Expression != "" {- if expr, err := wtParseExpr(col.Generation.Expression); err == nil {- def.Generated = &ast.GeneratedColumn{- Expr: expr,- Stored: col.Generation.Type == metadatapb.GenerationMetadata_TYPE_STORED,- }- }- }- return def, nil-}--func wtBuildIndexConstraint(idx *metadatapb.IndexMetadata) *ast.Constraint {- // IndexMetadata.Type mixes two axes: the constraint kind (FULLTEXT /- // SPATIAL) and the access method (BTREE / HASH). Pick them apart so- // the resulting ast.Constraint sets Type and IndexType correctly.- idxTypeUpper := strings.ToUpper(strings.TrimSpace(idx.Type))-- c := &ast.Constraint{- Name: idx.Name,- }- switch {- case idx.Primary:- c.Type = ast.ConstrPrimaryKey- c.IndexType = idxTypeUpper- case idx.Unique:- c.Type = ast.ConstrUnique- c.IndexType = idxTypeUpper- case idxTypeUpper == "FULLTEXT":- c.Type = ast.ConstrFulltextIndex- case idxTypeUpper == "SPATIAL":- c.Type = ast.ConstrSpatialIndex- default:- c.Type = ast.ConstrIndex- c.IndexType = idxTypeUpper- }- // Functional key parts (MySQL 8.0+) arrive as expressions in- // idx.Expressions — either parenthesized `(lower(name))` or- // function-call `lower(name)` form. Those MUST live in- // IndexColumns[*].Expr, not in the bare-identifier Columns slice.- // Per-part length or DESC also force IndexColumns.- needIC := false- for i, expr := range idx.Expressions {- if wtIsExpressionIndexKey(expr) ||- (i < len(idx.KeyLength) && idx.KeyLength[i] > 0) ||- (i < len(idx.Descending) && idx.Descending[i]) {- needIC = true- break- }- }- if needIC {- for i, expr := range idx.Expressions {- ic := &ast.IndexColumn{Expr: wtIndexKeyExpr(expr)}- if i < len(idx.KeyLength) && idx.KeyLength[i] > 0 {- ic.Length = int(idx.KeyLength[i])- }- if i < len(idx.Descending) && idx.Descending[i] {- ic.Desc = true- }- c.IndexColumns = append(c.IndexColumns, ic)- }- } else {- for _, expr := range idx.Expressions {- c.Columns = append(c.Columns, wtUnquoteIdent(expr))- }- }- return c-}--// wtIsExpressionIndexKey returns true if the key-part string looks like a-// functional index expression rather than a bare identifier. Heuristic:-// presence of a paren implies either a function call or a parenthesized-// expression. Backticked identifiers don't normally contain parens.-func wtIsExpressionIndexKey(s string) bool {- return strings.Contains(s, "(")-}--// wtIndexKeyExpr converts one idx.Expressions entry into an ExprNode. For-// functional keys we parse the expression text (source is MySQL's own-// information_schema output, not our deparse); bare identifiers become a-// plain ColumnRef.-func wtIndexKeyExpr(s string) ast.ExprNode {- if wtIsExpressionIndexKey(s) {- if parsed, err := wtParseExpr(s); err == nil {- return parsed- }- // Parse failed — best-effort: fall through to ColumnRef so at least- // the constraint installs with some key part instead of an empty- // IndexColumn.- }- return &ast.ColumnRef{Column: wtUnquoteIdent(s)}-}--// wtTypeSupportsDefault returns false for MySQL column types that disallow a-// literal DEFAULT clause (BLOB family, JSON, GEOMETRY). Mirrors the filter-// used by the bytebase writer — see get_database_definition.go:-// typeSupportsDefaultValue — so the loader refuses defaults for the same-// types the writer refuses to emit them for.-func wtTypeSupportsDefault(typeStr string) bool {- head := strings.ToLower(strings.TrimSpace(typeStr))- if i := strings.IndexAny(head, " ("); i >= 0 {- head = head[:i]- }- switch head {- case "blob", "tinyblob", "mediumblob", "longblob", "json", "geometry":- return false- }- return true-}--func wtBuildFKConstraint(fk *metadatapb.ForeignKeyMetadata) *ast.Constraint {- return &ast.Constraint{- Type: ast.ConstrForeignKey,- Name: fk.Name,- Columns: fk.Columns,- RefTable: &ast.TableRef{Schema: fk.ReferencedSchema, Name: fk.ReferencedTable},- RefColumns: fk.ReferencedColumns,- OnUpdate: wtFKAction(fk.OnUpdate),- OnDelete: wtFKAction(fk.OnDelete),- Match: strings.ToUpper(fk.MatchType),- }-}--func wtFKAction(s string) ast.ReferenceAction {- switch strings.ToUpper(strings.TrimSpace(s)) {- case "RESTRICT":- return ast.RefActRestrict- case "CASCADE":- return ast.RefActCascade- case "SET NULL":- return ast.RefActSetNull- case "SET DEFAULT":- return ast.RefActSetDefault- case "NO ACTION":- return ast.RefActNoAction- default:- return ast.RefActNone- }-}--func wtBuildCheckConstraint(chk *metadatapb.CheckConstraintMetadata) *ast.Constraint {- if chk.Expression == "" {- return nil- }- expr, err := wtParseExpr(chk.Expression)- if err != nil {- return nil- }- return &ast.Constraint{- Type: ast.ConstrCheck,- Name: chk.Name,- Expr: expr,- }-}--// wtBuildCreateViewStmt builds the CreateViewStmt directly; the SELECT body is-// parsed from VIEW_DEFINITION (a MySQL-produced string, not our deparse).-// DefineView tolerates a nil Select, so forward references, cyclic views and-// body-parse failures are all survivable.-func wtBuildCreateViewStmt(view *metadatapb.ViewMetadata) (*ast.CreateViewStmt, error) {- if view == nil || view.Name == "" {- return nil, errors.New("wtBuildCreateViewStmt: empty view")- }- stmt := &ast.CreateViewStmt{- OrReplace: true,- Name: &ast.TableRef{Name: view.Name},- SelectText: view.Definition,- }- for _, col := range view.Columns {- if col != nil && col.Name != "" {- stmt.Columns = append(stmt.Columns, col.Name)- }- }- if view.Definition != "" {- if sel, err := wtParseSelect(view.Definition); err == nil {- stmt.Select = sel- }- }- return stmt, nil-}--// wtParseCreateRoutineStmt parses the full SHOW CREATE FUNCTION / PROCEDURE-// text that MySQL sync stored in FunctionMetadata.Definition /-// ProcedureMetadata.Definition. The result is the AST form DefineFunction and-// DefineProcedure consume.-func wtParseCreateRoutineStmt(definition string, isProcedure bool) (*ast.CreateFunctionStmt, error) {- if strings.TrimSpace(definition) == "" {- return nil, errors.New("empty definition")- }- list, err := mysqlparser.Parse(definition)- if err != nil {- return nil, err- }- if list == nil {- return nil, errors.New("nil parse result")- }- for _, n := range list.Items {- if stmt, ok := n.(*ast.CreateFunctionStmt); ok {- stmt.IsProcedure = isProcedure- return stmt, nil- }- }- return nil, errors.New("no CreateFunctionStmt in parse result")-}--// wtParseTypeName parses a MySQL column-type string (the output of-// information_schema.COLUMNS.COLUMN_TYPE, e.g. "varchar(255)",-// "int unsigned", "enum('a','b')") into *ast.DataType by wrapping it in a-// throwaway CREATE TABLE and letting the omni parser do the work.-func wtParseTypeName(typeStr string) (*ast.DataType, error) {- typeStr = strings.TrimSpace(typeStr)- if typeStr == "" {- return nil, errors.New("empty type string")- }- sql := "CREATE TABLE `__bb_wt_type_probe` (`__bb_c` " + typeStr + ")"- list, err := mysqlparser.Parse(sql)- if err != nil {- return nil, errors.Wrapf(err, "parse type %q", typeStr)- }- if list == nil || len(list.Items) == 0 {- return nil, errors.Errorf("parse type %q: empty result", typeStr)- }- ct, ok := list.Items[0].(*ast.CreateTableStmt)- if !ok {- return nil, errors.Errorf("parse type %q: expected CreateTableStmt, got %T", typeStr, list.Items[0])- }- if len(ct.Columns) == 0 || ct.Columns[0].TypeName == nil {- return nil, errors.Errorf("parse type %q: no type in parsed column", typeStr)- }- return ct.Columns[0].TypeName, nil-}--// wtParseExpr parses an expression string via a SELECT wrapper. Used for-// DEFAULT, ON UPDATE, generated-column and CHECK expressions — all of which-// originate in MySQL's information_schema or SHOW CREATE output (reality,-// not our deparse).-func wtParseExpr(exprStr string) (ast.ExprNode, error) {- exprStr = strings.TrimSpace(exprStr)- if exprStr == "" {- return nil, errors.New("empty expression")- }- sql := "SELECT (" + exprStr + ") AS `__bb_wt_expr_probe`"- list, err := mysqlparser.Parse(sql)- if err != nil {- return nil, errors.Wrapf(err, "parse expr %q", exprStr)- }- if list == nil || len(list.Items) == 0 {- return nil, errors.Errorf("parse expr %q: empty result", exprStr)- }- sel, ok := list.Items[0].(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("parse expr %q: expected SelectStmt, got %T", exprStr, list.Items[0])- }- if len(sel.TargetList) == 0 {- return nil, errors.Errorf("parse expr %q: empty target list", exprStr)- }- rt, ok := sel.TargetList[0].(*ast.ResTarget)- if !ok {- return nil, errors.Errorf("parse expr %q: unexpected target %T", exprStr, sel.TargetList[0])- }- return rt.Val, nil-}--// wtParseSelect parses a SELECT body (as returned by VIEW_DEFINITION) into-// *ast.SelectStmt.-func wtParseSelect(body string) (*ast.SelectStmt, error) {- list, err := mysqlparser.Parse(body)- if err != nil {- return nil, err- }- if list == nil || len(list.Items) == 0 {- return nil, errors.New("empty parse result")- }- sel, ok := list.Items[0].(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("expected SelectStmt, got %T", list.Items[0])- }- return sel, nil-}--func wtUnquoteIdent(s string) string {- s = strings.TrimSpace(s)- if len(s) >= 2 && s[0] == '`' && s[len(s)-1] == '`' {- return strings.ReplaceAll(s[1:len(s)-1], "``", "`")- }- return s-}--// wtInstallPseudoTable installs a degraded table with all-TEXT columns and no-// constraints. Built as an AST and handed to DefineTable directly — no SQL-// string and no parser on the critical path.-func wtInstallPseudoTable(cat *catalog.Catalog, _ string, tbl *metadatapb.TableMetadata) error {- if tbl == nil || tbl.Name == "" {- return errors.New("pseudo table: missing name")- }-- stmt := &ast.CreateTableStmt{- Table: &ast.TableRef{Name: tbl.Name},- }-- seen := make(map[string]bool, len(tbl.Columns))- for _, col := range tbl.Columns {- if col == nil || col.Name == "" || seen[col.Name] {- continue- }- seen[col.Name] = true- stmt.Columns = append(stmt.Columns, &ast.ColumnDef{- Name: col.Name,- TypeName: wtPseudoTextType(),- })- }- if len(stmt.Columns) == 0 {- // MySQL requires at least one column — give it a placeholder so the- // object still exists and blocks future duplicate-name installs.- stmt.Columns = append(stmt.Columns, &ast.ColumnDef{- Name: "__bb_placeholder",- TypeName: wtPseudoTextType(),- })- }- return cat.DefineTable(stmt)-}--// wtInstallPseudoView installs a degraded view whose SELECT list is a series-// of NULL literals aliased to the original view's column names. Built in AST-// form and installed via DefineView directly.-func wtInstallPseudoView(cat *catalog.Catalog, _ string, view *metadatapb.ViewMetadata) error {- if view == nil || view.Name == "" {- return errors.New("pseudo view: missing name")- }-- var targets []ast.ExprNode- seen := make(map[string]bool)- addTarget := func(name string) {- if name == "" || seen[name] {- return- }- seen[name] = true- targets = append(targets, &ast.ResTarget{- Name: name,- Val: &ast.NullLit{},- })- }- for _, col := range view.Columns {- if col != nil {- addTarget(col.Name)- }- }- if len(targets) == 0 {- for _, dc := range view.DependencyColumns {- if dc != nil {- addTarget(dc.Column)- }- }- }- if len(targets) == 0 {- addTarget("__bb_placeholder")- }-- return cat.DefineView(&ast.CreateViewStmt{- OrReplace: true,- Name: &ast.TableRef{Name: view.Name},- Select: &ast.SelectStmt{TargetList: targets},- })-}--func wtPseudoTextType() *ast.DataType {- return &ast.DataType{Name: "TEXT"}-}--// wtBuildCreateTriggerStmt hand-constructs *ast.CreateTriggerStmt from-// TriggerMetadata. Sync fills only the body (information_schema-// TRIGGERS.ACTION_STATEMENT) without the CREATE TRIGGER header, so we wire-// Name/Timing/Event/Table/BodyText directly. Body is a parsed sp_proc_stmt —-// best-effort only; on parse failure we still hand DefineTrigger a usable-// stmt with BodyText populated.-func wtBuildCreateTriggerStmt(tm *metadatapb.TriggerMetadata, tableName string) *ast.CreateTriggerStmt {- stmt := &ast.CreateTriggerStmt{- Name: tm.Name,- Timing: tm.Timing,- Event: tm.Event,- Table: &ast.TableRef{Name: tableName},- BodyText: tm.Body,- }- if body := wtParseTriggerBody(tableName, tm.Timing, tm.Event, tm.Body); body != nil {- stmt.Body = body- }- return stmt-}--// wtParseTriggerBody wraps the raw trigger body in a known-good CREATE TRIGGER-// template so the omni parser can produce a Body node. Source is MySQL's-// information_schema output, not our own deparse.-func wtParseTriggerBody(tableName, timing, event, body string) ast.Node {- if strings.TrimSpace(body) == "" {- return nil- }- if tableName == "" {- tableName = "__bb_probe_table"- }- if timing == "" {- timing = "BEFORE"- }- if event == "" {- event = "INSERT"- }- sql := "CREATE TRIGGER `__bb_probe_trigger` " + timing + " " + event +- " ON `" + tableName + "` FOR EACH ROW " + body- list, err := mysqlparser.Parse(sql)- if err != nil || list == nil {- return nil- }- for _, n := range list.Items {- if ct, ok := n.(*ast.CreateTriggerStmt); ok {- return ct.Body- }- }- return nil-}--// wtInstallPseudoTrigger installs a trigger stub when the real install path-// fails (e.g. body node construction tripped up DefineTrigger, or the parent-// table name was empty in metadata). The stub preserves Name and routing-// (Timing/Event/Table) so references still resolve; Body collapses to the-// trivial no-op "BEGIN END".-func wtInstallPseudoTrigger(cat *catalog.Catalog, tm *metadatapb.TriggerMetadata, tableName string) error {- if tm == nil || tm.Name == "" {- return errors.New("pseudo trigger: missing name")- }- timing := tm.Timing- if timing == "" {- timing = "BEFORE"- }- event := tm.Event- if event == "" {- event = "INSERT"- }- if tableName == "" {- tableName = "__bb_placeholder"- }- return cat.DefineTrigger(&ast.CreateTriggerStmt{- Name: tm.Name,- Timing: timing,- Event: event,- Table: &ast.TableRef{Name: tableName},- BodyText: "BEGIN END",- })-}--// wtParseCreateEventStmt parses the full SHOW CREATE EVENT text that sync-// stored in EventMetadata.Definition into *ast.CreateEventStmt.-func wtParseCreateEventStmt(definition string) (*ast.CreateEventStmt, error) {- if strings.TrimSpace(definition) == "" {- return nil, errors.New("empty event definition")- }- list, err := mysqlparser.Parse(definition)- if err != nil {- return nil, err- }- if list == nil {- return nil, errors.New("nil parse result")- }- for _, n := range list.Items {- if ev, ok := n.(*ast.CreateEventStmt); ok {- return ev, nil- }- }- return nil, errors.New("no CreateEventStmt in parse result")-}--// wtInstallPseudoEvent installs a bare name-only event when the real install-// path fails (Definition was empty, malformed, or rejected by DefineEvent).-func wtInstallPseudoEvent(cat *catalog.Catalog, em *metadatapb.EventMetadata) error {- if em == nil || em.Name == "" {- return errors.New("pseudo event: missing name")- }- return cat.DefineEvent(&ast.CreateEventStmt{- Name: em.Name,- })-}diff --git a/backend/plugin/schema/mysql/walk_through_loader_test.go b/backend/plugin/schema/mysql/walk_through_loader_test.godeleted file mode 100644index b80305d9e8..0000000000--- a/backend/plugin/schema/mysql/walk_through_loader_test.go+++ /dev/null@@ -1,1138 +0,0 @@-package mysql--import (- "context"- "strings"- "testing"-- metadatapb "github.com/bytebase/omni/metadata"- "github.com/bytebase/omni/mysql/catalog"-)--// These tests exercise loadWalkThroughCatalog against synthetic metadata.-// Nothing here touches bb_export or any external fixture — every input is a-// storepb.*Metadata constructed in memory.--const testDBName = "testdb"--// newLoaderTestCatalog returns a catalog with the test database created and-// selected, matching the setup WalkThroughWithContext performs before the loader runs.-func newLoaderTestCatalog(t *testing.T) *catalog.Catalog {- t.Helper()- c := catalog.New()- if _, err := c.Exec(- "SET foreign_key_checks = 0;\nCREATE DATABASE IF NOT EXISTS `"+testDBName+"`;\nUSE `"+testDBName+"`;",- &catalog.ExecOptions{ContinueOnError: true},- ); err != nil {- t.Fatalf("init catalog: %v", err)- }- return c-}--func runLoader(t *testing.T, c *catalog.Catalog, meta *metadatapb.DatabaseSchemaMetadata) {- t.Helper()- if err := loadWalkThroughCatalog(context.Background(), c, testDBName, meta); err != nil {- t.Fatalf("loadWalkThroughCatalog: %v", err)- }-}--func mustGetTable(t *testing.T, c *catalog.Catalog, name string) *catalog.Table {- t.Helper()- db := c.GetDatabase(testDBName)- if db == nil {- t.Fatalf("database %q not in catalog", testDBName)- }- tbl := db.Tables[strings.ToLower(name)]- if tbl == nil {- t.Fatalf("table %q not in catalog", name)- }- return tbl-}--func mustGetView(t *testing.T, c *catalog.Catalog, name string) *catalog.View {- t.Helper()- db := c.GetDatabase(testDBName)- if db == nil {- t.Fatalf("database %q not in catalog", testDBName)- }- v := db.Views[strings.ToLower(name)]- if v == nil {- t.Fatalf("view %q not in catalog", name)- }- return v-}--func mustGetTrigger(t *testing.T, c *catalog.Catalog, name string) *catalog.Trigger {- t.Helper()- db := c.GetDatabase(testDBName)- if db == nil {- t.Fatalf("database %q not in catalog", testDBName)- }- trg := db.Triggers[strings.ToLower(name)]- if trg == nil {- t.Fatalf("trigger %q not in catalog", name)- }- return trg-}--func mustGetEvent(t *testing.T, c *catalog.Catalog, name string) *catalog.Event {- t.Helper()- db := c.GetDatabase(testDBName)- if db == nil {- t.Fatalf("database %q not in catalog", testDBName)- }- ev := db.Events[strings.ToLower(name)]- if ev == nil {- t.Fatalf("event %q not in catalog", name)- }- return ev-}--// -----------------------------------------------------------------------// wtParseTypeName — covers a representative cross-section of MySQL types.-// ------------------------------------------------------------------------func TestWtParseTypeName(t *testing.T) {- cases := []struct {- in string- wantName string- }{- {"int", "INT"},- {"int unsigned", "INT"},- {"bigint(20) unsigned zerofill", "BIGINT"},- {"varchar(255)", "VARCHAR"},- {"char(1)", "CHAR"},- {"text", "TEXT"},- {"decimal(10,2)", "DECIMAL"},- {"datetime(6)", "DATETIME"},- {"timestamp", "TIMESTAMP"},- {"tinyint(1)", "TINYINT"},- {"blob", "BLOB"},- {"json", "JSON"},- {"enum('a','b','c')", "ENUM"},- {"set('x','y')", "SET"},- {"geometry", "GEOMETRY"},- {"bit(1)", "BIT"},- }- for _, tc := range cases {- t.Run(tc.in, func(t *testing.T) {- dt, err := wtParseTypeName(tc.in)- if err != nil {- t.Fatalf("wtParseTypeName(%q): %v", tc.in, err)- }- if !strings.EqualFold(dt.Name, tc.wantName) {- t.Errorf("wtParseTypeName(%q).Name = %q, want %q", tc.in, dt.Name, tc.wantName)- }- })- }-}--func TestWtParseTypeName_Failures(t *testing.T) {- for _, in := range []string{"", "not a real type (((", "int UNSIGNED ZEROFILL NOT_A_THING"} {- if _, err := wtParseTypeName(in); err == nil {- t.Errorf("wtParseTypeName(%q): expected error, got nil", in)- }- }-}--// -----------------------------------------------------------------------// wtParseExpr — default/ON UPDATE/generated/check expressions.-// ------------------------------------------------------------------------func TestWtParseExpr(t *testing.T) {- for _, in := range []string{- `'hello'`,- `CURRENT_TIMESTAMP`,- `NULL`,- `0`,- `a + b`,- `CONCAT('x', 'y')`,- `DATE_FORMAT(NOW(), '%Y-%m-%d')`,- } {- if _, err := wtParseExpr(in); err != nil {- t.Errorf("wtParseExpr(%q) unexpectedly failed: %v", in, err)- }- }-}--func TestWtParseExpr_Failures(t *testing.T) {- // "SELECT 1" is intentionally NOT in this list: wrapped in our SELECT- // probe it parses as a scalar subquery expression, which is valid.- for _, in := range []string{"", "(((", "this is not sql at all ###"} {- if _, err := wtParseExpr(in); err == nil {- t.Errorf("wtParseExpr(%q): expected error, got nil", in)- }- }-}--// -----------------------------------------------------------------------// Real table install — exercises the full column + constraint + option path.-// ------------------------------------------------------------------------func TestLoader_TableBasic(t *testing.T) {- meta := schemaWithTables(&metadatapb.TableMetadata{- Name: "users",- Engine: "InnoDB",- Charset: "utf8mb4",- Collation: "utf8mb4_0900_ai_ci",- Comment: "users table",- Columns: []*metadatapb.ColumnMetadata{- {- Name: "id",- Type: "bigint unsigned",- Nullable: false,- Default: autoIncrementSentinel,- },- {- Name: "email",- Type: "varchar(255)",- Nullable: false,- CharacterSet: "utf8mb4",- Collation: "utf8mb4_0900_ai_ci",- },- {- Name: "created_at",- Type: "timestamp",- Nullable: false,- Default: "CURRENT_TIMESTAMP",- OnUpdate: "CURRENT_TIMESTAMP",- },- {- Name: "bio",- Type: "text",- Nullable: true,- Comment: "free form text",- },- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- {Name: "uniq_email", Type: "BTREE", Unique: true, Expressions: []string{"email"}},- },- })-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- tbl := mustGetTable(t, c, "users")- if got := len(tbl.Columns); got != 4 {- t.Fatalf("Columns: want 4, got %d", got)- }- if got := tbl.Engine; got != "InnoDB" {- t.Errorf("Engine: %q", got)- }- if got := tbl.Charset; got != "utf8mb4" {- t.Errorf("Charset: %q", got)- }- if got := tbl.Comment; got != "users table" {- t.Errorf("Comment: %q", got)- }-- // Column properties.- idCol := tbl.GetColumn("id")- if idCol == nil {- t.Fatal("id column missing")- }- if !idCol.AutoIncrement {- t.Error("id should be AUTO_INCREMENT (sentinel detection)")- }- if idCol.Nullable {- t.Error("id should be NOT NULL")- }- if strings.ToLower(idCol.DataType) != "bigint" {- t.Errorf("id.DataType = %q", idCol.DataType)- }-- emailCol := tbl.GetColumn("email")- if emailCol == nil || emailCol.Charset != "utf8mb4" {- t.Errorf("email.Charset: %q", emailCol.Charset)- }-- createdCol := tbl.GetColumn("created_at")- // Omni's catalog canonicalizes CURRENT_TIMESTAMP to now() when deparsing- // the parsed expression; both forms are equivalent so assert presence of- // a default and ON UPDATE rather than a specific string.- if createdCol == nil || createdCol.Default == nil || *createdCol.Default == "" {- t.Errorf("created_at default missing: %+v", createdCol)- }- if createdCol == nil || createdCol.OnUpdate == "" {- t.Error("created_at OnUpdate missing")- }-- bioCol := tbl.GetColumn("bio")- if bioCol == nil || bioCol.Comment != "free form text" {- t.Errorf("bio.Comment: %q", bioCol.Comment)- }-- // Primary key must be present as a constraint.- var hasPK, hasUnique bool- for _, con := range tbl.Constraints {- switch con.Type {- case catalog.ConPrimaryKey:- hasPK = true- case catalog.ConUniqueKey:- if con.Name == "uniq_email" {- hasUnique = true- }- default:- }- }- if !hasPK {- t.Error("missing PRIMARY KEY constraint")- }- if !hasUnique {- t.Error("missing uniq_email unique constraint")- }-}--// -----------------------------------------------------------------------// Fulltext / Spatial index fix — the regression we're covering in this PR.-// ------------------------------------------------------------------------func TestLoader_FulltextAndSpatialIndex(t *testing.T) {- meta := schemaWithTables(&metadatapb.TableMetadata{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},- {Name: "body", Type: "text", Nullable: true},- {Name: "location", Type: "geometry", Nullable: false},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- {Name: "ft_body", Type: "FULLTEXT", Expressions: []string{"body"}},- {Name: "sp_location", Type: "SPATIAL", Expressions: []string{"location"}},- },- })-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- tbl := mustGetTable(t, c, "t")- var gotFulltext, gotSpatial bool- for _, idx := range tbl.Indexes {- switch idx.Name {- case "ft_body":- gotFulltext = idx.Fulltext- if !idx.Fulltext {- t.Errorf("ft_body installed but not marked as Fulltext; IndexType=%q", idx.IndexType)- }- case "sp_location":- gotSpatial = idx.Spatial- if !idx.Spatial {- t.Errorf("sp_location installed but not marked as Spatial; IndexType=%q", idx.IndexType)- }- default:- // Other indexes (e.g. PRIMARY) are irrelevant here.- }- }- if !gotFulltext {- t.Error("fulltext index not found on table")- }- if !gotSpatial {- t.Error("spatial index not found on table")- }-}--// -----------------------------------------------------------------------// FK install — foreign_key_checks off, the constraint is recorded unchecked.-// ------------------------------------------------------------------------func TestLoader_ForeignKeyAcrossTables(t *testing.T) {- meta := schemaWithTables(- &metadatapb.TableMetadata{- Name: "parent",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- },- },- &metadatapb.TableMetadata{- Name: "child",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},- {Name: "parent_id", Type: "bigint unsigned", Nullable: false},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- {Name: "idx_parent_id", Type: "BTREE", Expressions: []string{"parent_id"}},- },- ForeignKeys: []*metadatapb.ForeignKeyMetadata{- {- Name: "fk_child_parent",- Columns: []string{"parent_id"},- ReferencedTable: "parent",- ReferencedColumns: []string{"id"},- OnDelete: "CASCADE",- OnUpdate: "RESTRICT",- },- },- },- )-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- child := mustGetTable(t, c, "child")- var fk *catalog.Constraint- for _, con := range child.Constraints {- if con.Type == catalog.ConForeignKey && con.Name == "fk_child_parent" {- fk = con- break- }- }- if fk == nil {- t.Fatal("fk_child_parent constraint missing")- }- if strings.ToUpper(fk.OnDelete) != "CASCADE" {- t.Errorf("OnDelete = %q, want CASCADE", fk.OnDelete)- }- if strings.ToUpper(fk.OnUpdate) != "RESTRICT" {- t.Errorf("OnUpdate = %q, want RESTRICT", fk.OnUpdate)- }-}--// -----------------------------------------------------------------------// Generated columns.-// ------------------------------------------------------------------------func TestLoader_GeneratedColumn(t *testing.T) {- meta := schemaWithTables(&metadatapb.TableMetadata{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "a", Type: "int", Nullable: true},- {Name: "b", Type: "int", Nullable: true},- {- Name: "c",- Type: "int",- Nullable: true,- Generation: &metadatapb.GenerationMetadata{- Type: metadatapb.GenerationMetadata_TYPE_STORED,- Expression: "a + b",- },- },- },- })-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- tbl := mustGetTable(t, c, "t")- col := tbl.GetColumn("c")- if col == nil || col.Generated == nil {- t.Fatalf("generated column not recorded: %+v", col)- }- if !col.Generated.Stored {- t.Error("expected STORED, got VIRTUAL")- }- if !strings.Contains(col.Generated.Expr, "a") || !strings.Contains(col.Generated.Expr, "b") {- t.Errorf("Generated.Expr = %q", col.Generated.Expr)- }-}--// -----------------------------------------------------------------------// View real install.-// ------------------------------------------------------------------------func TestLoader_ViewReal(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Name: testDBName,- Schemas: []*metadatapb.SchemaMetadata{- {- Name: "",- Tables: []*metadatapb.TableMetadata{- {- Name: "users",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},- {Name: "name", Type: "varchar(255)", Nullable: true},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- },- },- },- Views: []*metadatapb.ViewMetadata{- {- Name: "v_users",- Definition: "SELECT id, name FROM users",- },- },- },- },- }-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- v := mustGetView(t, c, "v_users")- if !strings.Contains(strings.ToUpper(v.Definition), "SELECT") {- t.Errorf("view Definition not preserved: %q", v.Definition)- }-}--// -----------------------------------------------------------------------// Pseudo fallbacks. We exercise them directly by calling the helpers because-// the AST-direct real path is permissive enough that natural failures are-// rare; going through the helper means we really test the Define* + ast-// construction that the fallback relies on.-// ------------------------------------------------------------------------func TestLoader_PseudoTable(t *testing.T) {- c := newLoaderTestCatalog(t)- // Minimal metadata — just column names.- tblMeta := &metadatapb.TableMetadata{- Name: "pt",- Columns: []*metadatapb.ColumnMetadata{- {Name: "a"},- {Name: "b"},- {Name: "c"},- },- }- if err := wtInstallPseudoTable(c, testDBName, tblMeta); err != nil {- t.Fatalf("wtInstallPseudoTable: %v", err)- }- tbl := mustGetTable(t, c, "pt")- if len(tbl.Columns) != 3 {- t.Fatalf("pseudo columns: got %d want 3", len(tbl.Columns))- }- for _, col := range tbl.Columns {- if !strings.EqualFold(col.DataType, "text") {- t.Errorf("pseudo column %q: DataType=%q, want text", col.Name, col.DataType)- }- }-}--func TestLoader_PseudoTable_EmptyColumns(t *testing.T) {- c := newLoaderTestCatalog(t)- if err := wtInstallPseudoTable(c, testDBName, &metadatapb.TableMetadata{Name: "empty"}); err != nil {- t.Fatalf("wtInstallPseudoTable(no columns): %v", err)- }- tbl := mustGetTable(t, c, "empty")- if len(tbl.Columns) != 1 {- t.Fatalf("placeholder columns: got %d want 1", len(tbl.Columns))- }- if tbl.Columns[0].Name != "__bb_placeholder" {- t.Errorf("placeholder column name: %q", tbl.Columns[0].Name)- }-}--func TestLoader_PseudoView(t *testing.T) {- c := newLoaderTestCatalog(t)- if err := wtInstallPseudoView(c, testDBName, &metadatapb.ViewMetadata{- Name: "pv",- Columns: []*metadatapb.ColumnMetadata{- {Name: "x"},- {Name: "y"},- },- }); err != nil {- t.Fatalf("wtInstallPseudoView: %v", err)- }- v := mustGetView(t, c, "pv")- if len(v.Columns) != 2 {- t.Fatalf("pseudo view columns: got %d want 2", len(v.Columns))- }-}--// -----------------------------------------------------------------------// Fallback chain — real install fails, pseudo kicks in.-// Simulate by giving a column an unparseable type string.-// ------------------------------------------------------------------------func TestLoader_RealFailsFallsBackToPseudo(t *testing.T) {- meta := schemaWithTables(&metadatapb.TableMetadata{- Name: "broken",- Columns: []*metadatapb.ColumnMetadata{- {Name: "good", Type: "int"},- // Type with unbalanced parens — wtParseTypeName returns an error- // for this, so wtBuildCreateTableStmt propagates and install_real- // fails, triggering pseudo install.- {Name: "bad", Type: "varchar((("},- },- })-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- tbl := mustGetTable(t, c, "broken")- if len(tbl.Columns) != 2 {- t.Fatalf("pseudo columns: got %d want 2", len(tbl.Columns))- }- // Pseudo collapses everything to TEXT.- for _, col := range tbl.Columns {- if !strings.EqualFold(col.DataType, "text") {- t.Errorf("pseudo fallback should collapse columns to TEXT; %q.DataType=%q", col.Name, col.DataType)- }- }-}--// -----------------------------------------------------------------------// Topological ordering — FK forward references work with FK checks off.-// ------------------------------------------------------------------------func TestLoader_ForwardFKInstallsViaFKChecksOff(t *testing.T) {- // Child declared first in the metadata, referencing parent that appears- // after. Because the loader flips foreign_key_checks off during bulk- // install, this must succeed.- meta := schemaWithTables(- &metadatapb.TableMetadata{- Name: "child",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},- {Name: "parent_id", Type: "bigint unsigned", Nullable: false},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- {Name: "idx_pid", Type: "BTREE", Expressions: []string{"parent_id"}},- },- ForeignKeys: []*metadatapb.ForeignKeyMetadata{- {- Name: "fk_fwd",- Columns: []string{"parent_id"},- ReferencedTable: "parent",- ReferencedColumns: []string{"id"},- },- },- },- &metadatapb.TableMetadata{- Name: "parent",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- },- },- )-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- // Both tables present, child's FK is recorded.- mustGetTable(t, c, "parent")- child := mustGetTable(t, c, "child")- var found bool- for _, con := range child.Constraints {- if con.Type == catalog.ConForeignKey && con.Name == "fk_fwd" {- found = true- }- }- if !found {- t.Error("fk_fwd not recorded on child; forward-FK install failed")- }-}--// schemaWithTables wraps tables into a DatabaseSchemaMetadata with the-// expected single empty-named schema that MySQL uses.-func schemaWithTables(tables ...*metadatapb.TableMetadata) *metadatapb.DatabaseSchemaMetadata {- return &metadatapb.DatabaseSchemaMetadata{- Name: testDBName,- Schemas: []*metadatapb.SchemaMetadata{- {- Name: "",- Tables: tables,- },- },- }-}--// schemaWithEvents wraps events into a DatabaseSchemaMetadata.-func schemaWithEvents(events ...*metadatapb.EventMetadata) *metadatapb.DatabaseSchemaMetadata {- return &metadatapb.DatabaseSchemaMetadata{- Name: testDBName,- Schemas: []*metadatapb.SchemaMetadata{- {Name: "", Events: events},- },- }-}--// -----------------------------------------------------------------------// Trigger — happy paths.-//-// Examples match MySQL 8.0 documentation §25.3.1 (Trigger Syntax and-// Examples). Each trigger is attached to a real table so the loader exercises-// the actual kindWTTable → kindWTTrigger ordering.-// ------------------------------------------------------------------------// accountTableForTriggers builds the `account` table used by the canonical-// trigger examples in the MySQL docs.-func accountTableForTriggers() *metadatapb.TableMetadata {- return &metadatapb.TableMetadata{- Name: "account",- Engine: "InnoDB",- Columns: []*metadatapb.ColumnMetadata{- {Name: "acct_num", Type: "int", Nullable: true},- {Name: "amount", Type: "decimal(10,2)", Nullable: true},- },- }-}--func TestLoader_Trigger_SimpleSetSum(t *testing.T) {- // MySQL docs §25.3.1 Example 1:- //- // CREATE TRIGGER ins_sum BEFORE INSERT ON account- // FOR EACH ROW SET @sum = @sum + NEW.amount;- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{- Name: "ins_sum",- Timing: "BEFORE",- Event: "INSERT",- Body: "SET @sum = @sum + NEW.amount",- }))-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- trg := mustGetTrigger(t, c, "ins_sum")- if !strings.EqualFold(trg.Timing, "BEFORE") {- t.Errorf("Timing=%q, want BEFORE", trg.Timing)- }- if !strings.EqualFold(trg.Event, "INSERT") {- t.Errorf("Event=%q, want INSERT", trg.Event)- }- if !strings.EqualFold(trg.Table, "account") {- t.Errorf("Table=%q, want account", trg.Table)- }- if !strings.Contains(trg.Body, "@sum") {- t.Errorf("Body lost raw text: %q", trg.Body)- }-}--func TestLoader_Trigger_BeginEndWithIfElse(t *testing.T) {- // MySQL docs §25.3.1 Example 2 (BEGIN … END with IF/ELSEIF/END IF):- //- // CREATE TRIGGER upd_check BEFORE UPDATE ON account- // FOR EACH ROW- // BEGIN- // IF NEW.amount < 0 THEN- // SET NEW.amount = 0;- // ELSEIF NEW.amount > 100 THEN- // SET NEW.amount = 100;- // END IF;- // END;- body := `BEGIN- IF NEW.amount < 0 THEN- SET NEW.amount = 0;- ELSEIF NEW.amount > 100 THEN- SET NEW.amount = 100;- END IF;- END`- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{- Name: "upd_check",- Timing: "BEFORE",- Event: "UPDATE",- Body: body,- }))-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- trg := mustGetTrigger(t, c, "upd_check")- if !strings.Contains(strings.ToUpper(trg.Body), "IF NEW.AMOUNT") {- t.Errorf("Body lost IF/THEN content: %q", trg.Body)- }-}--func TestLoader_Trigger_AfterUpdateWithSignal(t *testing.T) {- // MySQL docs §15.6.7.1 (SIGNAL inside a trigger):- //- // CREATE TRIGGER validate_amount AFTER UPDATE ON account- // FOR EACH ROW- // BEGIN- // IF NEW.amount < 0 THEN- // SIGNAL SQLSTATE '45000'- // SET MESSAGE_TEXT = 'Negative amount not allowed';- // END IF;- // END;- body := `BEGIN- IF NEW.amount < 0 THEN- SIGNAL SQLSTATE '45000'- SET MESSAGE_TEXT = 'Negative amount not allowed';- END IF;- END`- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{- Name: "validate_amount",- Timing: "AFTER",- Event: "UPDATE",- Body: body,- }))-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- trg := mustGetTrigger(t, c, "validate_amount")- if !strings.EqualFold(trg.Timing, "AFTER") {- t.Errorf("Timing=%q", trg.Timing)- }- if !strings.Contains(strings.ToUpper(trg.Body), "SIGNAL") {- t.Errorf("Body lost SIGNAL: %q", trg.Body)- }-}--// -----------------------------------------------------------------------// Trigger — pseudo fallback.-// ------------------------------------------------------------------------func TestLoader_Trigger_PseudoOnMissingFields(t *testing.T) {- // metadata has only Name (+ Body) — Timing and Event empty. Real install- // should fail (DefineTrigger rejects empty Timing/Event / empty Body- // couldn't parse). Pseudo fills defaults.- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{- Name: "trg_needs_defaults",- // Timing / Event / Body intentionally empty.- }))-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- trg := mustGetTrigger(t, c, "trg_needs_defaults")- if trg.Timing == "" {- t.Error("pseudo trigger Timing should be filled")- }- if trg.Event == "" {- t.Error("pseudo trigger Event should be filled")- }- if !strings.EqualFold(trg.Table, "account") {- t.Errorf("pseudo trigger Table=%q, want account", trg.Table)- }-}--func TestLoader_Trigger_PseudoOnUnparseableBody(t *testing.T) {- // A body the parser can't chew up — real install still succeeds with- // Body=nil + BodyText populated (DefineTrigger tolerates nil Body), or- // falls back to pseudo with body "BEGIN END". Either way the trigger- // must exist in the catalog.- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{- Name: "trg_bad_body",- Timing: "BEFORE",- Event: "INSERT",- Body: "this is not sql $$$",- }))-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- trg := mustGetTrigger(t, c, "trg_bad_body")- if !strings.EqualFold(trg.Timing, "BEFORE") {- t.Errorf("Timing=%q", trg.Timing)- }-}--func TestLoader_Trigger_OnPseudoParentTable(t *testing.T) {- // Parent table has a bad column type → real install fails, pseudo- // installs TEXT-column table. The trigger must still attach.- parent := &metadatapb.TableMetadata{- Name: "account",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int"},- {Name: "bad", Type: "varchar((("}, // forces real install to fail- },- }- meta := schemaWithTables(withTriggers(parent, &metadatapb.TriggerMetadata{- Name: "ins_stub",- Timing: "BEFORE",- Event: "INSERT",- Body: "SET @x = NEW.id",- }))-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- // Parent was pseudo-installed.- tbl := mustGetTable(t, c, "account")- if !strings.EqualFold(tbl.Columns[0].DataType, "text") {- t.Fatalf("expected pseudo parent table; columns=%+v", tbl.Columns)- }- // Trigger attached anyway.- mustGetTrigger(t, c, "ins_stub")-}--// -----------------------------------------------------------------------// Event — happy paths.-//-// Examples match MySQL 8.0 documentation §15.1.12 (CREATE EVENT). The-// Definition string is exactly what SHOW CREATE EVENT would produce — our-// loader parses it to AST via wtParseCreateEventStmt.-// ------------------------------------------------------------------------func TestLoader_Event_OneShotAT(t *testing.T) {- // MySQL docs §15.1.12 Example 1 (one-shot):- //- // CREATE EVENT myevent- // ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR- // DO UPDATE mytable SET mycol = mycol + 1;- def := "CREATE EVENT `myevent` " +- "ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR " +- "DO UPDATE mytable SET mycol = mycol + 1"-- c := newLoaderTestCatalog(t)- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{- Name: "myevent",- Definition: def,- }))-- ev := mustGetEvent(t, c, "myevent")- if !strings.Contains(strings.ToUpper(ev.Schedule), "CURRENT_TIMESTAMP") &&- !strings.Contains(strings.ToUpper(ev.Schedule), "INTERVAL") {- t.Errorf("schedule lost: %q", ev.Schedule)- }-}--func TestLoader_Event_RecurringEveryWithComment(t *testing.T) {- // MySQL docs §15.1.12 Example 2 (recurring + comment):- //- // CREATE EVENT e_hourly- // ON SCHEDULE EVERY 1 HOUR- // COMMENT 'Clears out sessions table each hour.'- // DO DELETE FROM site_activity.sessions;- def := "CREATE EVENT `e_hourly` " +- "ON SCHEDULE EVERY 1 HOUR " +- "COMMENT 'Clears out sessions table each hour.' " +- "DO DELETE FROM site_activity.sessions"-- c := newLoaderTestCatalog(t)- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{- Name: "e_hourly",- Definition: def,- }))-- ev := mustGetEvent(t, c, "e_hourly")- if !strings.Contains(strings.ToUpper(ev.Schedule), "EVERY") ||- !strings.Contains(strings.ToUpper(ev.Schedule), "HOUR") {- t.Errorf("schedule lost: %q", ev.Schedule)- }- if !strings.Contains(ev.Comment, "sessions") {- t.Errorf("comment lost: %q", ev.Comment)- }-}--func TestLoader_Event_DailyBeginEndBody(t *testing.T) {- // MySQL docs §15.1.12 Example 3 (EVERY + STARTS + compound body):- //- // CREATE EVENT e_daily- // ON SCHEDULE- // EVERY 1 DAY- // STARTS CURRENT_TIMESTAMP + INTERVAL 5 HOUR- // COMMENT '...'- // DO- // BEGIN- // INSERT INTO totals (time, total) SELECT NOW(), COUNT(*) FROM sessions;- // DELETE FROM sessions;- // END;- def := "CREATE EVENT `e_daily` " +- "ON SCHEDULE EVERY 1 DAY STARTS CURRENT_TIMESTAMP + INTERVAL 5 HOUR " +- "COMMENT 'Saves total then clears each day' " +- "DO BEGIN " +- " INSERT INTO totals (time, total) SELECT NOW(), COUNT(*) FROM sessions; " +- " DELETE FROM sessions; " +- "END"-- c := newLoaderTestCatalog(t)- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{- Name: "e_daily",- Definition: def,- }))-- ev := mustGetEvent(t, c, "e_daily")- if !strings.Contains(strings.ToUpper(ev.Schedule), "STARTS") {- t.Errorf("schedule lost STARTS: %q", ev.Schedule)- }-}--// -----------------------------------------------------------------------// Event — pseudo fallback.-// ------------------------------------------------------------------------func TestLoader_Event_PseudoOnEmptyDefinition(t *testing.T) {- c := newLoaderTestCatalog(t)- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{- Name: "e_empty",- // Definition intentionally empty — real install fails, pseudo kicks in.- }))- mustGetEvent(t, c, "e_empty")-}--func TestLoader_Event_PseudoOnGarbageDefinition(t *testing.T) {- c := newLoaderTestCatalog(t)- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{- Name: "e_broken",- Definition: "this is definitely not a CREATE EVENT statement $$",- }))- mustGetEvent(t, c, "e_broken")-}--// withTriggers is a tiny builder that attaches triggers to a given table.-func withTriggers(tbl *metadatapb.TableMetadata, triggers ...*metadatapb.TriggerMetadata) *metadatapb.TableMetadata {- tbl.Triggers = append(tbl.Triggers, triggers...)- return tbl-}--// -----------------------------------------------------------------------// Regression: MySQL sync populates `Default="NULL"` for every nullable-// column — including BLOB / JSON / GEOMETRY types whose grammar rejects a-// DEFAULT clause. The loader must skip DefaultValue for those types so-// DefineTable doesn't reject the whole table and force pseudo fallback.-// ------------------------------------------------------------------------func TestLoader_DefaultNull_OnBlobJsonGeometry_StaysReal(t *testing.T) {- meta := schemaWithTables(&metadatapb.TableMetadata{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int", Nullable: false, Default: autoIncrementSentinel},- // All three of these are nullable — sync fills Default="NULL"- // for them. MySQL grammar does NOT accept DEFAULT on these- // types, so the loader must drop the default silently.- {Name: "payload_json", Type: "json", Nullable: true, Default: "NULL"},- {Name: "blob_body", Type: "blob", Nullable: true, Default: "NULL"},- {Name: "location", Type: "geometry", Nullable: true, Default: "NULL"},- // For a type that DOES support DEFAULT, Default="NULL" should- // still be applied.- {Name: "name", Type: "varchar(255)", Nullable: true, Default: "NULL"},- },- })-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- tbl := mustGetTable(t, c, "t")- // Real install path keeps real column types — verify by checking a- // non-TEXT DataType on the JSON / BLOB / GEOMETRY columns.- for _, want := range []struct{ name, dt string }{- {"payload_json", "json"},- {"blob_body", "blob"},- {"location", "geometry"},- } {- got := tbl.GetColumn(want.name)- if got == nil {- t.Errorf("column %q missing", want.name)- continue- }- if !strings.EqualFold(got.DataType, want.dt) {- t.Errorf("column %q: DataType=%q, want %q (pseudo fallback leaked?)", want.name, got.DataType, want.dt)- }- // The DEFAULT must have been suppressed — BLOB/JSON/GEOMETRY must- // NOT carry a stored default.- if got.Default != nil {- t.Errorf("column %q: Default=%v, want nil for unsupported-default type", want.name, *got.Default)- }- }- // Regular varchar still carries DEFAULT NULL.- if name := tbl.GetColumn("name"); name == nil || name.Default == nil {- t.Error("varchar column lost its DEFAULT NULL")- }-}--// -----------------------------------------------------------------------// Regression: MySQL 8.0 functional indexes store key parts as expressions-// like `(lower(name))` or `lower(name)`. Dumping them into Constraint.Columns-// (bare identifier slice) makes DefineTable reject the table. Expression-// keys must land in IndexColumns[*].Expr instead.-// ------------------------------------------------------------------------func TestLoader_FunctionalIndex_ParenthesizedExpression(t *testing.T) {- // MySQL docs §10.3.10 Functional Key Parts:- //- // CREATE TABLE t1 (- // col1 INT, col2 INT,- // INDEX func_index ((ABS(col1)))- // );- meta := schemaWithTables(&metadatapb.TableMetadata{- Name: "t1",- Columns: []*metadatapb.ColumnMetadata{- {Name: "col1", Type: "int", Nullable: true},- {Name: "col2", Type: "int", Nullable: true},- },- Indexes: []*metadatapb.IndexMetadata{- {- Name: "func_index",- Type: "BTREE",- Expressions: []string{"(abs(col1))"},- },- },- })-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- tbl := mustGetTable(t, c, "t1")- if strings.EqualFold(tbl.Columns[0].DataType, "text") {- t.Fatal("table degraded to pseudo (all columns TEXT) — functional index broke real install")- }- // Index must exist.- var found bool- for _, idx := range tbl.Indexes {- if idx.Name == "func_index" {- found = true- // Catalog stores the key-part expression in IndexColumn.Expr.- if len(idx.Columns) != 1 {- t.Fatalf("func_index: expected 1 key part, got %d", len(idx.Columns))- }- }- }- if !found {- t.Error("func_index not installed on table")- }-}--func TestLoader_FunctionalIndex_FunctionCallForm(t *testing.T) {- // Same thing but key part is stored as bare function-call text- // (lower(name) without outer parens) — another shape the proto comment- // explicitly calls out.- meta := schemaWithTables(&metadatapb.TableMetadata{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int", Nullable: false, Default: autoIncrementSentinel},- {Name: "name", Type: "varchar(255)", Nullable: true},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- {Name: "idx_lower_name", Type: "BTREE", Expressions: []string{"lower(`name`)"}},- },- })-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- tbl := mustGetTable(t, c, "t")- if strings.EqualFold(tbl.Columns[0].DataType, "text") {- t.Fatal("table degraded to pseudo — function-call index broke real install")- }- var found bool- for _, idx := range tbl.Indexes {- if idx.Name == "idx_lower_name" {- found = true- }- }- if !found {- t.Error("idx_lower_name not installed")- }-}--// TestLoader_FunctionalIndex_MixedWithBareColumns is a negative-regression-// test: when one key part is an expression and another is a bare column, the-// whole index must still install — all parts route through IndexColumns.-func TestLoader_FunctionalIndex_MixedWithBareColumns(t *testing.T) {- meta := schemaWithTables(&metadatapb.TableMetadata{- Name: "t",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "int", Nullable: false, Default: autoIncrementSentinel},- {Name: "tag", Type: "varchar(64)", Nullable: true},- {Name: "name", Type: "varchar(255)", Nullable: true},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},- {- Name: "idx_mixed",- Type: "BTREE",- Expressions: []string{"tag", "(lower(`name`))"},- },- },- })-- c := newLoaderTestCatalog(t)- runLoader(t, c, meta)-- tbl := mustGetTable(t, c, "t")- if strings.EqualFold(tbl.Columns[0].DataType, "text") {- t.Fatal("table degraded to pseudo — mixed index broke real install")- }-}diff --git a/backend/plugin/schema/mysql/walk_through_test.go b/backend/plugin/schema/mysql/walk_through_test.goindex c2d2849f30..30aeb95f9c 100644--- a/backend/plugin/schema/mysql/walk_through_test.go+++ b/backend/plugin/schema/mysql/walk_through_test.go@@ -1,6 +1,7 @@package mysqlimport (+ "context""io""os""path/filepath"@@ -58,7 +59,7 @@ func TestWalkThrough(t *testing.T) {stmts, _ := sm.GetStatementsForChecks(storepb.Engine_MYSQL, test.Statement)asts := base.ExtractASTs(stmts)- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)if advice != nil {// Compare the advice fieldsrequire.NotNil(t, test.Advice, "unexpected advice for statement %q: %+v", test.Statement, advice)@@ -119,7 +120,7 @@ func TestWalkThroughCreateTableIfNotExistsCTASExistingTable(t *testing.T) {require.True(t, ok)require.True(t, createTable.IfNotExists)- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: statement}, state, asts)+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: statement}, state, asts)require.Nil(t, advice)require.NotNil(t, state.GetSchemaMetadata("").GetTable("t1"))}@@ -141,7 +142,7 @@ func TestWalkThroughSRIDInvisible(t *testing.T) {stmts, _ := sm.GetStatementsForChecks(storepb.Engine_MYSQL, statement)asts := base.ExtractASTs(stmts)- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: statement}, state, asts)+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: statement}, state, asts)require.Nil(t, advice)cols := map[string]*metadatapb.ColumnMetadata{}@@ -164,7 +165,7 @@ func TestWalkThroughSRIDInvisible(t *testing.T) {// TestWalkThroughSRIDInvisibleSeeded verifies the reverse leg of the WalkThroughWithContext// round-trip: when the walk-through starts from already-synced metadata that carries SRID-// / INVISIBLE columns, the proto->catalog seeding (wtBuildColumnDef) must install those+// / INVISIBLE columns, LoadMetadata must install those// attributes into the omni catalog so that running an UNRELATED DDL does not strip them// from the resulting proto. Without seeding, the pre-existing columns load as// plain/no-SRID and — because columnsEqual now compares these fields — surface as phantom@@ -196,7 +197,7 @@ func TestWalkThroughSRIDInvisibleSeeded(t *testing.T) {sm := sheet.NewManager()stmts, _ := sm.GetStatementsForChecks(storepb.Engine_MYSQL, statement)asts := base.ExtractASTs(stmts)- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: statement}, state, asts)+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: statement}, state, asts)require.Nil(t, advice)geo := state.GetSchemaMetadata("").GetTable("geo")@@ -211,3 +212,45 @@ func TestWalkThroughSRIDInvisibleSeeded(t *testing.T) {require.Equal(t, uint32(4326), *cols["pt"].Srid)require.True(t, cols["secret"].IsInvisible, "seeded INVISIBLE must survive")}++func TestWalkThroughKeepsSeededIndexTypes(t *testing.T) {+ origin := &metadatapb.DatabaseSchemaMetadata{+ Name: "test",+ Schemas: []*metadatapb.SchemaMetadata{{+ Tables: []*metadatapb.TableMetadata{+ {+ Name: "disk",+ Engine: "InnoDB",+ Columns: []*metadatapb.ColumnMetadata{{Name: "a", Position: 1, Type: "int"}},+ Indexes: []*metadatapb.IndexMetadata{{Name: "k", Expressions: []string{"a"}, Type: "BTREE", Visible: true}},+ },+ {+ Name: "mem",+ Engine: "MEMORY",+ Columns: []*metadatapb.ColumnMetadata{{Name: "a", Position: 1, Type: "int"}},+ Indexes: []*metadatapb.IndexMetadata{+ {Name: "k_hash", Expressions: []string{"a"}, Type: "HASH", Visible: true},+ {Name: "k_btree", Expressions: []string{"a"}, Type: "BTREE", Visible: true},+ },+ },+ },+ }},+ }+ state := model.NewDatabaseMetadata(origin, nil, nil, storepb.Engine_MYSQL, true)++ statement := "CREATE TABLE unrelated (x INT PRIMARY KEY);"+ sm := sheet.NewManager()+ stmts, _ := sm.GetStatementsForChecks(storepb.Engine_MYSQL, statement)+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: statement}, state, base.ExtractASTs(stmts))+ require.Nil(t, advice)++ // Each index reports its effective access method, the engine's default one+ // included.+ types := map[string]string{}+ for _, name := range []string{"disk", "mem"} {+ for _, idx := range state.GetSchemaMetadata("").GetTable(name).GetProto().GetIndexes() {+ types[name+"."+idx.GetName()] = idx.GetType()+ }+ }+ require.Equal(t, map[string]string{"disk.k": "BTREE", "mem.k_hash": "HASH", "mem.k_btree": "BTREE"}, types)+}diff --git a/backend/plugin/schema/pg/walk_through.go b/backend/plugin/schema/pg/walk_through.goindex 6ab8e9a42f..d6f4400f69 100644--- a/backend/plugin/schema/pg/walk_through.go+++ b/backend/plugin/schema/pg/walk_through.go@@ -23,30 +23,28 @@ func init() {// WalkThrough walks through the PostgreSQL DDL and builds catalog metadata.func WalkThrough(d *model.DatabaseMetadata, ast []base.AST) *storepb.Advice {- return WalkThroughWithContext(schema.WalkThroughContext{}, d, ast)+ return WalkThroughWithContext(context.Background(), schema.WalkThroughContext{}, d, ast)}// WalkThroughWithContext performs DDL simulation using the omni catalog.-// 1. loadWalkThroughCatalog(metadata) → load existing schema into catalog+// 1. catalog.LoadMetadata(metadata) → load existing schema into catalog// 2. catalog.Exec(userSQL) → execute user DDL, collect per-statement Changes// 3. Map exec errors → *storepb.Advice// 4. Merge Changes into original metadata → FinalMetadata for downstream rules-func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMetadata, _ []base.AST) *storepb.Advice {- if ctx.RawSQL == "" {+func WalkThroughWithContext(ctx context.Context, wtCtx schema.WalkThroughContext, d *model.DatabaseMetadata, _ []base.AST) *storepb.Advice {+ if wtCtx.RawSQL == "" {return nil}// Step 1: Load existing schema into two catalogs — one as snapshot, one for exec.- // TODO(BYT-9215): Thread a real context.Context through the call chain instead of using context.Background().- // WalkThroughContext is not a context.Context and the caller does not pass one.catBefore := catalog.New()- if ctx.SessionUser != "" {- catBefore.SetSessionUser(ctx.SessionUser)+ if wtCtx.SessionUser != "" {+ catBefore.SetSessionUser(wtCtx.SessionUser)}if searchPath := getConfiguredSearchPath(d); len(searchPath) > 0 {catBefore.SetSearchPath(searchPath)}- if err := loadWalkThroughCatalog(context.Background(), catBefore, d.GetProto()); err != nil {+ if _, err := catBefore.LoadMetadata(ctx, d.GetProto(), catalog.LoadMetadataOptions{Full: true}); err != nil {return &storepb.Advice{Status: storepb.Advice_ERROR,Code: code.DDLSimulationFailed.Int32(),@@ -59,7 +57,7 @@ func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMetacatAfter := catBefore.Clone()// Step 2: Execute user SQL on catAfter.- results, execErr := catAfter.Exec(ctx.RawSQL, &catalog.ExecOptions{ContinueOnError: true})+ results, execErr := catAfter.Exec(wtCtx.RawSQL, &catalog.ExecOptions{ContinueOnError: true})if execErr != nil {return &storepb.Advice{Status: storepb.Advice_ERROR,diff --git a/backend/plugin/schema/pg/walk_through_apply.go b/backend/plugin/schema/pg/walk_through_apply.goindex 7277c5fe54..6e07544975 100644--- a/backend/plugin/schema/pg/walk_through_apply.go+++ b/backend/plugin/schema/pg/walk_through_apply.go@@ -2,6 +2,7 @@ package pgimport ("fmt"+ "slices""strings""google.golang.org/protobuf/proto"@@ -34,7 +35,11 @@ func applyDiffToMetadata(original *metadatapb.DatabaseSchemaMetadata, catBefore,}}+ renamed := renamePartitions(result, catAfter, diff.Relations)for _, rel := range diff.Relations {+ if (rel.From != nil && renamed[rel.From.OID]) || (rel.To != nil && renamed[rel.To.OID]) {+ continue+ }if rel.Action == catalog.DiffDrop {if sm := findSchema(result, rel.SchemaName); sm != nil {dropRelation(sm, rel)@@ -220,6 +225,9 @@ func dropRelation(sm *metadatapb.SchemaMetadata, rel catalog.RelationDiffEntry)sm.Tables = removeTableByName(sm.Tables, name)sm.Views = removeViewByName(sm.Views, name)sm.MaterializedViews = removeMatViewByName(sm.MaterializedViews, name)+ for _, t := range sm.Tables {+ t.Partitions = removePartitionByName(t.Partitions, name)+ }}func modifyRelation(sm *metadatapb.SchemaMetadata, catBefore, cat *catalog.Catalog, rel catalog.RelationDiffEntry) {@@ -230,6 +238,10 @@ func modifyRelation(sm *metadatapb.SchemaMetadata, catBefore, cat *catalog.Catalcase 'r', 'p', 'f':tbl := findTable(sm, rel.Name)if tbl == nil {+ if p := findPartition(sm, rel.Name); p != nil {+ applyPartitionDiffs(p, catBefore, cat, rel)+ return+ }sm.Tables = append(sm.Tables, relationToTableProto(cat, rel.To))return}@@ -246,6 +258,46 @@ func modifyRelation(sm *metadatapb.SchemaMetadata, catBefore, cat *catalog.Catal}}+// renamePartitions renames each partition the diff renames where it stands,+// with its indexes and constraints as the catalog now has them. The diff shows+// a rename as a drop and an add of the same relation, which would otherwise+// turn the partition into a table of its own. It returns the OIDs it handled.+func renamePartitions(result *metadatapb.DatabaseSchemaMetadata, catAfter *catalog.Catalog, relations []catalog.RelationDiffEntry) map[uint32]bool {+ added := make(map[uint32]catalog.RelationDiffEntry)+ for _, rel := range relations {+ if rel.Action == catalog.DiffAdd && rel.To != nil {+ added[rel.To.OID] = rel+ }+ }+ renamed := make(map[uint32]bool)+ for _, rel := range relations {+ if rel.Action != catalog.DiffDrop || rel.From == nil {+ continue+ }+ add, ok := added[rel.From.OID]+ sm := findSchema(result, rel.SchemaName)+ if !ok || add.SchemaName != rel.SchemaName || sm == nil {+ continue+ }+ if p := findPartition(sm, rel.Name); p != nil {+ tbl := relationToTableProto(catAfter, add.To)+ p.Name = add.Name+ p.Indexes, p.CheckConstraints, p.ExcludeConstraints = tbl.Indexes, tbl.CheckConstraints, tbl.ExcludeConstraints+ renamed[rel.From.OID] = true+ }+ }+ return renamed+}++// applyPartitionDiffs applies a partition's index and constraint changes. A+// partition's metadata holds only those; its columns are its parent table's.+func applyPartitionDiffs(p *metadatapb.TablePartitionMetadata, catBefore, cat *catalog.Catalog, rel catalog.RelationDiffEntry) {+ tbl := &metadatapb.TableMetadata{Indexes: p.Indexes, CheckConstraints: p.CheckConstraints, ExcludeConstraints: p.ExcludeConstraints}+ applyIndexDiffs(tbl, cat, rel)+ applyConstraintDiffs(tbl, catBefore, cat, rel)+ p.Indexes, p.CheckConstraints, p.ExcludeConstraints = tbl.Indexes, tbl.CheckConstraints, tbl.ExcludeConstraints+}+func applyColumnDiffs(tbl *metadatapb.TableMetadata, cat *catalog.Catalog, rel catalog.RelationDiffEntry) {for _, cd := range rel.Columns {switch cd.Action {@@ -529,6 +581,37 @@ func findTable(sm *metadatapb.SchemaMetadata, name string) *metadatapb.TableMetareturn nil}+// findPartition returns the partition named name, at any depth, of a table in sm.+func findPartition(sm *metadatapb.SchemaMetadata, name string) *metadatapb.TablePartitionMetadata {+ var find func(partitions []*metadatapb.TablePartitionMetadata) *metadatapb.TablePartitionMetadata+ find = func(partitions []*metadatapb.TablePartitionMetadata) *metadatapb.TablePartitionMetadata {+ for _, p := range partitions {+ if p.Name == name {+ return p+ }+ if sub := find(p.Subpartitions); sub != nil {+ return sub+ }+ }+ return nil+ }+ for _, t := range sm.Tables {+ if p := find(t.Partitions); p != nil {+ return p+ }+ }+ return nil+}++// removePartitionByName removes the partition named name, at any depth.+func removePartitionByName(partitions []*metadatapb.TablePartitionMetadata, name string) []*metadatapb.TablePartitionMetadata {+ partitions = slices.DeleteFunc(partitions, func(p *metadatapb.TablePartitionMetadata) bool { return p.Name == name })+ for _, p := range partitions {+ p.Subpartitions = removePartitionByName(p.Subpartitions, name)+ }+ return partitions+}+func removeTableByName(tables []*metadatapb.TableMetadata, name string) []*metadatapb.TableMetadata {out := make([]*metadatapb.TableMetadata, 0, len(tables))for _, t := range tables {@@ -719,7 +802,7 @@ func removeFunctionByIdentity(funcs []*metadatapb.FunctionMetadata, identity str// metadata (the catalog does not track comments, and it only keeps the bare// collation name) for attributes that survive by name. Attributes unchanged// between from and to keep their previous metadata verbatim, so a composite-// degraded to the text-backed pseudo fallback does not rewrite untouched+// degraded to a text-backed stand-in does not rewrite untouched// attributes to text — mirroring the per-column granularity of table diffs.//// Within a degraded composite this is a deliberate trade-off: an attribute@@ -727,7 +810,7 @@ func removeFunctionByIdentity(funcs []*metadatapb.FunctionMetadata, identity str// read text in from and to), so that rare change is missed in favor of not// corrupting every untouched attribute. Walk-through metadata is advisory;// post-execution sync restores ground truth. The root fix is modeling the-// types the loader cannot install today (e.g. domains).+// types LoadMetadata cannot install today (e.g. domains).func compositeTypeToProto(cat *catalog.Catalog, name string, from, to *catalog.Relation, previous *metadatapb.CompositeTypeMetadata) *metadatapb.CompositeTypeMetadata {previousAttributes := make(map[string]*metadatapb.CompositeTypeAttribute)composite := &metadatapb.CompositeTypeMetadata{Name: name}@@ -877,3 +960,11 @@ func quoteWalkThroughIdent(name string) string {}return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`}++func wtIsSystemSchema(s string) bool {+ switch s {+ case "pg_catalog", "pg_toast", "information_schema":+ return true+ }+ return strings.HasPrefix(s, "pg_")+}diff --git a/backend/plugin/schema/pg/walk_through_loader_test.go b/backend/plugin/schema/pg/walk_through_apply_test.gosimilarity index 68%rename from backend/plugin/schema/pg/walk_through_loader_test.gorename to backend/plugin/schema/pg/walk_through_apply_test.goindex f11c673063..7e9afb0b28 100644--- a/backend/plugin/schema/pg/walk_through_loader_test.go+++ b/backend/plugin/schema/pg/walk_through_apply_test.go@@ -9,59 +9,9 @@ import ("github.com/stretchr/testify/require"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/schema""github.com/bytebase/bytebase/backend/store/model")-// TestClone_BasicIsolation verifies that Clone produces an independent catalog:-// mutations on the clone do not affect the original.-func TestClone_BasicIsolation(t *testing.T) {- original := catalog.New()- _, err := original.Exec(`- CREATE SCHEMA test_schema;- CREATE TABLE test_schema.t1 (id serial PRIMARY KEY, name text NOT NULL);- CREATE INDEX t1_name_idx ON test_schema.t1 (name);- `, nil)- require.NoError(t, err)-- clone := original.Clone()-- // Mutate clone- _, err = clone.Exec(`- ALTER TABLE test_schema.t1 ADD COLUMN email text;- CREATE TABLE test_schema.t2 (id int PRIMARY KEY);- DROP INDEX test_schema.t1_name_idx;- `, nil)- require.NoError(t, err)-- // Verify original is untouched- origRel := original.GetRelation("test_schema", "t1")- require.NotNil(t, origRel, "original t1 should still exist")- require.Equal(t, 2, len(origRel.Columns), "original t1 should have 2 columns (id, name), not 3")-- require.Nil(t, original.GetRelation("test_schema", "t2"), "original should NOT have t2")-- origIndexes := original.IndexesOf(origRel.OID)- found := false- for _, idx := range origIndexes {- if idx.Name == "t1_name_idx" {- found = true- }- }- require.True(t, found, "original should still have t1_name_idx")-- // Verify clone has the changes- cloneRel := clone.GetRelation("test_schema", "t1")- require.NotNil(t, cloneRel)- require.Equal(t, 3, len(cloneRel.Columns), "clone t1 should have 3 columns")- require.NotNil(t, clone.GetRelation("test_schema", "t2"), "clone should have t2")-- // Diff should show changes- diff := catalog.Diff(original, clone)- require.False(t, diff.IsEmpty(), "diff should not be empty")- require.Greater(t, len(diff.Relations), 0, "should have relation diffs")-}-// TestClone_WalkThroughIntegration tests the full walk-through flow using Clone:// load catalog → clone → exec user DDL on clone → diff → apply.func TestClone_WalkThroughIntegration(t *testing.T) {@@ -87,7 +37,7 @@ func TestClone_WalkThroughIntegration(t *testing.T) {}catBefore := catalog.New()- err := loadWalkThroughCatalog(context.Background(), catBefore, meta)+ _, err := catBefore.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})require.NoError(t, err)catAfter := catBefore.Clone()@@ -132,87 +82,49 @@ func TestClone_WalkThroughIntegration(t *testing.T) {require.Nil(t, origMeta.GetSchemaMetadata("public").GetTable("posts"), "original should not have posts")}-// TestClone_SearchPath tests Clone preserves search path and session user.-func TestClone_SearchPath(t *testing.T) {+func TestWalkThroughKeepsPartitionsUnderTheirTable(t *testing.T) {meta := &metadatapb.DatabaseSchemaMetadata{Name: "postgres",- Schemas: []*metadatapb.SchemaMetadata{- {Name: "public"},- {Name: "alice"},- },+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: "public",+ Tables: []*metadatapb.TableMetadata{{+ Name: "orders",+ Columns: []*metadatapb.ColumnMetadata{{Name: "id", Type: "integer"}},+ Partitions: []*metadatapb.TablePartitionMetadata{+ {Name: "orders_2023"},+ {Name: "orders_2024", Subpartitions: []*metadatapb.TablePartitionMetadata{{Name: "orders_2024_q1"}, {Name: "orders_2024_q2"}}},+ },+ }},+ }},}-catBefore := catalog.New()- catBefore.SetSessionUser("alice")- catBefore.SetSearchPath([]string{"$user", "public"})- err := loadWalkThroughCatalog(context.Background(), catBefore, meta)+ _, err := catBefore.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})require.NoError(t, err)-catAfter := catBefore.Clone()-- // CREATE TABLE without schema should use search path ($user → alice)- _, err = catAfter.Exec(`CREATE TABLE my_table (id int);`, nil)+ results, err := catAfter.Exec(`+ DROP TABLE public.orders_2023;+ DROP TABLE public.orders_2024_q2;+ CREATE INDEX orders_2024_q1_id_idx ON public.orders_2024_q1 (id);+ ALTER TABLE public.orders_2024 RENAME TO orders_2024_archived;+ CREATE INDEX orders_2024_archived_id_idx ON public.orders_2024_archived (id);+ `, &catalog.ExecOptions{ContinueOnError: true})require.NoError(t, err)-- // Table should be in alice schema on clone- require.NotNil(t, catAfter.GetRelation("alice", "my_table"))-- // Original should NOT have it- require.Nil(t, catBefore.GetRelation("alice", "my_table"))-- // Diff should show the new table- diff := catalog.Diff(catBefore, catAfter)- require.False(t, diff.IsEmpty())-- foundAliceTable := false- for _, rel := range diff.Relations {- if rel.SchemaName == "alice" && rel.Name == "my_table" && rel.Action == catalog.DiffAdd {- foundAliceTable = true- }- }- require.True(t, foundAliceTable, "diff should show alice.my_table as added")-}--// TestClone_WalkThroughFunction tests the actual WalkThroughWithContext function-// using Clone (if enabled) produces correct FinalMetadata.-func TestClone_WalkThroughFunction(t *testing.T) {- meta := &metadatapb.DatabaseSchemaMetadata{- Name: "postgres",- Schemas: []*metadatapb.SchemaMetadata{- {- Name: "public",- Tables: []*metadatapb.TableMetadata{- {- Name: "test",- Columns: []*metadatapb.ColumnMetadata{- {Name: "id", Type: "integer", Position: 1},- {Name: "name", Type: "text", Position: 2, Nullable: true},- },- Indexes: []*metadatapb.IndexMetadata{- {Name: "test_pkey", Expressions: []string{"id"}, Unique: true, Primary: true},- },- },- },- },- },- }-- state := model.NewDatabaseMetadata(meta, nil, nil, storepb.Engine_POSTGRES, true)- ctx := schema.WalkThroughContext{- RawSQL: `CREATE TABLE public.new_table (id int PRIMARY KEY, val text);`,+ for _, r := range results {+ require.NoError(t, r.Error, r.SQL)}- advice := WalkThroughWithContext(ctx, state, nil)- require.Nil(t, advice, "walk-through should succeed")-- // Check FinalMetadata has the new table- newTbl := state.GetSchemaMetadata("public").GetTable("new_table")- require.NotNil(t, newTbl, "new_table should exist in FinalMetadata")-- // Check original table is preserved- origTbl := state.GetSchemaMetadata("public").GetTable("test")- require.NotNil(t, origTbl, "test table should still exist")- require.Equal(t, 2, len(origTbl.GetProto().Columns))+ schema := applyDiffToMetadata(meta, catBefore, catAfter, catalog.Diff(catBefore, catAfter)).Schemas[0]+ require.Len(t, schema.Tables, 1, "a changed partition must stay under its table")+ partitions := schema.Tables[0].Partitions+ require.Len(t, partitions, 1)+ require.Equal(t, "orders_2024_archived", partitions[0].Name)+ require.Len(t, partitions[0].Indexes, 1)+ require.Equal(t, "orders_2024_archived_id_idx", partitions[0].Indexes[0].Name)+ require.Len(t, partitions[0].Subpartitions, 1)+ q1 := partitions[0].Subpartitions[0]+ require.Equal(t, "orders_2024_q1", q1.Name)+ require.Len(t, q1.Indexes, 1)+ require.Equal(t, "orders_2024_q1_id_idx", q1.Indexes[0].Name)}// compositeWalkThroughMetadata has an adversarial name pair (aa_nested sorts@@ -271,25 +183,10 @@ func compositeWalkThroughMetadata() *metadatapb.DatabaseSchemaMetadata {}}-func TestWalkThroughLoadsCompositeTypes(t *testing.T) {- meta := compositeWalkThroughMetadata()-- cat := catalog.New()- err := loadWalkThroughCatalog(context.Background(), cat, meta)- require.NoError(t, err)-- require.NotNil(t, cat.GetRelation("public", "zz_base"), "composite type must be installed")- require.NotNil(t, cat.GetRelation("public", "aa_nested"), "nested composite type must be installed")- arrayOnly := cat.GetRelation("public", "aa_array_only")- require.NotNil(t, arrayOnly, "array-only referencing composite must be installed")- require.Len(t, arrayOnly.Columns, 1, "array-only composite must install with its real attribute")- require.NotNil(t, cat.GetRelation("public", "users"), "table using a composite column must install")-}-func TestWalkThroughCompositeFallbackPreservesAttributeNames(t *testing.T) {- // A domain-typed attribute cannot install (domains are not loader- // objects), forcing the pseudo fallback — which must keep attribute- // names so later DDL targeting them still resolves.+ // LoadMetadata does not install domains, so a domain-typed attribute makes+ // the composite stand in — which must keep attribute names so later DDL+ // targeting them still resolves.meta := &metadatapb.DatabaseSchemaMetadata{Schemas: []*metadatapb.SchemaMetadata{{@@ -308,7 +205,7 @@ func TestWalkThroughCompositeFallbackPreservesAttributeNames(t *testing.T) {}cat := catalog.New()- err := loadWalkThroughCatalog(context.Background(), cat, meta)+ _, err := cat.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})require.NoError(t, err)rel := cat.GetRelation("public", "with_domain")@@ -326,7 +223,7 @@ func TestWalkThroughCompositeFallbackPreservesAttributeNames(t *testing.T) {}// A rename keeps the attribute number, so the renamed attribute of a- // degraded composite must also keep its real previous type.+ // stood-in composite must also keep its real previous type.renameResults, err := catAfter.Exec(`ALTER TYPE public.with_domain RENAME ATTRIBUTE p TO q;`, &catalog.ExecOptions{ContinueOnError: true})require.NoError(t, err)for _, r := range renameResults {@@ -373,7 +270,7 @@ func TestWalkThroughDropReaddAttributeReadsCatalogType(t *testing.T) {}cat := catalog.New()- err := loadWalkThroughCatalog(context.Background(), cat, meta)+ _, err := cat.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})require.NoError(t, err)catAfter := cat.Clone()@@ -412,7 +309,7 @@ func TestWalkThroughAppliesCompositeTypeChanges(t *testing.T) {meta := compositeWalkThroughMetadata()catBefore := catalog.New()- err := loadWalkThroughCatalog(context.Background(), catBefore, meta)+ _, err := catBefore.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})require.NoError(t, err)catAfter := catBefore.Clone()diff --git a/backend/plugin/schema/pg/walk_through_loader.go b/backend/plugin/schema/pg/walk_through_loader.godeleted file mode 100644index aa506e661a..0000000000--- a/backend/plugin/schema/pg/walk_through_loader.go+++ /dev/null@@ -1,1497 +0,0 @@-//nolint:unused-package pg--import (- "cmp"- "context"- "fmt"- "regexp"- "slices"- "strings"-- "github.com/pkg/errors"-- metadatapb "github.com/bytebase/omni/metadata"- "github.com/bytebase/omni/pg/ast"- "github.com/bytebase/omni/pg/catalog"- omniparser "github.com/bytebase/omni/pg/parser"-)--// loadWalkThroughCatalog installs every schema object from DatabaseSchemaMetadata-// into the omni catalog in dependency-topological order with pseudo fallback.-// Indexes and constraints are installed after their parent tables/views.-func loadWalkThroughCatalog(ctx context.Context, cat *catalog.Catalog, meta *metadatapb.DatabaseSchemaMetadata) error {- if cat == nil {- return errors.New("loadWalkThroughCatalog: nil catalog")- }- if meta == nil {- return nil- }-- objects := wtCollectObjects(meta)- sorted := wtTopoSort(objects)- for _, obj := range sorted {- if err := ctx.Err(); err != nil {- return err- }- wtInstallOne(cat, obj)- }- return nil-}--// wtObjectKind identifies the kind of a walk-through loader object.-type wtObjectKind int--const (- kindWTSchema wtObjectKind = iota- kindWTEnum- kindWTComposite- kindWTSequence- kindWTTable- kindWTView- kindWTMatView- kindWTFunction- kindWTIndex- kindWTConstraint-)--// wtObjectEntry is one unit of work for the walk-through loader.-type wtObjectEntry struct {- kind wtObjectKind- schema string- name string-- // Exactly one of the following is set based on kind.- enumMeta *metadatapb.EnumTypeMetadata- compositeMeta *metadatapb.CompositeTypeMetadata- seqMeta *metadatapb.SequenceMetadata- tableMeta *metadatapb.TableMetadata- viewMeta *metadatapb.ViewMetadata- matViewMeta *metadatapb.MaterializedViewMetadata- funcMeta *metadatapb.FunctionMetadata- idxMeta *metadatapb.IndexMetadata- // For kindWTConstraint: the parent table metadata.- constraintTable *metadatapb.TableMetadata- // For kindWTConstraint, kindWTIndex: the parent relation name.- parentName string-}--func (e *wtObjectEntry) key() string {- switch e.kind {- case kindWTSchema:- return "schema:" + e.schema- case kindWTEnum, kindWTComposite:- return "type:" + e.schema + "." + e.name- case kindWTSequence:- return "seq:" + e.schema + "." + e.name- case kindWTTable, kindWTView, kindWTMatView:- return "rel:" + e.schema + "." + e.name- case kindWTFunction:- return "func:" + e.schema + "." + e.name + "|" + wtFuncSigKey(e.funcMeta)- case kindWTIndex:- return "idx:" + e.schema + "." + e.parentName + "." + e.name- case kindWTConstraint:- return "con:" + e.schema + "." + e.parentName + "." + e.name- }- return "unknown:" + e.schema + "." + e.name-}--func (e *wtObjectEntry) sortKey() string {- base := e.schema + "\x00" + e.name + "\x00" + wtKindLabel(e.kind)- if e.kind == kindWTFunction {- base += "\x00" + wtFuncSigKey(e.funcMeta)- }- if e.kind == kindWTIndex || e.kind == kindWTConstraint {- base += "\x00" + e.parentName- }- return base-}--func wtFuncSigKey(fn *metadatapb.FunctionMetadata) string {- if fn == nil {- return ""- }- if fn.Signature != "" {- return fn.Signature- }- return fn.Definition-}--func wtKindLabel(k wtObjectKind) string {- switch k {- case kindWTSchema:- return "0schema"- case kindWTEnum:- return "1enum"- case kindWTComposite:- return "1composite"- case kindWTSequence:- return "2seq"- case kindWTTable:- return "3table"- case kindWTView:- return "4view"- case kindWTMatView:- return "5matview"- case kindWTFunction:- return "6function"- case kindWTIndex:- return "7index"- case kindWTConstraint:- return "8constraint"- }- return "9unknown"-}--// wtCollectObjects flattens DatabaseSchemaMetadata into wtObjectEntry values.-func wtCollectObjects(meta *metadatapb.DatabaseSchemaMetadata) []*wtObjectEntry {- var out []*wtObjectEntry- for _, sm := range meta.Schemas {- if sm.Name == "" {- continue- }- out = append(out, &wtObjectEntry{- kind: kindWTSchema,- schema: sm.Name,- name: sm.Name,- })- for _, enum := range sm.EnumTypes {- out = append(out, &wtObjectEntry{- kind: kindWTEnum,- schema: sm.Name,- name: enum.Name,- enumMeta: enum,- })- }- for _, composite := range sm.CompositeTypes {- out = append(out, &wtObjectEntry{- kind: kindWTComposite,- schema: sm.Name,- name: composite.Name,- compositeMeta: composite,- })- }- for _, seq := range sm.Sequences {- if wtIsIdentitySequence(seq, sm.Tables) {- continue- }- out = append(out, &wtObjectEntry{- kind: kindWTSequence,- schema: sm.Name,- name: seq.Name,- seqMeta: seq,- })- }- for _, tbl := range sm.Tables {- out = append(out, &wtObjectEntry{- kind: kindWTTable,- schema: sm.Name,- name: tbl.Name,- tableMeta: tbl,- })- for _, idx := range tbl.Indexes {- out = append(out, &wtObjectEntry{- kind: kindWTIndex,- schema: sm.Name,- name: idx.Name,- parentName: tbl.Name,- idxMeta: idx,- })- }- for _, fk := range tbl.ForeignKeys {- out = append(out, &wtObjectEntry{- kind: kindWTConstraint,- schema: sm.Name,- name: fk.Name,- parentName: tbl.Name,- constraintTable: tbl,- })- }- for _, chk := range tbl.CheckConstraints {- out = append(out, &wtObjectEntry{- kind: kindWTConstraint,- schema: sm.Name,- name: chk.Name,- parentName: tbl.Name,- constraintTable: tbl,- })- }- for _, exc := range tbl.ExcludeConstraints {- out = append(out, &wtObjectEntry{- kind: kindWTConstraint,- schema: sm.Name,- name: exc.Name,- parentName: tbl.Name,- constraintTable: tbl,- })- }- }- for _, view := range sm.Views {- out = append(out, &wtObjectEntry{- kind: kindWTView,- schema: sm.Name,- name: view.Name,- viewMeta: view,- })- }- for _, mv := range sm.MaterializedViews {- out = append(out, &wtObjectEntry{- kind: kindWTMatView,- schema: sm.Name,- name: mv.Name,- matViewMeta: mv,- })- for _, idx := range mv.Indexes {- out = append(out, &wtObjectEntry{- kind: kindWTIndex,- schema: sm.Name,- name: idx.Name,- parentName: mv.Name,- idxMeta: idx,- })- }- }- for _, fn := range sm.Functions {- out = append(out, &wtObjectEntry{- kind: kindWTFunction,- schema: sm.Name,- name: fn.Name,- funcMeta: fn,- })- }- for _, proc := range sm.Procedures {- out = append(out, &wtObjectEntry{- kind: kindWTFunction,- schema: sm.Name,- name: proc.Name,- funcMeta: &metadatapb.FunctionMetadata{- Name: proc.Name,- Definition: proc.Definition,- Signature: proc.Signature,- },- })- }- }- return out-}--// wtTopoSort orders objects by dependency using Tarjan SCC.-func wtTopoSort(objects []*wtObjectEntry) []*wtObjectEntry {- if len(objects) == 0 {- return nil- }- index := make(map[string]*wtObjectEntry, len(objects))- for _, obj := range objects {- index[obj.key()] = obj- }-- edges := wtBuildEdges(objects, index)- sccs := wtTarjanSCC(objects, edges)-- sccOf := make(map[string]int, len(objects))- for i, scc := range sccs {- for _, obj := range scc {- sccOf[obj.key()] = i- }- }-- condensedEdges := make([][]int, len(sccs))- inDegree := make([]int, len(sccs))- seenEdge := make(map[[2]int]bool)- for src, dsts := range edges {- srcSCC, ok := sccOf[src]- if !ok {- continue- }- for _, dst := range dsts {- dstSCC, ok2 := sccOf[dst]- if !ok2 || dstSCC == srcSCC {- continue- }- edge := [2]int{dstSCC, srcSCC}- if seenEdge[edge] {- continue- }- seenEdge[edge] = true- condensedEdges[dstSCC] = append(condensedEdges[dstSCC], srcSCC)- inDegree[srcSCC]++- }- }-- ready := make([]int, 0)- for i := range sccs {- if inDegree[i] == 0 {- ready = append(ready, i)- }- }- wtSortSCCsByMin(ready, sccs)-- var flat []*wtObjectEntry- for len(ready) > 0 {- next := ready[0]- ready = ready[1:]- flat = append(flat, wtSortedSCCMembers(sccs[next])...)- for _, nb := range condensedEdges[next] {- inDegree[nb]--- if inDegree[nb] == 0 {- ready = append(ready, nb)- }- }- wtSortSCCsByMin(ready, sccs)- }-- if len(flat) != len(objects) {- emitted := make(map[string]bool, len(flat))- for _, o := range flat {- emitted[o.key()] = true- }- var missed []*wtObjectEntry- for _, o := range objects {- if !emitted[o.key()] {- missed = append(missed, o)- }- }- slices.SortStableFunc(missed, func(a, b *wtObjectEntry) int {- return cmp.Compare(a.sortKey(), b.sortKey())- })- flat = append(flat, missed...)- }- return flat-}--// wtBuildEdges computes dependency edges for walk-through objects.-func wtBuildEdges(objects []*wtObjectEntry, index map[string]*wtObjectEntry) map[string][]string {- edges := make(map[string][]string, len(objects))- for _, obj := range objects {- var deps []string-- // Every non-schema object depends on its schema.- if obj.kind != kindWTSchema {- if key := "schema:" + obj.schema; index[key] != nil {- deps = append(deps, key)- }- }-- switch obj.kind {- case kindWTTable:- for _, col := range obj.tableMeta.Columns {- for _, ref := range wtExtractUserTypeRefs(col.Type) {- if k := "type:" + ref.Schema + "." + ref.Name; index[k] != nil {- deps = append(deps, k)- }- }- // Sequence dependency from nextval('schema.seq'::regclass).- if col.Default != "" {- if seqKey := wtExtractSeqRef(col.Default, obj.schema, index); seqKey != "" {- deps = append(deps, seqKey)- }- }- }- case kindWTView:- for _, dep := range obj.viewMeta.DependencyColumns {- if k := "rel:" + dep.Schema + "." + dep.Table; index[k] != nil {- deps = append(deps, k)- }- }- case kindWTMatView:- for _, dep := range obj.matViewMeta.DependencyColumns {- if k := "rel:" + dep.Schema + "." + dep.Table; index[k] != nil {- deps = append(deps, k)- }- }- case kindWTFunction:- argTypes, _ := wtParseFuncArgTypes(obj.funcMeta.Signature)- for _, at := range argTypes {- for _, ref := range wtExtractUserTypeRefs(at) {- if k := "type:" + ref.Schema + "." + ref.Name; index[k] != nil {- deps = append(deps, k)- }- }- }- case kindWTIndex:- if k := "rel:" + obj.schema + "." + obj.parentName; index[k] != nil {- deps = append(deps, k)- }- case kindWTConstraint:- if k := "rel:" + obj.schema + "." + obj.parentName; index[k] != nil {- deps = append(deps, k)- }- if obj.constraintTable != nil {- for _, fk := range obj.constraintTable.ForeignKeys {- if fk.Name == obj.name {- refSchema := fk.ReferencedSchema- if refSchema == "" {- refSchema = obj.schema- }- if k := "rel:" + refSchema + "." + fk.ReferencedTable; index[k] != nil {- deps = append(deps, k)- }- // FK depends on referenced table's PK/Unique indexes.- for _, o := range objects {- if o.kind == kindWTIndex && o.schema == refSchema &&- o.parentName == fk.ReferencedTable &&- (o.idxMeta.Primary || (o.idxMeta.IsConstraint && o.idxMeta.Unique)) {- if index[o.key()] != nil {- deps = append(deps, o.key())- }- }- }- break- }- }- }- case kindWTComposite:- for _, attribute := range obj.compositeMeta.Attributes {- for _, ref := range wtExtractUserTypeRefs(attribute.Type) {- if k := "type:" + ref.Schema + "." + ref.Name; index[k] != nil {- deps = append(deps, k)- }- // An attribute may be typed with a table/view row type.- if k := "rel:" + ref.Schema + "." + ref.Name; index[k] != nil {- deps = append(deps, k)- }- }- }- default:- // kindWTSchema, kindWTEnum, kindWTSequence have no additional deps.- }-- if len(deps) > 0 {- edges[obj.key()] = wtDedupStrings(deps)- }- }- return edges-}--// wtTarjanSCC runs Tarjan's SCC algorithm over walk-through objects.-func wtTarjanSCC(objects []*wtObjectEntry, edges map[string][]string) [][]*wtObjectEntry {- type state struct {- index, low int- onStack bool- }- st := make(map[string]*state, len(objects))- byKey := make(map[string]*wtObjectEntry, len(objects))- for _, obj := range objects {- byKey[obj.key()] = obj- }-- keys := make([]string, 0, len(objects))- for _, obj := range objects {- keys = append(keys, obj.key())- }- slices.Sort(keys)-- var (- idx int- stack []string- result [][]*wtObjectEntry- )-- var strongconnect func(v string)- strongconnect = func(v string) {- st[v] = &state{index: idx, low: idx, onStack: true}- idx++- stack = append(stack, v)- for _, w := range edges[v] {- if _, ok := byKey[w]; !ok {- continue- }- if _, seen := st[w]; !seen {- strongconnect(w)- st[v].low = min(st[v].low, st[w].low)- } else if st[w].onStack {- st[v].low = min(st[v].low, st[w].index)- }- }- if st[v].low == st[v].index {- var scc []*wtObjectEntry- for {- w := stack[len(stack)-1]- stack = stack[:len(stack)-1]- st[w].onStack = false- scc = append(scc, byKey[w])- if w == v {- break- }- }- result = append(result, scc)- }- }-- for _, k := range keys {- if _, seen := st[k]; !seen {- strongconnect(k)- }- }- return result-}--func wtSortedSCCMembers(scc []*wtObjectEntry) []*wtObjectEntry {- out := make([]*wtObjectEntry, len(scc))- copy(out, scc)- slices.SortStableFunc(out, func(a, b *wtObjectEntry) int {- return cmp.Compare(a.sortKey(), b.sortKey())- })- return out-}--func wtSortSCCsByMin(indices []int, sccs [][]*wtObjectEntry) {- slices.SortStableFunc(indices, func(a, b int) int {- return cmp.Compare(wtMinSortKey(sccs[a]), wtMinSortKey(sccs[b]))- })-}--func wtMinSortKey(scc []*wtObjectEntry) string {- if len(scc) == 0 {- return ""- }- best := scc[0].sortKey()- for _, o := range scc[1:] {- if k := o.sortKey(); k < best {- best = k- }- }- return best-}--func wtDedupStrings(in []string) []string {- if len(in) < 2 {- return in- }- seen := make(map[string]bool, len(in))- out := in[:0]- for _, s := range in {- if !seen[s] {- seen[s] = true- out = append(out, s)- }- }- return out-}--// wtInstallOne attempts real install, falls back to pseudo for tables/views/matviews.-func wtInstallOne(cat *catalog.Catalog, obj *wtObjectEntry) {- if err := wtInstallReal(cat, obj); err == nil {- return- }- // Only pseudo-fallback for relations; other kinds are left uninstalled on failure.- switch obj.kind {- case kindWTTable:- cols := wtColNames(obj.tableMeta)- _ = cat.DefineRelation(wtPseudoTableStmt(obj.schema, obj.name, cols), 'r')- case kindWTView:- cols := wtViewColNames(obj.viewMeta)- if stmt, err := wtPseudoViewStmt(obj.schema, obj.name, cols); err == nil {- _ = cat.DefineView(stmt)- }- case kindWTMatView:- cols := wtMatViewColNames(obj.matViewMeta)- if stmt, err := wtPseudoMatViewStmt(obj.schema, obj.name, cols); err == nil {- _ = cat.ExecCreateTableAs(stmt)- }- case kindWTComposite:- // Name-preserving, text-backed fallback (mirroring the table pseudo)- // so later DDL targeting an existing attribute still resolves.- items := make([]ast.Node, 0, len(obj.compositeMeta.Attributes))- for _, attribute := range obj.compositeMeta.Attributes {- if attribute.Name == "" {- continue- }- items = append(items, &ast.ColumnDef{- Colname: attribute.Name,- TypeName: wtPseudoTextTypeName(),- })- }- _ = cat.DefineCompositeType(&ast.CompositeTypeStmt{- Typevar: &ast.RangeVar{Schemaname: obj.schema, Relname: obj.name},- Coldeflist: &ast.List{Items: items},- })- default:- // No pseudo form for schemas, enums, sequences, functions, indexes, constraints.- }-}--// wtInstallReal performs the real catalog install for one object.-func wtInstallReal(cat *catalog.Catalog, obj *wtObjectEntry) error {- switch obj.kind {- case kindWTSchema:- err := cat.CreateSchemaCommand(&ast.CreateSchemaStmt{Schemaname: obj.schema})- var cErr *catalog.Error- if errors.As(err, &cErr) && cErr.Code == catalog.CodeDuplicateSchema {- return nil- }- return err-- case kindWTEnum:- vals := make([]ast.Node, 0, len(obj.enumMeta.Values))- for _, v := range obj.enumMeta.Values {- vals = append(vals, &ast.String{Str: v})- }- return cat.DefineEnum(&ast.CreateEnumStmt{- TypeName: wtQualifiedList(obj.schema, obj.name),- Vals: &ast.List{Items: vals},- })-- case kindWTComposite:- cols := make([]ast.Node, 0, len(obj.compositeMeta.Attributes))- for _, attribute := range obj.compositeMeta.Attributes {- if attribute.Name == "" {- continue- }- tn, err := wtTypeNameFromString(attribute.Type)- if err != nil {- return errors.Wrapf(err, "attribute %q", attribute.Name)- }- cols = append(cols, &ast.ColumnDef{- Colname: attribute.Name,- TypeName: tn,- CollClause: collateClauseFromReference(attribute.Collation),- })- }- return cat.DefineCompositeType(&ast.CompositeTypeStmt{- Typevar: &ast.RangeVar{Schemaname: obj.schema, Relname: obj.name},- Coldeflist: &ast.List{Items: cols},- })-- case kindWTSequence:- return wtInstallSequence(cat, obj)-- case kindWTTable:- return wtInstallTable(cat, obj)-- case kindWTView:- return wtInstallView(cat, obj)-- case kindWTMatView:- return wtInstallMatView(cat, obj)-- case kindWTFunction:- return wtInstallFunction(cat, obj)-- case kindWTIndex:- if obj.idxMeta.Primary {- return cat.AddConstraint(obj.schema, obj.parentName, catalog.ConstraintDef{- Name: obj.idxMeta.Name,- Type: catalog.ConstraintPK,- Columns: wtUnquoteColumns(obj.idxMeta.Expressions),- })- }- if obj.idxMeta.IsConstraint && obj.idxMeta.Unique {- return cat.AddConstraint(obj.schema, obj.parentName, catalog.ConstraintDef{- Name: obj.idxMeta.Name,- Type: catalog.ConstraintUnique,- Columns: wtUnquoteColumns(obj.idxMeta.Expressions),- })- }- return wtInstallIndex(cat, obj)-- case kindWTConstraint:- return wtInstallConstraint(cat, obj)- }- return errors.Errorf("unknown object kind %d", obj.kind)-}--func wtInstallSequence(cat *catalog.Catalog, obj *wtObjectEntry) error {- seq := obj.seqMeta- if seq == nil {- return errors.New("sequence has no metadata")- }-- stmt := &ast.CreateSeqStmt{- Sequence: &ast.RangeVar{- Schemaname: obj.schema,- Relname: seq.Name,- Relpersistence: 'p',- },- }-- var opts []ast.Node- if seq.Increment != "" && seq.Increment != "0" {- opts = append(opts, &ast.DefElem{Defname: "increment", Arg: &ast.Integer{Ival: wtParseInt64(seq.Increment)}})- }- if seq.MinValue != "" && seq.MinValue != "0" {- opts = append(opts, &ast.DefElem{Defname: "minvalue", Arg: &ast.Integer{Ival: wtParseInt64(seq.MinValue)}})- }- if seq.MaxValue != "" && seq.MaxValue != "0" {- opts = append(opts, &ast.DefElem{Defname: "maxvalue", Arg: &ast.Integer{Ival: wtParseInt64(seq.MaxValue)}})- }- if seq.Start != "" && seq.Start != "0" {- opts = append(opts, &ast.DefElem{Defname: "start", Arg: &ast.Integer{Ival: wtParseInt64(seq.Start)}})- }- if seq.CacheSize != "" && seq.CacheSize != "0" {- opts = append(opts, &ast.DefElem{Defname: "cache", Arg: &ast.Integer{Ival: wtParseInt64(seq.CacheSize)}})- }- if seq.Cycle {- opts = append(opts, &ast.DefElem{Defname: "cycle", Arg: &ast.Boolean{Boolval: true}})- }- if len(opts) > 0 {- stmt.Options = &ast.List{Items: opts}- }-- return cat.DefineSequence(stmt)-}--// wtIsIdentitySequence checks if a sequence is owned by an identity column.-// Identity columns auto-create their sequence during DefineRelation, so-// pre-creating it would cause a duplicate error.-func wtIsIdentitySequence(seq *metadatapb.SequenceMetadata, tables []*metadatapb.TableMetadata) bool {- if seq.OwnerTable == "" || seq.OwnerColumn == "" {- return false- }- for _, tbl := range tables {- if tbl.Name != seq.OwnerTable {- continue- }- for _, col := range tbl.Columns {- if col.Name == seq.OwnerColumn && col.IsIdentity {- return true- }- }- }- return false-}--func wtParseInt64(s string) int64 {- var v int64- _, _ = fmt.Sscanf(s, "%d", &v)- return v-}--func wtInstallTable(cat *catalog.Catalog, obj *wtObjectEntry) error {- items := make([]ast.Node, 0, len(obj.tableMeta.Columns))- for _, col := range obj.tableMeta.Columns {- if col.Name == "" {- continue- }- if col.Type == "" {- return errors.Errorf("column %q: empty type", col.Name)- }- tn, err := wtTypeNameFromString(col.Type)- if err != nil {- return errors.Wrapf(err, "column %q", col.Name)- }- colDef := &ast.ColumnDef{- Colname: col.Name,- TypeName: tn,- IsNotNull: !col.Nullable,- }- if col.Default != "" && col.Generation == nil {- if rawDefault, err := wtParseDefaultExpr(col.Default); err == nil {- colDef.RawDefault = rawDefault- }- }- if col.Generation != nil && col.Generation.Type == metadatapb.GenerationMetadata_TYPE_STORED && col.Generation.Expression != "" {- colDef.Generated = 's'- if genExpr := wtParseExpr(col.Generation.Expression); genExpr != nil {- colDef.Constraints = &ast.List{Items: []ast.Node{- &ast.Constraint{Contype: ast.CONSTR_GENERATED, RawExpr: genExpr},- }}- }- }- if col.IsIdentity {- switch col.IdentityGeneration {- case metadatapb.ColumnMetadata_ALWAYS:- colDef.Identity = 'a'- case metadatapb.ColumnMetadata_BY_DEFAULT:- colDef.Identity = 'd'- default:- colDef.Identity = 'd'- }- }- items = append(items, colDef)- }- return cat.DefineRelation(&ast.CreateStmt{- Relation: &ast.RangeVar{- Schemaname: obj.schema,- Relname: obj.tableMeta.Name,- Relpersistence: 'p',- },- TableElts: &ast.List{Items: items},- }, 'r')-}--func wtInstallView(cat *catalog.Catalog, obj *wtObjectEntry) error {- if obj.viewMeta.Definition == "" {- return errors.New("empty view definition")- }- sel, err := wtParseSelectBody(obj.viewMeta.Definition)- if err != nil {- return errors.Wrapf(err, "view %q", obj.name)- }- return cat.DefineView(&ast.ViewStmt{- View: &ast.RangeVar{- Schemaname: obj.schema,- Relname: obj.name,- Relpersistence: 'p',- },- Query: sel,- })-}--func wtInstallMatView(cat *catalog.Catalog, obj *wtObjectEntry) error {- if obj.matViewMeta.Definition == "" {- return errors.New("empty matview definition")- }- sel, err := wtParseSelectBody(obj.matViewMeta.Definition)- if err != nil {- return errors.Wrapf(err, "matview %q", obj.name)- }- return cat.ExecCreateTableAs(&ast.CreateTableAsStmt{- Query: sel,- Objtype: ast.OBJECT_MATVIEW,- Into: &ast.IntoClause{- Rel: &ast.RangeVar{- Schemaname: obj.schema,- Relname: obj.name,- Relpersistence: 'p',- },- },- })-}--func wtInstallFunction(cat *catalog.Catalog, obj *wtObjectEntry) error {- fn := obj.funcMeta- if fn.Definition != "" {- nodes, err := omniparser.Parse(fn.Definition)- if err == nil && nodes != nil && len(nodes.Items) == 1 {- node := nodes.Items[0]- if raw, ok := node.(*ast.RawStmt); ok {- node = raw.Stmt- }- if parsed, ok := node.(*ast.CreateFunctionStmt); ok {- return cat.CreateFunctionStmt(parsed)- }- }- }- // Fallback: build from signature.- argTypes, err := wtParseFuncArgTypes(fn.Signature)- if err != nil {- return errors.Wrapf(err, "function %q signature %q", fn.Name, fn.Signature)- }- params := make([]ast.Node, 0, len(argTypes))- for i, at := range argTypes {- tn, err := wtTypeNameFromString(at)- if err != nil {- return errors.Wrapf(err, "function %q arg %d", fn.Name, i)- }- params = append(params, &ast.FunctionParameter{- ArgType: tn,- Mode: ast.FUNC_PARAM_IN,- })- }- stmt := &ast.CreateFunctionStmt{- Funcname: wtQualifiedList(obj.schema, fn.Name),- ReturnType: wtPseudoTextTypeName(),- }- if len(params) > 0 {- stmt.Parameters = &ast.List{Items: params}- }- stmt.Options = &ast.List{Items: []ast.Node{- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},- &ast.DefElem{- Defname: "as",- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT NULL::text"}}},- },- }}- return cat.CreateFunctionStmt(stmt)-}--func wtInstallIndex(cat *catalog.Catalog, obj *wtObjectEntry) error {- idx := obj.idxMeta- if idx.Name == "" {- return errors.New("index has no name")- }- params := make([]ast.Node, 0, len(idx.Expressions))- for i, expr := range idx.Expressions {- if expr == "" {- continue- }- elem := &ast.IndexElem{}- if i < len(idx.Descending) && idx.Descending[i] {- elem.Ordering = ast.SORTBY_DESC- }- bare := wtUnquoteIdent(expr)- if wtIsExpressionIndex(bare) {- if parsed := wtParseExpr(expr); parsed != nil {- elem.Expr = parsed- } else {- elem.Name = bare- }- } else {- elem.Name = bare- }- params = append(params, elem)- }- return cat.DefineIndex(&ast.IndexStmt{- Idxname: idx.Name,- Relation: &ast.RangeVar{Schemaname: obj.schema, Relname: obj.parentName},- AccessMethod: idx.Type,- IndexParams: &ast.List{Items: params},- Unique: idx.Unique,- Primary: idx.Primary,- })-}--func wtInstallConstraint(cat *catalog.Catalog, obj *wtObjectEntry) error {- tbl := obj.constraintTable- if tbl == nil {- return errors.New("constraint has no parent table")- }-- // Find which constraint this entry represents.- for _, fk := range tbl.ForeignKeys {- if fk.Name != obj.name {- continue- }- refSchema := fk.ReferencedSchema- if refSchema == "" {- refSchema = obj.schema- }- def := catalog.ConstraintDef{- Name: fk.Name,- Type: catalog.ConstraintFK,- Columns: fk.Columns,- RefSchema: refSchema,- RefTable: fk.ReferencedTable,- RefColumns: fk.ReferencedColumns,- FKUpdAction: wtFKActionByte(fk.OnUpdate),- FKDelAction: wtFKActionByte(fk.OnDelete),- FKMatchType: wtFKMatchByte(fk.MatchType),- }- return cat.AddConstraint(obj.schema, obj.parentName, def)- }- for _, chk := range tbl.CheckConstraints {- if chk.Name != obj.name {- continue- }- return cat.AddConstraint(obj.schema, obj.parentName, catalog.ConstraintDef{- Name: chk.Name,- Type: catalog.ConstraintCheck,- CheckExpr: chk.Expression,- })- }- for _, exc := range tbl.ExcludeConstraints {- if exc.Name != obj.name {- continue- }- return cat.AddConstraint(obj.schema, obj.parentName, catalog.ConstraintDef{- Name: exc.Name,- Type: catalog.ConstraintExclude,- CheckExpr: exc.Expression,- })- }- return errors.Errorf("constraint %q not found in table %q", obj.name, obj.parentName)-}--// ---- pseudo forms ------func wtPseudoTableStmt(schema, name string, cols []string) *ast.CreateStmt {- items := make([]ast.Node, 0, len(cols))- for _, col := range cols {- if col == "" {- continue- }- items = append(items, &ast.ColumnDef{- Colname: col,- TypeName: wtPseudoTextTypeName(),- })- }- return &ast.CreateStmt{- Relation: &ast.RangeVar{- Schemaname: schema,- Relname: name,- Relpersistence: 'p',- },- TableElts: &ast.List{Items: items},- }-}--func wtPseudoViewStmt(schema, name string, cols []string) (*ast.ViewStmt, error) {- sel, err := wtPseudoConstantSelect(cols)- if err != nil {- return nil, err- }- return &ast.ViewStmt{- View: &ast.RangeVar{- Schemaname: schema,- Relname: name,- Relpersistence: 'p',- },- Query: sel,- }, nil-}--func wtPseudoMatViewStmt(schema, name string, cols []string) (*ast.CreateTableAsStmt, error) {- sel, err := wtPseudoConstantSelect(cols)- if err != nil {- return nil, err- }- return &ast.CreateTableAsStmt{- Query: sel,- Objtype: ast.OBJECT_MATVIEW,- Into: &ast.IntoClause{- Rel: &ast.RangeVar{- Schemaname: schema,- Relname: name,- Relpersistence: 'p',- },- },- }, nil-}--func wtPseudoConstantSelect(cols []string) (*ast.SelectStmt, error) {- var targets []string- for _, col := range cols {- if col == "" {- continue- }- targets = append(targets, "NULL::text AS "+wtQuoteIdent(col))- }- if len(targets) == 0 {- targets = []string{"NULL::text"}- }- sql := "SELECT " + strings.Join(targets, ", ")- nodes, err := omniparser.Parse(sql)- if err != nil {- return nil, errors.Wrap(err, "parse pseudo constant select")- }- if nodes == nil || len(nodes.Items) != 1 {- return nil, errors.New("pseudo constant select: expected 1 statement")- }- node := nodes.Items[0]- if raw, ok := node.(*ast.RawStmt); ok {- node = raw.Stmt- }- sel, ok := node.(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("pseudo constant select: expected SelectStmt, got %T", node)- }- return sel, nil-}--// ---- column name helpers ------func wtColNames(tbl *metadatapb.TableMetadata) []string {- if tbl == nil {- return nil- }- seen := make(map[string]bool, len(tbl.Columns))- out := make([]string, 0, len(tbl.Columns))- for _, col := range tbl.Columns {- if col.Name == "" || seen[col.Name] {- continue- }- seen[col.Name] = true- out = append(out, col.Name)- }- return out-}--func wtViewColNames(v *metadatapb.ViewMetadata) []string {- if v == nil {- return nil- }- if len(v.Columns) > 0 {- seen := make(map[string]bool, len(v.Columns))- out := make([]string, 0, len(v.Columns))- for _, col := range v.Columns {- if col.Name == "" || seen[col.Name] {- continue- }- seen[col.Name] = true- out = append(out, col.Name)- }- return out- }- seen := make(map[string]bool)- var out []string- for _, dc := range v.DependencyColumns {- if dc.Column == "" || seen[dc.Column] {- continue- }- seen[dc.Column] = true- out = append(out, dc.Column)- }- return out-}--func wtMatViewColNames(m *metadatapb.MaterializedViewMetadata) []string {- if m == nil {- return nil- }- seen := make(map[string]bool)- var out []string- for _, dc := range m.DependencyColumns {- if dc.Column == "" || seen[dc.Column] {- continue- }- seen[dc.Column] = true- out = append(out, dc.Column)- }- return out-}--// ---- type parsing helpers ------// wtTypeNameFromString parses a PG type string into *ast.TypeName via-// "SELECT NULL::<type>" to reuse omni's SELECT parse path.-func wtTypeNameFromString(typeStr string) (*ast.TypeName, error) {- nodes, err := omniparser.Parse("SELECT NULL::" + typeStr)- if err != nil {- return nil, errors.Wrapf(err, "parse type %q", typeStr)- }- if nodes == nil || len(nodes.Items) != 1 {- return nil, errors.Errorf("type %q: expected 1 statement", typeStr)- }- node := nodes.Items[0]- if raw, ok := node.(*ast.RawStmt); ok {- node = raw.Stmt- }- sel, ok := node.(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("type %q: expected SelectStmt, got %T", typeStr, node)- }- if sel.TargetList == nil || len(sel.TargetList.Items) != 1 {- return nil, errors.Errorf("type %q: expected 1 target", typeStr)- }- rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)- if !ok {- return nil, errors.Errorf("type %q: expected ResTarget, got %T", typeStr, sel.TargetList.Items[0])- }- cast, ok := rt.Val.(*ast.TypeCast)- if !ok {- return nil, errors.Errorf("type %q: expected TypeCast, got %T", typeStr, rt.Val)- }- return cast.TypeName, nil-}--// wtParseDefaultExpr parses a default expression string and returns the AST node.-func wtParseDefaultExpr(expr string) (ast.Node, error) {- nodes, err := omniparser.Parse("SELECT " + expr)- if err != nil {- return nil, errors.Wrapf(err, "parse default %q", expr)- }- if nodes == nil || len(nodes.Items) != 1 {- return nil, errors.Errorf("default %q: expected 1 statement", expr)- }- node := nodes.Items[0]- if raw, ok := node.(*ast.RawStmt); ok {- node = raw.Stmt- }- sel, ok := node.(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("default %q: expected SelectStmt", expr)- }- if sel.TargetList == nil || len(sel.TargetList.Items) != 1 {- return nil, errors.Errorf("default %q: expected 1 target", expr)- }- rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)- if !ok {- return nil, errors.Errorf("default %q: expected ResTarget", expr)- }- return rt.Val, nil-}--// wtParseSelectBody parses a SQL string into *ast.SelectStmt.-func wtParseSelectBody(sql string) (*ast.SelectStmt, error) {- nodes, err := omniparser.Parse(sql)- if err != nil {- return nil, errors.Wrap(err, "parse")- }- if nodes == nil || len(nodes.Items) != 1 {- return nil, errors.Errorf("expected 1 statement, got %d", len(nodes.Items))- }- node := nodes.Items[0]- if raw, ok := node.(*ast.RawStmt); ok {- node = raw.Stmt- }- sel, ok := node.(*ast.SelectStmt)- if !ok {- return nil, errors.Errorf("expected SelectStmt, got %T", node)- }- return sel, nil-}--// ---- type ref extraction (local, no import from parser/pg) ------type wtUserTypeRef struct {- Schema string- Name string-}--func wtExtractUserTypeRefs(typeStr string) []wtUserTypeRef {- if typeStr == "" {- return nil- }- base := wtStripTypeModifiers(typeStr)- if base == "" || wtIsBuiltinType(base) {- return nil- }- if strings.HasPrefix(base, "_") {- return nil- }- schema, name, ok := wtSplitQualifiedName(base)- if !ok {- return nil- }- if wtIsSystemSchema(schema) {- return nil- }- return []wtUserTypeRef{{Schema: schema, Name: name}}-}--func wtStripTypeModifiers(s string) string {- s = strings.TrimSpace(s)- // Array suffixes wrap the element type: "public.addr[]" refers to- // public.addr. Quoted identifiers end with a quote, so a genuine name- // containing "[]" is never clipped.- for strings.HasSuffix(s, "[]") {- s = strings.TrimSpace(strings.TrimSuffix(s, "[]"))- }- lower := strings.ToLower(s)- for _, sfx := range []string{" with time zone", " without time zone"} {- if strings.HasSuffix(lower, sfx) {- s = s[:len(s)-len(sfx)]- lower = lower[:len(lower)-len(sfx)]- }- }- if i := strings.Index(s, "("); i >= 0 {- s = strings.TrimSpace(s[:i])- }- return s-}--func wtSplitQualifiedName(s string) (schema, name string, ok bool) {- var parts []string- var cur strings.Builder- inQuote := false- for i := 0; i < len(s); i++ {- c := s[i]- switch {- case c == '"':- inQuote = !inQuote- case c == '.' && !inQuote:- parts = append(parts, cur.String())- cur.Reset()- default:- cur.WriteByte(c)- }- }- if inQuote {- return "", "", false- }- parts = append(parts, cur.String())- if len(parts) != 2 {- return "", "", false- }- return parts[0], parts[1], true-}--func wtIsSystemSchema(s string) bool {- switch s {- case "pg_catalog", "pg_toast", "information_schema":- return true- }- return strings.HasPrefix(s, "pg_")-}--func wtIsBuiltinType(s string) bool {- _, ok := wtBuiltinTypes[strings.ToLower(s)]- return ok-}--var wtBuiltinTypes = map[string]struct{}{- "smallint": {}, "integer": {}, "int": {}, "int2": {}, "int4": {}, "int8": {},- "bigint": {}, "decimal": {}, "numeric": {}, "real": {}, "double precision": {},- "float4": {}, "float8": {}, "smallserial": {}, "serial": {}, "bigserial": {},- "money": {}, "character": {}, "character varying": {}, "char": {}, "varchar": {},- "text": {}, "bytea": {}, "bpchar": {}, "name": {}, "bit": {}, "bit varying": {},- "varbit": {}, "date": {}, "time": {}, "timetz": {}, "timestamp": {}, "timestamptz": {},- "interval": {}, "boolean": {}, "bool": {}, "json": {}, "jsonb": {}, "xml": {},- "uuid": {}, "point": {}, "line": {}, "lseg": {}, "box": {}, "path": {},- "polygon": {}, "circle": {}, "cidr": {}, "inet": {}, "macaddr": {}, "macaddr8": {},- "tsvector": {}, "tsquery": {}, "void": {}, "record": {}, "anyelement": {},- "anyarray": {}, "anynonarray": {}, "anyenum": {}, "anyrange": {}, "any": {},- "trigger": {}, "event_trigger": {}, "cstring": {}, "internal": {},- "language_handler": {}, "fdw_handler": {}, "index_am_handler": {}, "tsm_handler": {},- "pg_lsn": {}, "oid": {}, "regclass": {}, "regproc": {}, "regprocedure": {},- "regoper": {}, "regoperator": {}, "regtype": {}, "regconfig": {}, "regdictionary": {},- "regnamespace": {}, "regrole": {},-}--// wtExtractSeqRef extracts the seq key from a nextval('schema.seq'::regclass) default.-// Returns the "seq:schema.name" key if found, empty string otherwise.-var reNextval = regexp.MustCompile(`nextval\('([^']+)'`)--func wtExtractSeqRef(defaultExpr, tableSchema string, index map[string]*wtObjectEntry) string {- m := reNextval.FindStringSubmatch(defaultExpr)- if m == nil {- return ""- }- // m[1] is the content: "schema.seq_name" or "seq_name"- raw := strings.TrimSuffix(m[1], "::regclass")- raw = strings.Trim(raw, "\"")- var seqSchema, seqName string- if dot := strings.Index(raw, "."); dot >= 0 {- seqSchema = strings.Trim(raw[:dot], "\"")- seqName = strings.Trim(raw[dot+1:], "\"")- } else {- seqSchema = tableSchema- seqName = raw- }- k := "seq:" + seqSchema + "." + seqName- if index[k] != nil {- return k- }- return ""-}--// ---- function signature helpers ------func wtParseFuncArgTypes(signature string) ([]string, error) {- open := strings.Index(signature, "(")- if open < 0 {- return nil, nil- }- closeIdx := strings.LastIndex(signature, ")")- if closeIdx < 0 || closeIdx <= open {- return nil, errors.Errorf("signature %q: unbalanced parens", signature)- }- return wtSplitTopLevelCommas(signature[open+1 : closeIdx]), nil-}--func wtSplitTopLevelCommas(s string) []string {- s = strings.TrimSpace(s)- if s == "" {- return nil- }- var parts []string- depth := 0- start := 0- for i := 0; i < len(s); i++ {- switch s[i] {- case '(':- depth++- case ')':- if depth > 0 {- depth--- }- case ',':- if depth == 0 {- parts = append(parts, strings.TrimSpace(s[start:i]))- start = i + 1- }- default:- // other bytes are part of the current token- }- }- parts = append(parts, strings.TrimSpace(s[start:]))- return parts-}--// ---- FK action helpers ------func wtFKActionByte(action string) byte {- switch strings.ToUpper(action) {- case "RESTRICT":- return 'r'- case "CASCADE":- return 'c'- case "SET NULL":- return 'n'- case "SET DEFAULT":- return 'd'- default:- return 'a' // NO ACTION- }-}--func wtFKMatchByte(match string) byte {- switch strings.ToUpper(match) {- case "FULL":- return 'f'- case "PARTIAL":- return 'p'- default:- return 's' // SIMPLE- }-}--// ---- small AST helpers ------func wtQualifiedList(schema, name string) *ast.List {- items := make([]ast.Node, 0, 2)- if schema != "" {- items = append(items, &ast.String{Str: schema})- }- items = append(items, &ast.String{Str: name})- return &ast.List{Items: items}-}--func wtPseudoTextTypeName() *ast.TypeName {- return &ast.TypeName{- Names: &ast.List{Items: []ast.Node{&ast.String{Str: "text"}}},- Typemod: -1,- }-}--func wtQuoteIdent(name string) string {- return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`-}--func wtIsExpressionIndex(expr string) bool {- return strings.ContainsAny(expr, "( ") || strings.Contains(expr, "::")-}--func wtUnquoteIdent(s string) string {- if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {- return strings.ReplaceAll(s[1:len(s)-1], `""`, `"`)- }- return s-}--func wtUnquoteColumns(cols []string) []string {- out := make([]string, len(cols))- for i, c := range cols {- out[i] = wtUnquoteIdent(c)- }- return out-}--func wtParseExpr(expr string) ast.Node {- stmts, err := omniparser.Parse("SELECT (" + expr + ")")- if err != nil || stmts == nil || len(stmts.Items) == 0 {- return nil- }- raw, ok := stmts.Items[0].(*ast.RawStmt)- if !ok {- return nil- }- sel, ok := raw.Stmt.(*ast.SelectStmt)- if !ok || sel.TargetList == nil || len(sel.TargetList.Items) == 0 {- return nil- }- rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)- if !ok {- return nil- }- return rt.Val-}--// collateClauseFromReference converts a stored emit-ready collation reference-// (quoted as needed, schema-qualified outside pg_catalog, e.g. `"C"` or-// `locale.en_us`) into an AST collate clause. Returns nil for no collation.-func collateClauseFromReference(reference string) *ast.CollateClause {- if reference == "" {- return nil- }- var items []ast.Node- if schemaName, collationName, ok := parseQualifiedTypeIdent(reference); ok {- items = []ast.Node{&ast.String{Str: schemaName}, &ast.String{Str: collationName}}- } else {- name := strings.Trim(reference, `"`)- name = strings.ReplaceAll(name, `""`, `"`)- items = []ast.Node{&ast.String{Str: name}}- }- return &ast.CollateClause{Collname: &ast.List{Items: items}}-}diff --git a/backend/plugin/schema/pg/walk_through_test.go b/backend/plugin/schema/pg/walk_through_test.goindex 58a25df9eb..55c853c079 100644--- a/backend/plugin/schema/pg/walk_through_test.go+++ b/backend/plugin/schema/pg/walk_through_test.go@@ -1,12 +1,14 @@package pgimport (+ "context""io""os""path/filepath""testing"metadatapb "github.com/bytebase/omni/metadata"+ "github.com/bytebase/omni/pg/catalog""github.com/google/go-cmp/cmp""github.com/stretchr/testify/require""google.golang.org/protobuf/proto"@@ -103,7 +105,7 @@ func TestWalkThrough(t *testing.T) {stmts, _ := sm.GetStatementsForChecks(storepb.Engine_POSTGRES, test.Statement)asts := base.ExtractASTs(stmts)- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)if test.Advice != nil {require.NotNil(t, advice)require.Equal(t, test.Advice.Code, advice.Code)@@ -207,7 +209,7 @@ func TestWalkThroughANTLR(t *testing.T) {asts := base.ExtractASTs(stmts)// Call WalkThrough with AST- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)if advice != nil {// Compare the advice fieldsif test.Advice != nil {@@ -323,9 +325,9 @@ func TestWalkThroughSearchPathState(t *testing.T) {state := newSearchPathTestState(test.searchPath)stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, test.sql)require.NoError(t, err)- ctx := test.session- ctx.RawSQL = test.sql- advice := WalkThroughWithContext(ctx, state, base.ExtractASTs(stmts))+ wtCtx := test.session+ wtCtx.RawSQL = test.sql+ advice := WalkThroughWithContext(context.Background(), wtCtx, state, base.ExtractASTs(stmts))require.Nil(t, advice)test.assert(t, state)})@@ -369,3 +371,86 @@ func newSearchPathTestTable(name string) *metadatapb.TableMetadata {},}}++// TestClone_SearchPath tests Clone preserves search path and session user.+func TestClone_SearchPath(t *testing.T) {+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "postgres",+ Schemas: []*metadatapb.SchemaMetadata{+ {Name: "public"},+ {Name: "alice"},+ },+ }++ catBefore := catalog.New()+ catBefore.SetSessionUser("alice")+ catBefore.SetSearchPath([]string{"$user", "public"})+ _, err := catBefore.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})+ require.NoError(t, err)++ catAfter := catBefore.Clone()++ // CREATE TABLE without schema should use search path ($user → alice)+ _, err = catAfter.Exec(`CREATE TABLE my_table (id int);`, nil)+ require.NoError(t, err)++ // Table should be in alice schema on clone+ require.NotNil(t, catAfter.GetRelation("alice", "my_table"))++ // Original should NOT have it+ require.Nil(t, catBefore.GetRelation("alice", "my_table"))++ // Diff should show the new table+ diff := catalog.Diff(catBefore, catAfter)+ require.False(t, diff.IsEmpty())++ foundAliceTable := false+ for _, rel := range diff.Relations {+ if rel.SchemaName == "alice" && rel.Name == "my_table" && rel.Action == catalog.DiffAdd {+ foundAliceTable = true+ }+ }+ require.True(t, foundAliceTable, "diff should show alice.my_table as added")+}++// TestClone_WalkThroughFunction tests the actual WalkThroughWithContext function+// using Clone produces correct FinalMetadata.+func TestClone_WalkThroughFunction(t *testing.T) {+ meta := &metadatapb.DatabaseSchemaMetadata{+ Name: "postgres",+ Schemas: []*metadatapb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*metadatapb.TableMetadata{+ {+ Name: "test",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "integer", Position: 1},+ {Name: "name", Type: "text", Position: 2, Nullable: true},+ },+ Indexes: []*metadatapb.IndexMetadata{+ {Name: "test_pkey", Expressions: []string{"id"}, Unique: true, Primary: true},+ },+ },+ },+ },+ },+ }++ state := model.NewDatabaseMetadata(meta, nil, nil, storepb.Engine_POSTGRES, true)+ wtCtx := schema.WalkThroughContext{+ RawSQL: `CREATE TABLE public.new_table (id int PRIMARY KEY, val text);`,+ }++ advice := WalkThroughWithContext(context.Background(), wtCtx, state, nil)+ require.Nil(t, advice, "walk-through should succeed")++ // Check FinalMetadata has the new table+ newTbl := state.GetSchemaMetadata("public").GetTable("new_table")+ require.NotNil(t, newTbl, "new_table should exist in FinalMetadata")++ // Check original table is preserved+ origTbl := state.GetSchemaMetadata("public").GetTable("test")+ require.NotNil(t, origTbl, "test table should still exist")+ require.Equal(t, 2, len(origTbl.GetProto().Columns))+}diff --git a/backend/plugin/schema/schema.go b/backend/plugin/schema/schema.goindex 71c62d5c11..d7128afb7e 100644--- a/backend/plugin/schema/schema.go+++ b/backend/plugin/schema/schema.go@@ -1,6 +1,7 @@package schemaimport (+ "context""fmt""strings""sync"@@ -65,7 +66,7 @@ type sdlDropAdvices func(userSDLText string, currentSchema *model.DatabaseMetadatype diffSDLMigration func(sourceSDL, targetSDL, engineVersion string, sessionCtx *SDLSessionContextMap) (string, error)type diffMetadataMigration func(oldSchema, newSchema *model.DatabaseMetadata) (string, error)type walkThrough func(*model.DatabaseMetadata, []base.AST) *storepb.Advice-type walkThroughWithContext func(WalkThroughContext, *model.DatabaseMetadata, []base.AST) *storepb.Advice+type walkThroughWithContext func(context.Context, WalkThroughContext, *model.DatabaseMetadata, []base.AST) *storepb.Advice// WalkThroughContext carries optional session state into schema walk-through implementations.type WalkThroughContext struct {@@ -533,12 +534,12 @@ func RegisterWalkThroughWithContext(engine storepb.Engine, f walkThroughWithCont}func WalkThrough(engine storepb.Engine, d *model.DatabaseMetadata, ast []base.AST) *storepb.Advice {- return WalkThroughWithContext(engine, WalkThroughContext{}, d, ast)+ return WalkThroughWithContext(context.Background(), engine, WalkThroughContext{}, d, ast)}-func WalkThroughWithContext(engine storepb.Engine, ctx WalkThroughContext, d *model.DatabaseMetadata, ast []base.AST) *storepb.Advice {+func WalkThroughWithContext(ctx context.Context, engine storepb.Engine, wtCtx WalkThroughContext, d *model.DatabaseMetadata, ast []base.AST) *storepb.Advice {if f, ok := walkThroughsWithContext[engine]; ok {- return f(ctx, d, ast)+ return f(ctx, wtCtx, d, ast)}f, ok := walkThroughs[engine]if !ok {diff --git a/go.mod b/go.modindex 79421ce3e3..0bb16e5fe8 100644--- a/go.mod+++ b/go.mod@@ -34,7 +34,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0- github.com/bytebase/omni v0.0.0-20260911085056-71713ba5c090+ github.com/bytebase/omni v0.0.0-20260912023254-4574e69bb9f1github.com/caarlos0/env/v11 v11.4.1github.com/cenkalti/backoff/v7 v7.0.0github.com/cockroachdb/cockroachdb-parser v0.25.2diff --git a/go.sum b/go.sumindex 5d42a50795..5e26f4029e 100644--- a/go.sum+++ b/go.sum@@ -252,8 +252,8 @@ github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735 h1:v8y6mXeHmYVsdMgithub.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735/go.mod h1:vTHjcvfn8Pmf65WjnV1ytTCYyC7xNhFjrniv4YWvtA8=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/omni v0.0.0-20260911085056-71713ba5c090 h1:0EZ0/nFVpkwDBp88Zb6nJAUmHAeP570s5dkOegvElxQ=-github.com/bytebase/omni v0.0.0-20260911085056-71713ba5c090/go.mod h1:9XZE1hqtvr09Oc+DdLu1/yCb8MDcoy/EPdA4AONDHlE=+github.com/bytebase/omni v0.0.0-20260912023254-4574e69bb9f1 h1:F0XizxSJZ5z5mdDkEMWwjgIso2mWG6kZB4MBNjwwmE0=+github.com/bytebase/omni v0.0.0-20260912023254-4574e69bb9f1/go.mod h1:9XZE1hqtvr09Oc+DdLu1/yCb8MDcoy/EPdA4AONDHlE=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=github.com/bytebase/pkcs8 v0.0.0-20240612095628-fcd0a7484c94 h1:6PNsuNmSqCuR8Z616uQLzDIvujWCFsKzeVc3ECmVubk=