Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: load every engine's catalog with omni's LoadMetadata (#21429)

Danny Xu650aa31db836
diff --git a/backend/plugin/advisor/sql_review.go b/backend/plugin/advisor/sql_review.go
index f0d46e2efe..6b7b87e1a7 100644
--- a/backend/plugin/advisor/sql_review.go
+++ b/backend/plugin/advisor/sql_review.go
@@ -133,7 +133,7 @@ func SQLReviewCheck(
SessionUser: checkContext.SessionUser,
RawSQL: statements,
}
- if advice := schema.WalkThroughWithContext(checkContext.DBType, walkThroughContext, checkContext.FinalMetadata, asts); advice != nil {
+ if advice := schema.WalkThroughWithContext(ctx, checkContext.DBType, walkThroughContext, checkContext.FinalMetadata, asts); advice != nil {
for _, rule := range ruleList {
if rule.Engine == checkContext.DBType && rule.Type == storepb.SQLReviewRule_BUILTIN_WALK_THROUGH_CHECK {
if status, err := NewStatusBySQLReviewRuleLevel(rule.Level); err == nil {
diff --git a/backend/plugin/parser/mongodb/completion.go b/backend/plugin/parser/mongodb/completion.go
index 0c6732dbfc..07c0f50121 100644
--- a/backend/plugin/parser/mongodb/completion.go
+++ b/backend/plugin/parser/mongodb/completion.go
@@ -82,16 +82,7 @@ func buildCatalog(ctx context.Context, cCtx base.CompletionContext) *catalog.Cat
return cat
}
- for _, schema := range metadata.ListSchemaNames() {
- schemaMeta := metadata.GetSchemaMetadata(schema)
- if schemaMeta == nil {
- continue
- }
- for _, table := range schemaMeta.ListTableNames() {
- cat.AddCollection(table)
- }
- }
-
+ cat.LoadMetadata(metadata.GetProto())
return cat
}
diff --git a/backend/plugin/parser/partiql/completion.go b/backend/plugin/parser/partiql/completion.go
index 2a9dc6fa5e..db293c6f2e 100644
--- a/backend/plugin/parser/partiql/completion.go
+++ b/backend/plugin/parser/partiql/completion.go
@@ -30,12 +30,7 @@ func Completion(ctx context.Context, cCtx base.CompletionContext, statement stri
_, dbMeta, err := cCtx.Metadata(ctx, cCtx.InstanceID, cCtx.DefaultDatabase)
if err == nil && dbMeta != nil {
databaseMetadata = dbMeta
- schema := dbMeta.GetSchemaMetadata("")
- if schema != nil {
- for _, table := range schema.ListTableNames() {
- cat.AddTable(table)
- }
- }
+ cat.LoadMetadata(dbMeta.GetProto())
}
}
diff --git a/backend/plugin/parser/pg/query_span.go b/backend/plugin/parser/pg/query_span.go
index 9dfc978677..0bf1f2e12b 100644
--- a/backend/plugin/parser/pg/query_span.go
+++ b/backend/plugin/parser/pg/query_span.go
@@ -75,20 +75,6 @@ type omniQuerySpanExtractor struct {
// fallbackCTEAliases holds CTE column aliases (e.g., WITH t1(cc1, cc2) AS (...)).
// Keyed by lowercase CTE name.
fallbackCTEAliases map[string][]string
-
- // Catalog loader state — populated by initCatalog.
- // degraded records objects whose real install failed and fell back to
- // pseudo. Keyed by loader object key (schema.name with kind prefix).
- degraded map[string]error
- // trulyBroken records objects whose pseudo install also failed.
- trulyBroken map[string]error
- // loaderObjects records every object collected from metadata. Used by
- // tests to distinguish loader bugs from metadata gaps.
- loaderObjects map[string]bool
- // lastFallbackReason is the classifier's verdict for the most recent
- // AnalyzeSelectStmt error at any of the three fallback sites. Used by
- // tests; not consumed by production code.
- lastFallbackReason fallbackReason
}
// newOmniQuerySpanExtractor creates a new omni-based query span extractor.
@@ -121,12 +107,9 @@ func (e *omniQuerySpanExtractor) getDatabaseMetadata(database string) (*model.Da
return meta, nil
}
-// initCatalog initializes the omni catalog from the database metadata using
-// the Catalog loader. Every metadata object is installed in dependency order,
-// with an inline pseudo fallback at any failed slot (see query_span_e3_*.go).
-//
-// The DDL render + batch-parse path used before this refactor is deleted: BYT-9215 and
-// BYT-9261 are unreachable by construction here.
+// initCatalog loads the database metadata into a new omni catalog. An object
+// that fails to install gets a text-typed stand-in or is left out; either way
+// the rest of the catalog still loads.
func (e *omniQuerySpanExtractor) initCatalog() error {
meta, err := e.getDatabaseMetadata(e.defaultDatabase)
if err != nil {
@@ -140,24 +123,20 @@ func (e *omniQuerySpanExtractor) initCatalog() error {
return nil
}
- loader := newCatalogLoader(e.cat, meta.GetProto())
- if err := loader.Load(e.ctx); err != nil {
- return errors.Wrapf(err, "catalog loader failed for %q", e.defaultDatabase)
+ report, err := e.cat.LoadMetadata(e.ctx, meta.GetProto(), catalog.LoadMetadataOptions{})
+ if err != nil {
+ return errors.Wrapf(err, "failed to load the catalog for %q", e.defaultDatabase)
}
- e.degraded = loader.degraded
- e.trulyBroken = loader.trulyBroken
- e.loaderObjects = loader.loaderObjects
-
- if len(loader.degraded) > 0 || len(loader.trulyBroken) > 0 {
- slog.Debug("catalog loader degraded objects",
+ if len(report.Degraded) > 0 || len(report.Missing) > 0 {
+ slog.Debug("catalog loaded with stand-ins",
slog.String("database", e.defaultDatabase),
- slog.Int("degraded", len(loader.degraded)),
- slog.Int("truly_broken", len(loader.trulyBroken)),
+ slog.Int("degraded", len(report.Degraded)),
+ slog.Int("missing", len(report.Missing)),
)
}
// Populate original (non-stubbed) function definitions for PL/pgSQL body
- // analysis. This is independent of catalog loading and unchanged by the loader.
+ // analysis.
for _, s := range meta.GetProto().GetSchemas() {
for _, f := range s.GetFunctions() {
if f.Definition == "" {
@@ -340,10 +319,6 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)
query, err := e.cat.AnalyzeSelectStmt(selStmt)
if err != nil {
- // Record the classifier's verdict for test inspection. This does not
- // gate behavior — the fallback below always runs on analyzer errors.
- e.lastFallbackReason = classifyAnalyzeError(err)
-
// Before falling back, try to handle user-defined table-returning functions
// that omni's AnalyzeSelectStmt can't resolve (e.g., RETURNS TABLE functions
// used as table sources: SELECT * FROM func()).
diff --git a/backend/plugin/parser/pg/query_span_composite_test.go b/backend/plugin/parser/pg/query_span_composite_test.go
deleted file mode 100644
index 4a1c885047..0000000000
--- a/backend/plugin/parser/pg/query_span_composite_test.go
+++ /dev/null
@@ -1,182 +0,0 @@
-package pg
-
-import (
- "context"
- "testing"
-
- metadatapb "github.com/bytebase/omni/metadata"
- "github.com/bytebase/omni/pg/ast"
- "github.com/bytebase/omni/pg/catalog"
-)
-
-// TestLoaderInstallsCompositeTypes proves the query-span catalog loader
-// installs composite types from metadata with real attributes, in dependency
-// order (aa_nested sorts before its dependency zz_base alphabetically), and
-// that tables typed with them install cleanly.
-func TestLoaderInstallsCompositeTypes(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{
- {
- Name: "public",
- EnumTypes: []*metadatapb.EnumTypeMetadata{
- {Name: "status", Values: []string{"a", "b"}},
- },
- CompositeTypes: []*metadatapb.CompositeTypeMetadata{
- {
- // References zz_base only through an array suffix.
- Name: "aa_array_only",
- Attributes: []*metadatapb.CompositeTypeAttribute{
- {Name: "items", Type: "public.zz_base[]"},
- },
- },
- {
- Name: "aa_nested",
- Attributes: []*metadatapb.CompositeTypeAttribute{
- {Name: "home", Type: "public.zz_base"},
- {Name: "s", Type: "public.status"},
- },
- },
- {
- Name: "zz_base",
- Attributes: []*metadatapb.CompositeTypeAttribute{
- {Name: "street", Type: "text"},
- {Name: "city", Type: "character varying(50)"},
- },
- },
- },
- Tables: []*metadatapb.TableMetadata{
- {
- Name: "users",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "integer"},
- {Name: "home", Type: "public.zz_base", Nullable: true},
- },
- },
- },
- },
- },
- }
-
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- loader := newCatalogLoader(cat, meta)
- if err := loader.Load(context.Background()); err != nil {
- t.Fatalf("Load: %v", err)
- }
-
- base := cat.GetRelation("public", "zz_base")
- if base == nil {
- t.Fatal("composite type zz_base not installed")
- }
- if len(base.Columns) != 2 {
- t.Errorf("zz_base attribute count: got %d, want 2", len(base.Columns))
- }
- if base.Columns[0].Name != "street" {
- t.Errorf("zz_base first attribute: got %q, want street", base.Columns[0].Name)
- }
- nested := cat.GetRelation("public", "aa_nested")
- if nested == nil {
- t.Fatal("composite type aa_nested not installed")
- }
- if len(nested.Columns) != 2 {
- t.Errorf("aa_nested attribute count: got %d, want 2", len(nested.Columns))
- }
- arrayOnly := cat.GetRelation("public", "aa_array_only")
- if arrayOnly == nil {
- t.Fatal("composite type aa_array_only not installed")
- }
- if len(arrayOnly.Columns) != 1 || arrayOnly.Columns[0].Name != "items" {
- t.Errorf("aa_array_only must install with its real attribute, got %d columns", len(arrayOnly.Columns))
- }
- if cat.GetRelation("public", "users") == nil {
- t.Fatal("table with composite column not installed")
- }
-}
-
-// TestLoaderCompositeFallbackPreservesAttributeNames forces the pseudo
-// fallback via a domain-typed attribute (domains are not loader objects) and
-// asserts the fallback keeps the metadata attribute names.
-func TestLoaderCompositeFallbackPreservesAttributeNames(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{
- {
- Name: "public",
- CompositeTypes: []*metadatapb.CompositeTypeMetadata{
- {
- Name: "with_domain",
- Attributes: []*metadatapb.CompositeTypeAttribute{
- {Name: "p", Type: "public.pos_int"},
- {Name: "note", Type: "text"},
- },
- },
- },
- },
- },
- }
-
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- loader := newCatalogLoader(cat, meta)
- if err := loader.Load(context.Background()); err != nil {
- t.Fatalf("Load: %v", err)
- }
-
- rel := cat.GetRelation("public", "with_domain")
- if rel == nil {
- t.Fatal("composite must fall back, not vanish")
- }
- if len(rel.Columns) != 2 {
- t.Fatalf("fallback must preserve attribute names, got %d columns", len(rel.Columns))
- }
- if rel.Columns[0].Name != "p" || rel.Columns[1].Name != "note" {
- t.Errorf("fallback attribute names: got %q, %q", rel.Columns[0].Name, rel.Columns[1].Name)
- }
-}
-
-// TestRangeVarFallbackSkipsCompositeTypes proves the query-span fallback does
-// not expand a standalone composite type as a FROM source — PostgreSQL
-// rejects composite types as table sources.
-func TestRangeVarFallbackSkipsCompositeTypes(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{
- {
- Name: "public",
- CompositeTypes: []*metadatapb.CompositeTypeMetadata{
- {
- Name: "addr",
- Attributes: []*metadatapb.CompositeTypeAttribute{
- {Name: "street", Type: "text"},
- },
- },
- },
- Tables: []*metadatapb.TableMetadata{
- {
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "integer"},
- },
- },
- },
- },
- },
- }
-
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- loader := newCatalogLoader(cat, meta)
- if err := loader.Load(context.Background()); err != nil {
- t.Fatalf("Load: %v", err)
- }
-
- e := &omniQuerySpanExtractor{
- cat: cat,
- searchPath: []string{"public"},
- defaultDatabase: "db",
- }
- if results := e.extractColumnsFromRangeVar(&ast.RangeVar{Schemaname: "public", Relname: "addr"}); results != nil {
- t.Errorf("composite type must not expand as a FROM source, got %d columns", len(results))
- }
- if results := e.extractColumnsFromRangeVar(&ast.RangeVar{Schemaname: "public", Relname: "t"}); len(results) != 1 {
- t.Errorf("real table must still expand, got %d columns", len(results))
- }
-}
diff --git a/backend/plugin/parser/pg/query_span_loader.go b/backend/plugin/parser/pg/query_span_loader.go
deleted file mode 100644
index 4fd76a1dc3..0000000000
--- a/backend/plugin/parser/pg/query_span_loader.go
+++ /dev/null
@@ -1,745 +0,0 @@
-package pg
-
-import (
- "cmp"
- "context"
- "slices"
-
- "github.com/pkg/errors"
-
- metadatapb "github.com/bytebase/omni/metadata"
- "github.com/bytebase/omni/pg/ast"
- "github.com/bytebase/omni/pg/catalog"
-)
-
-// catalogLoader installs every schema object from a metadatapb.DatabaseSchemaMetadata
-// into an omni catalog, in dependency-topological order, with an inline
-// pseudo fallback at each failed slot.
-//
-// Invariants after Load() returns without error:
-// 1. Every object from the metadata is present in the catalog — as either a
-// real install or a pseudo install. `loaderObjects` records its identity.
-// 2. `degraded[key]` contains the real-install error for objects whose real
-// install failed and fell back to pseudo.
-// 3. `trulyBroken[key]` contains the pseudo-install error for objects whose
-// even-pseudo install failed. These are the few objects callers can
-// expect analyzer errors on.
-//
-// The loader holds no reference to the catalog after Load() completes; the
-// catalog object belongs to the caller.
-type catalogLoader struct {
- cat *catalog.Catalog
- meta *metadatapb.DatabaseSchemaMetadata
-
- // loaderObjects maps canonical object keys (schema.name) to true for
- // every object the loader collected from metadata. Used by the classifier
- // to tell "loader knew about X and something went wrong" from "X was
- // never in metadata."
- loaderObjects map[string]bool
-
- // degraded records objects whose real install failed. The pseudo version
- // is in the catalog for these entries.
- degraded map[string]error
-
- // trulyBroken records objects whose pseudo install also failed. The
- // catalog has nothing at this slot.
- trulyBroken map[string]error
-}
-
-// objectKind identifies the kind of an Catalog loader object. It drives the
-// install switch in installReal/installPseudo and the dependency-edge
-// computation in buildDependencyGraph.
-type objectKind int
-
-const (
- kindSchema objectKind = iota
- kindEnum
- kindComposite
- kindTable
- kindView
- kindMatView
- kindFunction
-)
-
-// objectEntry is the flattened, order-free unit of work the loader processes.
-// One entry per schema object. Entries for the same (schema, name) across
-// different kinds are allowed (e.g. a table and a function with the same
-// name) and treated as independent nodes in the dep graph.
-type objectEntry struct {
- kind objectKind
- schema string
- name string
-
- // Exactly one of the following is populated based on kind.
- enumMeta *metadatapb.EnumTypeMetadata
- compositeMeta *metadatapb.CompositeTypeMetadata
- tableMeta *metadatapb.TableMetadata
- viewMeta *metadatapb.ViewMetadata
- matViewMeta *metadatapb.MaterializedViewMetadata
- funcMeta *metadatapb.FunctionMetadata
-}
-
-// key returns a canonical kind-prefixed identifier used for dependency
-// lookups and set membership. Tables, views, and matviews share a namespace
-// (relation namespace in PG), so they use the same prefix "rel:".
-//
-// Functions are identified by schema, name AND signature so that overloaded
-// names (e.g. `public.fn(int4)` and `public.fn(text)`) do not collide and
-// collapse into a single entry during topo-sort bookkeeping.
-func (e *objectEntry) key() string {
- switch e.kind {
- case kindSchema:
- return "schema:" + e.schema
- case kindEnum, kindComposite:
- return "type:" + e.schema + "." + e.name
- case kindTable, kindView, kindMatView:
- return "rel:" + e.schema + "." + e.name
- case kindFunction:
- return "func:" + e.schema + "." + e.name + "|" + funcSignatureKey(e.funcMeta)
- }
- return "unknown:" + e.schema + "." + e.name
-}
-
-// sortKey returns the intra-SCC lexicographic key. Matches hard contract C10.
-// For functions the signature is appended so overloads have distinct sort
-// keys and the install order is fully deterministic.
-func (e *objectEntry) sortKey() string {
- base := e.schema + "\x00" + e.name + "\x00" + kindLabel(e.kind)
- if e.kind == kindFunction {
- base += "\x00" + funcSignatureKey(e.funcMeta)
- }
- return base
-}
-
-// funcSignatureKey returns a stable, comparable identifier for a function
-// signature. Uses the metadata Signature string when present; falls back to
-// the definition hash to keep distinct overloads distinguishable even when
-// sync emitted an empty Signature field.
-func funcSignatureKey(fn *metadatapb.FunctionMetadata) string {
- if fn == nil {
- return ""
- }
- if fn.Signature != "" {
- return fn.Signature
- }
- return fn.Definition
-}
-
-func kindLabel(k objectKind) string {
- switch k {
- case kindSchema:
- return "0schema"
- case kindEnum:
- return "1enum"
- case kindComposite:
- return "1composite"
- case kindTable:
- return "2table"
- case kindView:
- return "3view"
- case kindMatView:
- return "4matview"
- case kindFunction:
- return "5function"
- }
- return "9unknown"
-}
-
-// newCatalogLoader returns a loader primed for Load(). It does not touch the
-// catalog until Load() is called.
-func newCatalogLoader(cat *catalog.Catalog, meta *metadatapb.DatabaseSchemaMetadata) *catalogLoader {
- return &catalogLoader{
- cat: cat,
- meta: meta,
- loaderObjects: make(map[string]bool),
- degraded: make(map[string]error),
- trulyBroken: make(map[string]error),
- }
-}
-
-// Load collects every object from metadata, topologically orders them, and
-// installs each either real or (on failure) pseudo. The returned error is
-// non-nil only for catastrophic conditions (nil catalog, ctx cancellation);
-// per-object failures are recorded in degraded/trulyBroken and do not stop
-// the loader.
-func (l *catalogLoader) Load(ctx context.Context) error {
- if l.cat == nil {
- return errors.New("catalogLoader: nil catalog")
- }
- if l.meta == nil {
- return nil
- }
-
- objects := l.collectObjects()
- for _, obj := range objects {
- l.loaderObjects[obj.key()] = true
- }
-
- sorted := topoSortObjects(objects)
- for _, obj := range sorted {
- if err := ctx.Err(); err != nil {
- return err
- }
- l.installOne(obj)
- }
- return nil
-}
-
-// collectObjects flattens DatabaseSchemaMetadata into a list of objectEntry
-// values, one per schema object the loader will install. Schemas themselves
-// become kindSchema entries so CREATE SCHEMA is part of the topo order.
-func (l *catalogLoader) collectObjects() []*objectEntry {
- var out []*objectEntry
- for _, sm := range l.meta.Schemas {
- if sm.Name == "" {
- // PG sync sometimes emits a no-schema placeholder; skip.
- continue
- }
- out = append(out, &objectEntry{
- kind: kindSchema,
- schema: sm.Name,
- name: sm.Name,
- })
- for _, enum := range sm.EnumTypes {
- out = append(out, &objectEntry{
- kind: kindEnum,
- schema: sm.Name,
- name: enum.Name,
- enumMeta: enum,
- })
- }
- for _, composite := range sm.CompositeTypes {
- out = append(out, &objectEntry{
- kind: kindComposite,
- schema: sm.Name,
- name: composite.Name,
- compositeMeta: composite,
- })
- }
- for _, tbl := range sm.Tables {
- out = append(out, &objectEntry{
- kind: kindTable,
- schema: sm.Name,
- name: tbl.Name,
- tableMeta: tbl,
- })
- }
- for _, view := range sm.Views {
- out = append(out, &objectEntry{
- kind: kindView,
- schema: sm.Name,
- name: view.Name,
- viewMeta: view,
- })
- }
- for _, mv := range sm.MaterializedViews {
- out = append(out, &objectEntry{
- kind: kindMatView,
- schema: sm.Name,
- name: mv.Name,
- matViewMeta: mv,
- })
- }
- for _, fn := range sm.Functions {
- out = append(out, &objectEntry{
- kind: kindFunction,
- schema: sm.Name,
- name: fn.Name,
- funcMeta: fn,
- })
- }
- }
- return out
-}
-
-// topoSortObjects orders objects by their dependency edges so each object is
-// processed only after its dependencies. Cycles are broken via Tarjan SCC
-// with intra-SCC lexicographic ordering (C10): one member of an SCC installs
-// first (and typically fails → pseudo), the rest install against the pseudo.
-func topoSortObjects(objects []*objectEntry) []*objectEntry {
- if len(objects) == 0 {
- return nil
- }
-
- index := make(map[string]*objectEntry, len(objects))
- for _, obj := range objects {
- index[obj.key()] = obj
- }
-
- edges := buildDependencyEdges(objects, index)
- sccs := tarjanSCC(objects, edges)
-
- // Build a condensed DAG of SCCs, then topo sort and flatten.
- sccOf := make(map[string]int, len(objects))
- for i, scc := range sccs {
- for _, obj := range scc {
- sccOf[obj.key()] = i
- }
- }
- condensedEdges := make([][]int, len(sccs))
- inDegree := make([]int, len(sccs))
- seenEdge := make(map[[2]int]bool)
- for src, dsts := range edges {
- srcSCC, ok := sccOf[src]
- if !ok {
- continue
- }
- for _, dst := range dsts {
- dstSCC, ok := sccOf[dst]
- if !ok || dstSCC == srcSCC {
- continue
- }
- edge := [2]int{dstSCC, srcSCC} // dst must come before src
- if seenEdge[edge] {
- continue
- }
- seenEdge[edge] = true
- condensedEdges[dstSCC] = append(condensedEdges[dstSCC], srcSCC)
- inDegree[srcSCC]++
- }
- }
-
- // Kahn's algorithm on the condensed graph with a deterministic tiebreak:
- // ready SCCs are sorted by the lex-smallest sort key of their members.
- readyHeap := make([]int, 0)
- for i := range sccs {
- if inDegree[i] == 0 {
- readyHeap = append(readyHeap, i)
- }
- }
- sortSCCsByMin(readyHeap, sccs)
-
- var flat []*objectEntry
- for len(readyHeap) > 0 {
- next := readyHeap[0]
- readyHeap = readyHeap[1:]
- flat = append(flat, sortedSCCMembers(sccs[next])...)
- for _, nb := range condensedEdges[next] {
- inDegree[nb]--
- if inDegree[nb] == 0 {
- readyHeap = append(readyHeap, nb)
- }
- }
- sortSCCsByMin(readyHeap, sccs)
- }
-
- // Safety net: if any objects were not emitted (e.g. invariant violation
- // in Tarjan / edge building), emit them in deterministic order so the
- // loader still covers them.
- if len(flat) != len(objects) {
- emitted := make(map[string]bool, len(flat))
- for _, o := range flat {
- emitted[o.key()] = true
- }
- var missed []*objectEntry
- for _, o := range objects {
- if !emitted[o.key()] {
- missed = append(missed, o)
- }
- }
- slices.SortStableFunc(missed, func(a, b *objectEntry) int {
- return cmp.Compare(a.sortKey(), b.sortKey())
- })
- flat = append(flat, missed...)
- }
- return flat
-}
-
-// buildDependencyEdges returns a map from each object's key to the keys of
-// objects it depends on. Only edges whose target actually exists in the
-// collected set are recorded; references to unknown objects become no-ops
-// (the install will fail and trigger pseudo).
-func buildDependencyEdges(objects []*objectEntry, index map[string]*objectEntry) map[string][]string {
- edges := make(map[string][]string, len(objects))
- for _, obj := range objects {
- var deps []string
- // Every non-schema object depends on its schema.
- if obj.kind != kindSchema {
- if key := "schema:" + obj.schema; index[key] != nil {
- deps = append(deps, key)
- }
- }
- switch obj.kind {
- case kindTable:
- for _, col := range obj.tableMeta.Columns {
- for _, ref := range extractUserTypeRefs(col.Type) {
- if k := typeRefKey(ref); index[k] != nil {
- deps = append(deps, k)
- }
- }
- }
- case kindView:
- for _, dep := range obj.viewMeta.DependencyColumns {
- if k := relationKey(dep.Schema, dep.Table); index[k] != nil {
- deps = append(deps, k)
- }
- }
- case kindMatView:
- for _, dep := range obj.matViewMeta.DependencyColumns {
- if k := relationKey(dep.Schema, dep.Table); index[k] != nil {
- deps = append(deps, k)
- }
- }
- case kindFunction:
- argTypes, _ := parseFunctionSignatureArgTypes(obj.funcMeta.Signature)
- for _, at := range argTypes {
- for _, ref := range extractUserTypeRefs(at) {
- if k := typeRefKey(ref); index[k] != nil {
- deps = append(deps, k)
- }
- }
- }
- case kindComposite:
- for _, attribute := range obj.compositeMeta.Attributes {
- for _, ref := range extractUserTypeRefs(attribute.Type) {
- if k := typeRefKey(ref); index[k] != nil {
- deps = append(deps, k)
- }
- // An attribute may be typed with a table/view row type.
- if k := relationKey(ref.Schema, ref.Name); index[k] != nil {
- deps = append(deps, k)
- }
- }
- }
- case kindSchema, kindEnum:
- // Schemas have no outgoing edges (they are the top of the graph).
- // Enums depend only on their schema, already captured above.
- default:
- // Unknown kinds contribute no edges; install will surface the
- // problem at real-install time.
- }
- if len(deps) > 0 {
- edges[obj.key()] = dedupStrings(deps)
- }
- }
- return edges
-}
-
-func typeRefKey(r UserTypeRef) string {
- return "type:" + r.Schema + "." + r.Name
-}
-
-func relationKey(schema, name string) string {
- return "rel:" + schema + "." + name
-}
-
-func dedupStrings(in []string) []string {
- if len(in) < 2 {
- return in
- }
- seen := make(map[string]bool, len(in))
- out := in[:0]
- for _, s := range in {
- if !seen[s] {
- seen[s] = true
- out = append(out, s)
- }
- }
- return out
-}
-
-// tarjanSCC runs Tarjan's strongly-connected-components algorithm on the
-// dependency graph and returns SCCs. Each SCC's members are returned in the
-// order the algorithm discovered them; callers sort for determinism.
-func tarjanSCC(objects []*objectEntry, edges map[string][]string) [][]*objectEntry {
- type state struct {
- index, low int
- onStack bool
- }
- st := make(map[string]*state, len(objects))
- byKey := make(map[string]*objectEntry, len(objects))
- for _, obj := range objects {
- byKey[obj.key()] = obj
- }
-
- // Walk nodes in deterministic order so tests and shadow diffs reproduce.
- keys := make([]string, 0, len(objects))
- for _, obj := range objects {
- keys = append(keys, obj.key())
- }
- slices.Sort(keys)
-
- var (
- index int
- stack []string
- result [][]*objectEntry
- )
-
- var strongconnect func(v string)
- strongconnect = func(v string) {
- st[v] = &state{index: index, low: index, onStack: true}
- index++
- stack = append(stack, v)
-
- for _, w := range edges[v] {
- if _, ok := byKey[w]; !ok {
- continue
- }
- if _, seen := st[w]; !seen {
- strongconnect(w)
- st[v].low = min(st[v].low, st[w].low)
- } else if st[w].onStack {
- st[v].low = min(st[v].low, st[w].index)
- }
- }
-
- if st[v].low == st[v].index {
- var scc []*objectEntry
- for {
- w := stack[len(stack)-1]
- stack = stack[:len(stack)-1]
- st[w].onStack = false
- scc = append(scc, byKey[w])
- if w == v {
- break
- }
- }
- result = append(result, scc)
- }
- }
-
- for _, k := range keys {
- if _, seen := st[k]; !seen {
- strongconnect(k)
- }
- }
- return result
-}
-
-// sortedSCCMembers returns a copy of scc sorted by sortKey for deterministic
-// install order.
-func sortedSCCMembers(scc []*objectEntry) []*objectEntry {
- out := make([]*objectEntry, len(scc))
- copy(out, scc)
- slices.SortStableFunc(out, func(a, b *objectEntry) int {
- return cmp.Compare(a.sortKey(), b.sortKey())
- })
- return out
-}
-
-// sortSCCsByMin sorts a slice of SCC indices in place by the lex-smallest
-// sortKey among each SCC's members. Stable ordering.
-func sortSCCsByMin(indices []int, sccs [][]*objectEntry) {
- slices.SortStableFunc(indices, func(a, b int) int {
- return cmp.Compare(minSortKey(sccs[a]), minSortKey(sccs[b]))
- })
-}
-
-func minSortKey(scc []*objectEntry) string {
- if len(scc) == 0 {
- return ""
- }
- best := scc[0].sortKey()
- for _, o := range scc[1:] {
- if k := o.sortKey(); k < best {
- best = k
- }
- }
- return best
-}
-
-// installOne attempts a real install; on failure installs a pseudo at the
-// same slot. All outcomes are recorded in l.degraded / l.trulyBroken; this
-// function never returns an error to the caller because per-object failures
-// are not fatal to the loader.
-//
-// Functions are a special case: when a function's real install fails, we
-// deliberately leave the slot empty. Query span has a dedicated fallback for
-// user functions (`tryMetadataFuncLookup` in query_span.go) that
-// reconstructs a synthetic proc from metadata at query time. Installing a
-// text-backed pseudo function would crowd out that path, producing wrong
-// overload resolution and losing body-level lineage. Tables/views/types are
-// fine to pseudo-install because their lookups only need the object to exist.
-func (l *catalogLoader) installOne(obj *objectEntry) {
- realErr := l.installReal(obj)
- if realErr == nil {
- return
- }
- l.degraded[obj.key()] = realErr
- if obj.kind == kindFunction {
- // No pseudo — rely on tryMetadataFuncLookup at query time.
- return
- }
- if pErr := l.installPseudo(obj); pErr != nil {
- l.trulyBroken[obj.key()] = pErr
- }
-}
-
-// installReal translates the object's metadata into an AST and calls the
-// matching omni DefineX / ExecCreateTableAs / CreateFunctionStmt. Returns
-// the first error encountered.
-func (l *catalogLoader) installReal(obj *objectEntry) error {
- switch obj.kind {
- case kindSchema:
- err := l.cat.CreateSchemaCommand(&ast.CreateSchemaStmt{
- Schemaname: obj.schema,
- })
- // Built-in schemas (public, pg_catalog, pg_toast) are preloaded by
- // catalog.New(). Treating duplicate-schema as success keeps the
- // loader's "every metadata object is present post-Load" invariant
- // without needing a hardcoded exclusion list.
- var cErr *catalog.Error
- if errors.As(err, &cErr) && cErr.Code == catalog.CodeDuplicateSchema {
- return nil
- }
- return err
- case kindEnum:
- return l.cat.DefineEnum(buildCreateEnumStmt(obj.schema, obj.enumMeta))
- case kindComposite:
- stmt, err := buildCompositeTypeStmt(obj.schema, obj.compositeMeta)
- if err != nil {
- return err
- }
- return l.cat.DefineCompositeType(stmt)
- case kindTable:
- stmt, err := buildCreateStmt(obj.schema, obj.tableMeta)
- if err != nil {
- return err
- }
- return l.cat.DefineRelation(stmt, 'r')
- case kindView:
- stmt, err := buildViewStmt(obj.schema, obj.viewMeta)
- if err != nil {
- return err
- }
- return l.cat.DefineView(stmt)
- case kindMatView:
- stmt, err := buildCreateTableAsStmt(obj.schema, obj.matViewMeta)
- if err != nil {
- return err
- }
- return l.cat.ExecCreateTableAs(stmt)
- case kindFunction:
- stmt, err := buildCreateFunctionStmt(obj.schema, obj.funcMeta)
- if err != nil {
- return err
- }
- return l.cat.CreateFunctionStmt(stmt)
- }
- return errors.Errorf("unknown object kind %d", obj.kind)
-}
-
-// installPseudo builds and installs the pseudo variant of an object. Every
-// pseudo form is text-backed and free of user-object dependencies, so this
-// function essentially never fails; the few error paths exist for defense in
-// depth.
-func (l *catalogLoader) installPseudo(obj *objectEntry) error {
- switch obj.kind {
- case kindSchema:
- // Schemas have no pseudo form — they either exist or they don't.
- // If CreateSchemaCommand failed, treat as trulyBroken.
- return errors.New("schema has no pseudo form")
- case kindEnum:
- return l.cat.DefineEnum(pseudoCreateEnumStmt(obj.schema, obj.name))
- case kindComposite:
- // Name-preserving, text-backed fallback when metadata attributes are
- // known, so (col).field access still resolves; the "_broken" shape
- // remains for composites with no attribute information at all.
- if len(obj.compositeMeta.GetAttributes()) > 0 {
- items := make([]ast.Node, 0, len(obj.compositeMeta.Attributes))
- for _, attribute := range obj.compositeMeta.Attributes {
- if attribute.Name == "" {
- continue
- }
- items = append(items, &ast.ColumnDef{
- Colname: attribute.Name,
- TypeName: pseudoTextTypeName(),
- })
- }
- return l.cat.DefineCompositeType(&ast.CompositeTypeStmt{
- Typevar: &ast.RangeVar{Schemaname: obj.schema, Relname: obj.name},
- Coldeflist: &ast.List{Items: items},
- })
- }
- return l.cat.DefineCompositeType(pseudoCompositeTypeStmt(obj.schema, obj.name))
- case kindTable:
- cols := columnNamesFromTableMetadata(obj.tableMeta)
- return l.cat.DefineRelation(pseudoCreateTableStmt(obj.schema, obj.name, cols), 'r')
- case kindView:
- cols := viewColumnNames(obj.viewMeta)
- stmt, err := pseudoViewStmt(obj.schema, obj.name, cols)
- if err != nil {
- return err
- }
- return l.cat.DefineView(stmt)
- case kindMatView:
- cols := matviewColumnNames(obj.matViewMeta)
- stmt, err := pseudoCreateTableAsStmt(obj.schema, obj.name, cols)
- if err != nil {
- return err
- }
- return l.cat.ExecCreateTableAs(stmt)
- case kindFunction:
- argCount := functionArgCountFromSignature(obj.funcMeta.Signature)
- return l.cat.CreateFunctionStmt(pseudoCreateFunctionStmt(obj.schema, obj.name, argCount))
- }
- return errors.Errorf("unknown object kind %d", obj.kind)
-}
-
-// columnNamesFromTableMetadata returns the deduplicated column names of a
-// table's metadata, in order, for use as pseudo column names.
-func columnNamesFromTableMetadata(tbl *metadatapb.TableMetadata) []string {
- if tbl == nil {
- return nil
- }
- seen := make(map[string]bool, len(tbl.Columns))
- out := make([]string, 0, len(tbl.Columns))
- for _, col := range tbl.Columns {
- if col.Name == "" || seen[col.Name] {
- continue
- }
- seen[col.Name] = true
- out = append(out, col.Name)
- }
- return out
-}
-
-// viewColumnNames returns the pseudo column-name list for a view. Prefers
-// explicit Columns (when sync populated them); falls back to the set of
-// dependency_columns.column values.
-func viewColumnNames(v *metadatapb.ViewMetadata) []string {
- if v == nil {
- return nil
- }
- if len(v.Columns) > 0 {
- seen := make(map[string]bool, len(v.Columns))
- out := make([]string, 0, len(v.Columns))
- for _, col := range v.Columns {
- if col.Name == "" || seen[col.Name] {
- continue
- }
- seen[col.Name] = true
- out = append(out, col.Name)
- }
- return out
- }
- seen := make(map[string]bool, len(v.DependencyColumns))
- out := make([]string, 0, len(v.DependencyColumns))
- for _, dc := range v.DependencyColumns {
- if dc.Column == "" || seen[dc.Column] {
- continue
- }
- seen[dc.Column] = true
- out = append(out, dc.Column)
- }
- return out
-}
-
-// matviewColumnNames returns pseudo column names for a materialized view.
-// MaterializedViewMetadata does not carry a Columns field, so we fall back
-// to dependency_columns.
-func matviewColumnNames(m *metadatapb.MaterializedViewMetadata) []string {
- if m == nil {
- return nil
- }
- seen := make(map[string]bool, len(m.DependencyColumns))
- out := make([]string, 0, len(m.DependencyColumns))
- for _, dc := range m.DependencyColumns {
- if dc.Column == "" || seen[dc.Column] {
- continue
- }
- seen[dc.Column] = true
- out = append(out, dc.Column)
- }
- return out
-}
diff --git a/backend/plugin/parser/pg/query_span_loader_builders.go b/backend/plugin/parser/pg/query_span_loader_builders.go
deleted file mode 100644
index 9d4b6c120c..0000000000
--- a/backend/plugin/parser/pg/query_span_loader_builders.go
+++ /dev/null
@@ -1,290 +0,0 @@
-package pg
-
-import (
- "strings"
-
- "github.com/pkg/errors"
-
- metadatapb "github.com/bytebase/omni/metadata"
- "github.com/bytebase/omni/pg/ast"
-)
-
-// This file translates storepb metadata into omni AST nodes. Each build*
-// function is deterministic, allocates fresh nodes, and does not touch the
-// catalog. Builders may return errors for inputs the cheat parser rejects;
-// the loader catches those errors and installs a pseudo at the same slot.
-
-// buildCreateEnumStmt translates EnumTypeMetadata + schema into a
-// CreateEnumStmt. Enum values are preserved as plain string literals.
-func buildCreateEnumStmt(schema string, enum *metadatapb.EnumTypeMetadata) *ast.CreateEnumStmt {
- vals := make([]ast.Node, 0, len(enum.Values))
- for _, v := range enum.Values {
- vals = append(vals, &ast.String{Str: v})
- }
- return &ast.CreateEnumStmt{
- TypeName: qualifiedNameList(schema, enum.Name),
- Vals: &ast.List{Items: vals},
- }
-}
-
-// buildCompositeTypeStmt translates CompositeTypeMetadata + schema into a
-// CompositeTypeStmt. Attribute types are parsed via typeNameFromString; an
-// error on any attribute propagates so the loader can pseudo the whole type.
-func buildCompositeTypeStmt(schema string, composite *metadatapb.CompositeTypeMetadata) (*ast.CompositeTypeStmt, error) {
- cols := make([]ast.Node, 0, len(composite.Attributes))
- for _, attribute := range composite.Attributes {
- if attribute.Name == "" {
- continue
- }
- if attribute.Type == "" {
- return nil, errors.Errorf("attribute %q: empty type", attribute.Name)
- }
- tn, err := typeNameFromString(attribute.Type)
- if err != nil {
- return nil, errors.Wrapf(err, "attribute %q", attribute.Name)
- }
- cols = append(cols, &ast.ColumnDef{
- Colname: attribute.Name,
- TypeName: tn,
- CollClause: qsCollateClauseFromReference(attribute.Collation),
- })
- }
- return &ast.CompositeTypeStmt{
- Typevar: &ast.RangeVar{Schemaname: schema, Relname: composite.Name},
- Coldeflist: &ast.List{Items: cols},
- }, nil
-}
-
-// buildCreateStmt translates TableMetadata into a CreateStmt with real
-// column types. Column types are parsed via typeNameFromString; an error on
-// any single column propagates back so the loader can pseudo the whole
-// relation rather than install a partially-typed catalog entry.
-//
-// Columns with empty names are skipped.
-func buildCreateStmt(schema string, table *metadatapb.TableMetadata) (*ast.CreateStmt, error) {
- items := make([]ast.Node, 0, len(table.Columns))
- for _, col := range table.Columns {
- if col.Name == "" {
- continue
- }
- if col.Type == "" {
- return nil, errors.Errorf("column %q: empty type", col.Name)
- }
- tn, err := typeNameFromString(col.Type)
- if err != nil {
- return nil, errors.Wrapf(err, "column %q", col.Name)
- }
- items = append(items, &ast.ColumnDef{
- Colname: col.Name,
- TypeName: tn,
- IsNotNull: !col.Nullable,
- })
- }
- return &ast.CreateStmt{
- Relation: &ast.RangeVar{
- Schemaname: schema,
- Relname: table.Name,
- Relpersistence: 'p',
- },
- TableElts: &ast.List{Items: items},
- }, nil
-}
-
-// buildViewStmt translates ViewMetadata into a ViewStmt. The view body is
-// parsed via ParsePg; the result must be a *ast.SelectStmt (not a RawStmt
-// wrapper) because omni's DefineView type-asserts directly.
-//
-// If the body is empty or unparseable, returns an error; the loader will
-// pseudo the view using dependency_columns metadata.
-func buildViewStmt(schema string, view *metadatapb.ViewMetadata) (*ast.ViewStmt, error) {
- if view.Definition == "" {
- return nil, errors.New("empty view definition")
- }
- sel, err := parseSelectBody(view.Definition)
- if err != nil {
- return nil, errors.Wrapf(err, "view %q body", view.Name)
- }
- return &ast.ViewStmt{
- View: &ast.RangeVar{
- Schemaname: schema,
- Relname: view.Name,
- Relpersistence: 'p',
- },
- Query: sel,
- }, nil
-}
-
-// buildCreateTableAsStmt translates MaterializedViewMetadata into a
-// CreateTableAsStmt with Objtype OBJECT_MATVIEW. Same SelectStmt contract
-// as buildViewStmt: body must parse to *ast.SelectStmt.
-func buildCreateTableAsStmt(schema string, mv *metadatapb.MaterializedViewMetadata) (*ast.CreateTableAsStmt, error) {
- if mv.Definition == "" {
- return nil, errors.New("empty matview definition")
- }
- sel, err := parseSelectBody(mv.Definition)
- if err != nil {
- return nil, errors.Wrapf(err, "matview %q body", mv.Name)
- }
- return &ast.CreateTableAsStmt{
- Query: sel,
- Objtype: ast.OBJECT_MATVIEW,
- Into: &ast.IntoClause{
- Rel: &ast.RangeVar{
- Schemaname: schema,
- Relname: mv.Name,
- Relpersistence: 'p',
- },
- },
- }, nil
-}
-
-// buildCreateFunctionStmt translates FunctionMetadata into a
-// *ast.CreateFunctionStmt. The primary path parses the function's full
-// Definition (an `omni`-compatible CREATE FUNCTION/PROCEDURE statement),
-// which preserves parameter types, return-type signatures (including
-// RETURNS TABLE / RETURNS SETOF), and the dollar-quoted body verbatim.
-//
-// When the definition string is missing or cannot be parsed as a
-// CreateFunctionStmt, we fall back to a minimal text-typed shape
-// reconstructed from the signature string. The fallback loses return-type
-// fidelity but keeps analyzer name resolution working for queries that
-// reference the function by name.
-func buildCreateFunctionStmt(schema string, fn *metadatapb.FunctionMetadata) (*ast.CreateFunctionStmt, error) {
- if fn.Definition != "" {
- stmts, err := ParsePg(fn.Definition)
- if err == nil && len(stmts) == 1 {
- node := stmts[0].AST
- if raw, ok := node.(*ast.RawStmt); ok {
- node = raw.Stmt
- }
- if parsed, ok := node.(*ast.CreateFunctionStmt); ok {
- return parsed, nil
- }
- }
- }
- return buildCreateFunctionStmtFromSignature(schema, fn)
-}
-
-// buildCreateFunctionStmtFromSignature is the minimal-shape fallback used
-// when the full definition is missing or unparseable. Parameters and return
-// type collapse to text; the body is a trivial `SELECT NULL::text`.
-func buildCreateFunctionStmtFromSignature(schema string, fn *metadatapb.FunctionMetadata) (*ast.CreateFunctionStmt, error) {
- argTypes, err := parseFunctionSignatureArgTypes(fn.Signature)
- if err != nil {
- return nil, errors.Wrapf(err, "function %q signature %q", fn.Name, fn.Signature)
- }
- params := make([]ast.Node, 0, len(argTypes))
- for i, argType := range argTypes {
- tn, err := typeNameFromString(argType)
- if err != nil {
- return nil, errors.Wrapf(err, "function %q arg %d", fn.Name, i)
- }
- params = append(params, &ast.FunctionParameter{
- ArgType: tn,
- Mode: ast.FUNC_PARAM_IN,
- })
- }
- stmt := &ast.CreateFunctionStmt{
- Funcname: qualifiedNameList(schema, fn.Name),
- ReturnType: pseudoTextTypeName(),
- }
- if len(params) > 0 {
- stmt.Parameters = &ast.List{Items: params}
- }
- stmt.Options = &ast.List{Items: []ast.Node{
- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},
- &ast.DefElem{
- Defname: "as",
- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT NULL::text"}}},
- },
- }}
- return stmt, nil
-}
-
-// parseSelectBody parses a SQL string expected to be a single SELECT
-// statement and returns the *ast.SelectStmt. Rejects empty input, multiple
-// statements, and non-SELECT top-level nodes. Unwraps a RawStmt wrapper if
-// ParsePg returns one.
-func parseSelectBody(sql string) (*ast.SelectStmt, error) {
- stmts, err := ParsePg(sql)
- if err != nil {
- return nil, errors.Wrap(err, "parse")
- }
- if len(stmts) != 1 {
- return nil, errors.Errorf("expected 1 statement, got %d", len(stmts))
- }
- node := stmts[0].AST
- if raw, ok := node.(*ast.RawStmt); ok {
- node = raw.Stmt
- }
- sel, ok := node.(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("expected SelectStmt, got %T", node)
- }
- return sel, nil
-}
-
-// parseFunctionSignatureArgTypes extracts the comma-separated argument types
-// from a signature like "fn(integer, text)" or "fn(numeric(10,2), boolean)".
-// An empty parameter list returns an empty slice.
-func parseFunctionSignatureArgTypes(signature string) ([]string, error) {
- open := strings.Index(signature, "(")
- if open < 0 {
- return nil, nil
- }
- closeIdx := strings.LastIndex(signature, ")")
- if closeIdx < 0 || closeIdx <= open {
- return nil, errors.Errorf("signature %q: unbalanced parentheses", signature)
- }
- return splitTopLevelCommas(signature[open+1 : closeIdx]), nil
-}
-
-// splitTopLevelCommas splits a string on commas that are not nested inside
-// parentheses. Used for function signature argument lists so that types like
-// `numeric(10,2)` are not split.
-func splitTopLevelCommas(s string) []string {
- s = strings.TrimSpace(s)
- if s == "" {
- return nil
- }
- var parts []string
- depth := 0
- start := 0
- for i := 0; i < len(s); i++ {
- switch s[i] {
- case '(':
- depth++
- case ')':
- if depth > 0 {
- depth--
- }
- case ',':
- if depth == 0 {
- parts = append(parts, strings.TrimSpace(s[start:i]))
- start = i + 1
- }
- default:
- // other bytes are part of the current segment
- }
- }
- parts = append(parts, strings.TrimSpace(s[start:]))
- return parts
-}
-
-// qsCollateClauseFromReference converts a stored emit-ready collation
-// reference (quoted as needed, schema-qualified outside pg_catalog) into an
-// AST collate clause. Returns nil for no collation.
-func qsCollateClauseFromReference(reference string) *ast.CollateClause {
- if reference == "" {
- return nil
- }
- var items []ast.Node
- if schemaName, collationName, ok := splitQualifiedName(reference); ok {
- items = []ast.Node{&ast.String{Str: schemaName}, &ast.String{Str: collationName}}
- } else {
- name := strings.Trim(reference, `"`)
- name = strings.ReplaceAll(name, `""`, `"`)
- items = []ast.Node{&ast.String{Str: name}}
- }
- return &ast.CollateClause{Collname: &ast.List{Items: items}}
-}
diff --git a/backend/plugin/parser/pg/query_span_loader_classify.go b/backend/plugin/parser/pg/query_span_loader_classify.go
deleted file mode 100644
index e10d1d6c71..0000000000
--- a/backend/plugin/parser/pg/query_span_loader_classify.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package pg
-
-import (
- "github.com/pkg/errors"
-
- "github.com/bytebase/omni/pg/catalog"
-)
-
-// fallbackReason classifies why analyzer analysis fell through to
-// extractFallbackColumns / fallbackExtractLineage. It is a pure classification
-// of the error from omni's AnalyzeSelectStmt, used by tests to assert which
-// branch a given query takes.
-//
-// This is NOT a runtime gate, a metric, or a telemetry signal. It exists so
-// that unit tests can distinguish "pseudo-induced degradation" from
-// "undefined reference" from "analyzer does not support this feature", and so
-// that debugging from logs carries a meaningful category.
-type fallbackReason int
-
-const (
- reasonNone fallbackReason = iota
- // reasonExpectedPseudoSemantic: analyzer failed because a pseudo-installed
- // object looks like text in place of its original user type. Common forms:
- // UndefinedFunction (overload signature no longer matches), DatatypeMismatch
- // (operator resolution against text-backed pseudo), AmbiguousFunction /
- // AmbiguousColumn (candidate pile from pseudo-widening).
- reasonExpectedPseudoSemantic
- // reasonUndefinedReference: analyzer could not find a relation, column,
- // schema, or object. This can be a loader bug (we should have installed
- // it) OR a legitimate user error (query typos, stale metadata). Without a
- // structured ErrorIdent on *catalog.Error we cannot programmatically
- // distinguish the two; humans triage from logs.
- reasonUndefinedReference
- // reasonAnalyzerUnsupported: non-*catalog.Error internal analyzer failure
- // (returned as fmt.Errorf from analyze.go code paths that hit an
- // unsupported PG feature or a defensive check).
- reasonAnalyzerUnsupported
-)
-
-// classifyAnalyzeError buckets an omni AnalyzeSelectStmt error into one of
-// the fallbackReason categories. Non-nil errors always return a non-None
-// reason; a nil error returns reasonNone.
-func classifyAnalyzeError(err error) fallbackReason {
- if err == nil {
- return reasonNone
- }
- var cErr *catalog.Error
- if !errors.As(err, &cErr) {
- return reasonAnalyzerUnsupported
- }
- switch cErr.Code {
- case catalog.CodeUndefinedFunction,
- catalog.CodeAmbiguousFunction,
- catalog.CodeDatatypeMismatch,
- catalog.CodeFeatureNotSupported,
- catalog.CodeAmbiguousColumn:
- return reasonExpectedPseudoSemantic
- case catalog.CodeUndefinedTable,
- catalog.CodeUndefinedColumn,
- catalog.CodeUndefinedObject,
- catalog.CodeUndefinedSchema:
- return reasonUndefinedReference
- default:
- return reasonAnalyzerUnsupported
- }
-}
-
-// String returns a short lowercase label suitable for test assertions and
-// log lines.
-func (r fallbackReason) String() string {
- switch r {
- case reasonNone:
- return "none"
- case reasonExpectedPseudoSemantic:
- return "expected_pseudo_semantic"
- case reasonUndefinedReference:
- return "undefined_reference"
- case reasonAnalyzerUnsupported:
- return "analyzer_unsupported"
- default:
- return "unknown"
- }
-}
diff --git a/backend/plugin/parser/pg/query_span_loader_integration_test.go b/backend/plugin/parser/pg/query_span_loader_integration_test.go
deleted file mode 100644
index e1e53c3343..0000000000
--- a/backend/plugin/parser/pg/query_span_loader_integration_test.go
+++ /dev/null
@@ -1,671 +0,0 @@
-package pg
-
-import (
- "context"
- "os"
- "os/exec"
- "runtime/debug"
- "testing"
-
- metadatapb "github.com/bytebase/omni/metadata"
- "github.com/bytebase/omni/pg/ast"
- "github.com/bytebase/omni/pg/catalog"
-
- "github.com/bytebase/bytebase/backend/plugin/parser/base"
-)
-
-// ---------------------------------------------------------------------------
-// Catalog loader integration tests
-// ---------------------------------------------------------------------------
-//
-// These tests drive the full GetQuerySpan pipeline (parse → initCatalog via
-// catalogLoader → AnalyzeSelectStmt → extractLineage) against hand-built
-// DatabaseSchemaMetadata fixtures that reproduce the bugs the loader is designed to
-// prevent, plus the cascade-prevention story.
-//
-// Each test asserts one of:
-// - A query against an unrelated object succeeds even when a bad object is
-// present in the schema (BYT-9215 / BYT-9261 class, blast-radius).
-// - A query against a broken-but-pseudo'd object still returns lineage.
-// - A cascade chain (bad root → dependent table → dependent view) installs
-// downstream objects as real, with query lineage intact.
-
-func TestLoaderIntegration_BYT9215_BadQuotedIdentifier(t *testing.T) {
- // BYT-9215 class: a table name contains a character sequence the DDL
- // deparse-and-reparse loop chokes on. Under the old path this would kill
- // query span for the entire database. Under the loader, the bad table is
- // pseudo-installed and unrelated queries succeed.
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Tables: []*metadatapb.TableMetadata{
- // A table with a quoted identifier containing an apostrophe —
- // the kind of input BYT-9215 reported failing under Exec(ddl).
- // Real install should succeed (no DDL roundtrip), but even if
- // it didn't, pseudo would catch it.
- {
- Name: "'weird'table",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- },
- },
- // An unrelated healthy table.
- {
- Name: "accounts",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- {Name: "email", Type: "text"},
- },
- },
- },
- }},
- }
-
- span := mustGetQuerySpan(t, meta, `SELECT id, email FROM accounts`)
- if len(span.Results) != 2 {
- t.Fatalf("accounts query: got %d results, want 2", len(span.Results))
- }
- for _, r := range span.Results {
- if len(r.SourceColumns) == 0 {
- t.Errorf("result %q has empty sources (lineage lost)", r.Name)
- }
- }
-}
-
-func TestGetQuerySpanWithSelectedSchemaFallsBackToPublic(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{
- {
- Name: "app",
- },
- {
- Name: "public",
- Tables: []*metadatapb.TableMetadata{{
- Name: "customer",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "ssn", Type: "text"},
- },
- }},
- },
- },
- }
- getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})
- span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{
- GetDatabaseMetadataFunc: getter,
- ListDatabaseNamesFunc: lister,
- }, base.Statement{Text: `SELECT ssn FROM customer`}, "db", "app", false)
- if err != nil {
- t.Fatalf("GetQuerySpan: %v", err)
- }
- if len(span.Results) != 1 {
- t.Fatalf("got %d results, want 1", len(span.Results))
- }
- want := base.ColumnResource{Database: "db", Schema: "public", Table: "customer", Column: "ssn"}
- if _, ok := span.Results[0].SourceColumns[want]; !ok {
- t.Fatalf("result %q missing public fallback source %+v; have %+v", span.Results[0].Name, want, span.Results[0].SourceColumns)
- }
- wantAccess := base.ColumnResource{Database: "db", Schema: "public", Table: "customer"}
- if _, ok := span.SourceColumns[wantAccess]; !ok {
- t.Fatalf("span missing public fallback access source %+v; have %+v", wantAccess, span.SourceColumns)
- }
-}
-
-func TestGetQuerySpanSelectedSchemaTakesPrecedenceOverPublic(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{
- {
- Name: "app",
- Tables: []*metadatapb.TableMetadata{{
- Name: "customer",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "email", Type: "text"},
- },
- }},
- },
- {
- Name: "public",
- Tables: []*metadatapb.TableMetadata{{
- Name: "customer",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "ssn", Type: "text"},
- },
- }},
- },
- },
- }
- getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})
- span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{
- GetDatabaseMetadataFunc: getter,
- ListDatabaseNamesFunc: lister,
- }, base.Statement{Text: `SELECT email FROM customer`}, "db", "app", false)
- if err != nil {
- t.Fatalf("GetQuerySpan: %v", err)
- }
- if len(span.Results) != 1 {
- t.Fatalf("got %d results, want 1", len(span.Results))
- }
- want := base.ColumnResource{Database: "db", Schema: "app", Table: "customer", Column: "email"}
- if _, ok := span.Results[0].SourceColumns[want]; !ok {
- t.Fatalf("result %q missing selected-schema source %+v; have %+v", span.Results[0].Name, want, span.Results[0].SourceColumns)
- }
-}
-
-func TestLoaderIntegration_BrokenEnumCascade(t *testing.T) {
- // Real failure chain: table references a user-defined type that does
- // not exist in metadata. Real install of the table fails (omni cannot
- // resolve the type). Pseudo install of the table succeeds with text
- // columns. Query against the table returns its metadata column names.
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Tables: []*metadatapb.TableMetadata{{
- Name: "tasks",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- // References an enum that is NOT declared in metadata —
- // buildCreateStmt will succeed (typeNameFromString works)
- // but DefineRelation fails (enum not in catalog).
- {Name: "status", Type: "public.nonexistent_enum"},
- {Name: "title", Type: "text"},
- },
- }},
- }},
- }
-
- span := mustGetQuerySpan(t, meta, `SELECT id, status, title FROM tasks`)
- if len(span.Results) != 3 {
- t.Fatalf("tasks query: got %d results, want 3", len(span.Results))
- }
- wantNames := []string{"id", "status", "title"}
- for i, want := range wantNames {
- if i >= len(span.Results) {
- break
- }
- if span.Results[i].Name != want {
- t.Errorf("result[%d]: got %q, want %q", i, span.Results[i].Name, want)
- }
- if len(span.Results[i].SourceColumns) == 0 {
- t.Errorf("result[%d] (%s): empty sources", i, span.Results[i].Name)
- }
- }
-}
-
-func TestLoaderIntegration_BrokenRootTableAndHealthyNeighbor(t *testing.T) {
- // A query against a healthy table must succeed even when an unrelated
- // table in the same schema references a broken type. This is the core
- // blast-radius claim: one bad object does not poison all queries.
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Tables: []*metadatapb.TableMetadata{
- {
- Name: "broken",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- {Name: "bad", Type: "public.nonexistent_type"},
- },
- },
- {
- Name: "healthy",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- {Name: "label", Type: "text"},
- },
- },
- },
- }},
- }
-
- span := mustGetQuerySpan(t, meta, `SELECT id, label FROM healthy`)
- if len(span.Results) != 2 {
- t.Fatalf("healthy query: got %d results, want 2", len(span.Results))
- }
- for _, r := range span.Results {
- if len(r.SourceColumns) == 0 {
- t.Errorf("result %q lost lineage because of unrelated broken table", r.Name)
- }
- }
-}
-
-func TestLoaderIntegration_ViewOverBrokenTableStillResolves(t *testing.T) {
- // Chain: enum missing → table T references it (degrades to pseudo) →
- // view V on T installs real (against pseudo T) → query on V resolves.
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Tables: []*metadatapb.TableMetadata{{
- Name: "records",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- {Name: "status", Type: "public.nonexistent_enum"},
- {Name: "title", Type: "text"},
- },
- }},
- Views: []*metadatapb.ViewMetadata{{
- Name: "records_view",
- Definition: "SELECT id, title, status FROM records",
- DependencyColumns: []*metadatapb.DependencyColumn{
- {Schema: "public", Table: "records", Column: "id"},
- {Schema: "public", Table: "records", Column: "title"},
- {Schema: "public", Table: "records", Column: "status"},
- },
- }},
- }},
- }
-
- span := mustGetQuerySpan(t, meta, `SELECT id, title, status FROM records_view`)
- if len(span.Results) != 3 {
- t.Fatalf("view query: got %d results, want 3", len(span.Results))
- }
- for i, r := range span.Results {
- if len(r.SourceColumns) == 0 {
- t.Errorf("result[%d] (%s) lost lineage", i, r.Name)
- }
- }
-}
-
-func TestLoaderIntegration_SimpleHealthyPath(t *testing.T) {
- // Baseline sanity: a clean schema must produce exact lineage down to
- // (schema, table, column) — no degraded flags.
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Tables: []*metadatapb.TableMetadata{{
- Name: "users",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- {Name: "email", Type: "text"},
- {Name: "created_at", Type: "timestamp with time zone"},
- },
- }},
- }},
- }
-
- span := mustGetQuerySpan(t, meta, `SELECT id, email FROM users WHERE id > 0`)
- if len(span.Results) != 2 {
- t.Fatalf("got %d results, want 2", len(span.Results))
- }
- mustHaveExactSource(t, span.Results[0], "users", "id")
- mustHaveExactSource(t, span.Results[1], "users", "email")
-}
-
-func TestLoaderIntegration_EnumWorksWhenDeclared(t *testing.T) {
- // When an enum IS declared in metadata, the table installs as real and
- // enum-typed columns resolve through their real type. This is the
- // positive control for TestLoaderIntegration_BrokenEnumCascade.
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- EnumTypes: []*metadatapb.EnumTypeMetadata{{
- Name: "task_status",
- Values: []string{"pending", "running", "done"},
- }},
- Tables: []*metadatapb.TableMetadata{{
- Name: "tasks",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- {Name: "status", Type: "public.task_status"},
- },
- }},
- }},
- }
-
- span := mustGetQuerySpan(t, meta, `SELECT id, status FROM tasks WHERE status = 'pending'`)
- if len(span.Results) != 2 {
- t.Fatalf("got %d results, want 2", len(span.Results))
- }
- mustHaveExactSource(t, span.Results[0], "tasks", "id")
- mustHaveExactSource(t, span.Results[1], "tasks", "status")
-}
-
-func TestLoaderIntegration_CorrelatedRangeFunctionSubquery(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Tables: []*metadatapb.TableMetadata{
- {
- Name: "compliance_case_record_audits",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "case_id", Type: "text"},
- {Name: "entity_id", Type: "text"},
- {Name: "reason", Type: "text"},
- {Name: "remark", Type: "text"},
- {Name: "case_record", Type: "jsonb"},
- {Name: "reviewer_by", Type: "text"},
- {Name: "deleted_at", Type: "timestamptz"},
- },
- },
- {
- Name: "compliance_cases",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "case_id", Type: "text"},
- {Name: "case_info", Type: "jsonb"},
- {Name: "deleted_at", Type: "timestamptz"},
- },
- },
- },
- }},
- }
- sql := `
-select
- a.case_id,
- -- c.member_id,
- a.entity_id as uid,
- a.reason,
- a.remark,
- (
- select elem->>'matchedDateTimeValue'
- from jsonb_array_elements(a.case_record::jsonb->'secondaryFieldResults') elem
- where elem->>'typeId' = '******'
- limit 1
- ) as wc_dob,
- (
- select elem->>'matchedValue'
- from jsonb_array_elements(a.case_record::jsonb->'secondaryFieldResults') elem
- where elem->>'typeId' = '******'
- limit 1
- ) as wc_citizenship,
- c.case_info->>'birth_date' as kyc_dob,
- c.case_info->>'nationality' as kyc_citizenship,
- a.reviewer_by as review_by
-from compliance_case_record_audits a
-inner join compliance_cases c
- on c.case_id = a.case_id and c.deleted_at is null
-where a.reviewer_by in ('****** ', '******')
- and a.deleted_at is null;
-`
- span := mustGetQuerySpan(t, meta, sql)
- if len(span.Results) != 9 {
- t.Fatalf("got %d results, want 9", len(span.Results))
- }
- mustHaveExactSource(t, span.Results[4], "compliance_case_record_audits", "case_record")
- mustHaveExactSource(t, span.Results[5], "compliance_case_record_audits", "case_record")
- mustHaveExactSource(t, span.Results[6], "compliance_cases", "case_info")
- mustHaveExactSource(t, span.Results[7], "compliance_cases", "case_info")
-}
-
-func TestLoaderIntegration_MultipleCTEsWithLateralJoinKeepsJSONBLineage(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "db",
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Tables: []*metadatapb.TableMetadata{{
- Name: "ai_conversation",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "text"},
- {Name: "parts", Type: "jsonb"},
- },
- }},
- }},
- }
-
- tests := []struct {
- name string
- sql string
- }{
- {
- name: "single_cte",
- sql: `
-WITH convs AS (
- SELECT ac.id, ac.parts AS conv_parts
- FROM ai_conversation ac
- LIMIT 1
-)
-SELECT
- LEFT(
- COALESCE(
- (SELECT string_agg(v::text, ' ')
- FROM jsonb_array_elements_text(
- jsonb_path_query_array(t.part->'bodyData', 'strict $.**.text', '{}'::jsonb, true)
- ) AS v),
- '<no text>'
- ),
- 2000
- ) AS content
-FROM convs c
-CROSS JOIN LATERAL jsonb_array_elements(c.conv_parts) WITH ORDINALITY AS t(part, part_ord)
-WHERE t.part->>'type' IN ('prompt', 'text');
-`,
- },
- {
- name: "multiple_ctes",
- sql: `
-WITH cte1 AS (
- SELECT ac.id, ac.parts AS conv_parts
- FROM ai_conversation ac
- LIMIT 1
-),
-cte2 AS (
- SELECT id, conv_parts
- FROM cte1
-)
-SELECT
- LEFT(
- COALESCE(
- (SELECT string_agg(v::text, ' ')
- FROM jsonb_array_elements_text(
- jsonb_path_query_array(t.part->'bodyData', 'strict $.**.text', '{}'::jsonb, true)
- ) AS v),
- '<no text>'
- ),
- 2000
- ) AS content
-FROM cte2 c
-CROSS JOIN LATERAL jsonb_array_elements(c.conv_parts) WITH ORDINALITY AS t(part, part_ord)
-WHERE t.part->>'type' IN ('prompt', 'text');
-`,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- span := mustGetQuerySpan(t, meta, tt.sql)
- if len(span.Results) != 1 {
- t.Fatalf("got %d results, want 1", len(span.Results))
- }
- mustHaveExactSource(t, span.Results[0], "ai_conversation", "parts")
- })
- }
-}
-
-func TestAppendQueryDoesNotMutateSharedStack(t *testing.T) {
- outer := &catalog.Query{}
- parent := &catalog.Query{}
- current := &catalog.Query{}
- nested := &catalog.Query{}
- queryStack := []*catalog.Query{outer, parent, current}
-
- nextStack := appendQuery(queryStack[:2], nested)
-
- if queryStack[2] != current {
- t.Fatalf("appendQuery mutated caller stack: got %p, want %p", queryStack[2], current)
- }
- if len(nextStack) != 3 || nextStack[2] != nested {
- t.Fatalf("appendQuery returned unexpected stack: %+v", nextStack)
- }
-}
-
-func TestLoaderIntegration_BuiltinFunctionSelfReferenceDoesNotOverflow(t *testing.T) {
- if os.Getenv("BYTEBASE_TEST_FUNCTION_SELF_REFERENCE") == "1" {
- debug.SetMaxStack(1 << 20)
- varExpr := &catalog.VarExpr{RangeIdx: 0, AttNum: 1}
- q := &catalog.Query{
- TargetList: []*catalog.TargetEntry{{
- Expr: varExpr,
- }},
- RangeTable: []*catalog.RangeTableEntry{{
- Kind: catalog.RTEFunction,
- FuncExprs: []catalog.AnalyzedExpr{&catalog.FuncCallExpr{
- Args: []catalog.AnalyzedExpr{&catalog.OpExpr{
- Left: varExpr,
- Right: &catalog.ConstExpr{
- Value: "items",
- },
- }},
- }},
- }},
- }
- extractor := newOmniQuerySpanExtractor(loaderTestDB, []string{loaderTestSchema}, base.GetQuerySpanContext{})
- extractor.cat = catalog.New()
- extractor.walkExpr(q, q.TargetList[0].Expr, make(base.SourceColumnSet))
- return
- }
-
- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_BuiltinFunctionSelfReferenceDoesNotOverflow$")
- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_FUNCTION_SELF_REFERENCE=1")
- if output, err := cmd.CombinedOutput(); err != nil {
- t.Fatalf("walkExpr overflowed on self-referential function RTE: %v\n%s", err, output)
- }
-}
-
-func TestLoaderIntegration_PlainColumnCycleDoesNotOverflow(t *testing.T) {
- if os.Getenv("BYTEBASE_TEST_PLAIN_COLUMN_CYCLE") == "1" {
- debug.SetMaxStack(1 << 20)
- varExpr := &catalog.VarExpr{RangeIdx: 0, AttNum: 1}
- q := &catalog.Query{
- TargetList: []*catalog.TargetEntry{{
- Expr: varExpr,
- }},
- RangeTable: []*catalog.RangeTableEntry{{
- Kind: catalog.RTESubquery,
- }},
- }
- q.RangeTable[0].Subquery = q
- _ = isUltimatelyPlainColumn(q, varExpr)
- return
- }
-
- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_PlainColumnCycleDoesNotOverflow$")
- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_PLAIN_COLUMN_CYCLE=1")
- if output, err := cmd.CombinedOutput(); err != nil {
- t.Fatalf("isUltimatelyPlainColumn overflowed on cyclic subquery RTE: %v\n%s", err, output)
- }
-}
-
-func TestLoaderIntegration_PredicateQueryCycleDoesNotOverflow(t *testing.T) {
- if os.Getenv("BYTEBASE_TEST_PREDICATE_QUERY_CYCLE") == "1" {
- debug.SetMaxStack(1 << 20)
- q := &catalog.Query{}
- q.CTEList = []*catalog.CommonTableExprQ{{Query: q}}
- extractor := newOmniQuerySpanExtractor(loaderTestDB, []string{loaderTestSchema}, base.GetQuerySpanContext{})
- analyzer := &plpgsqlAnalyzer{
- extractor: extractor,
- scope: newVariableScope(nil),
- }
- analyzer.collectQueryPredicateColumns(q)
- return
- }
-
- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_PredicateQueryCycleDoesNotOverflow$")
- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_PREDICATE_QUERY_CYCLE=1")
- if output, err := cmd.CombinedOutput(); err != nil {
- t.Fatalf("collectQueryPredicateColumns overflowed on cyclic query graph: %v\n%s", err, output)
- }
-}
-
-func TestLoaderIntegration_SetOpQueryCycleDoesNotOverflow(t *testing.T) {
- if os.Getenv("BYTEBASE_TEST_SET_OP_QUERY_CYCLE") == "1" {
- debug.SetMaxStack(1 << 20)
- q := &catalog.Query{SetOp: catalog.SetOpUnion}
- q.LArg = q
- q.RArg = &catalog.Query{TargetList: []*catalog.TargetEntry{{
- Expr: &catalog.ConstExpr{
- Value: "1",
- },
- }}}
- selStmt := &ast.SelectStmt{}
- selStmt.Larg = selStmt
- selStmt.Rarg = &ast.SelectStmt{}
- extractor := newOmniQuerySpanExtractor(loaderTestDB, []string{loaderTestSchema}, base.GetQuerySpanContext{})
- extractor.extractLineage(q, selStmt)
- return
- }
-
- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_SetOpQueryCycleDoesNotOverflow$")
- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_SET_OP_QUERY_CYCLE=1")
- if output, err := cmd.CombinedOutput(); err != nil {
- t.Fatalf("extractLineage overflowed on cyclic set-op query graph: %v\n%s", err, output)
- }
-}
-
-func TestLoaderIntegration_FallbackCTECycleDoesNotOverflow(t *testing.T) {
- if os.Getenv("BYTEBASE_TEST_FALLBACK_CTE_CYCLE") == "1" {
- debug.SetMaxStack(1 << 20)
- cteSel := &ast.SelectStmt{
- TargetList: &ast.List{Items: []ast.Node{&ast.ResTarget{
- Name: "x",
- Val: &ast.ColumnRef{Fields: &ast.List{Items: []ast.Node{
- &ast.String{Str: "c"},
- &ast.String{Str: "x"},
- }}},
- }}},
- FromClause: &ast.List{Items: []ast.Node{&ast.RangeVar{Relname: "c"}}},
- }
- extractor := newOmniQuerySpanExtractor(loaderTestDB, []string{loaderTestSchema}, base.GetQuerySpanContext{})
- extractor.cat = catalog.New()
- analyzer := &plpgsqlAnalyzer{
- extractor: extractor,
- scope: newVariableScope(nil),
- cteMap: map[string]*ast.SelectStmt{"c": cteSel},
- }
- analyzer.resolveThroughCTE(cteSel, "x", nil, make(base.SourceColumnSet))
- return
- }
-
- cmd := exec.Command(os.Args[0], "-test.run=^TestLoaderIntegration_FallbackCTECycleDoesNotOverflow$")
- cmd.Env = append(os.Environ(), "BYTEBASE_TEST_FALLBACK_CTE_CYCLE=1")
- if output, err := cmd.CombinedOutput(); err != nil {
- t.Fatalf("resolveThroughCTE overflowed on cyclic fallback CTE: %v\n%s", err, output)
- }
-}
-
-// ---------- helpers ----------
-
-// loaderTestDB / loaderTestSchema are the fixed database/schema names
-// every catalog-loader integration test uses; keeping them as constants keeps the call
-// sites terse and satisfies the unparam linter.
-const (
- loaderTestDB = "db"
- loaderTestSchema = "public"
-)
-
-func mustGetQuerySpan(t *testing.T, meta *metadatapb.DatabaseSchemaMetadata, sql string) *base.QuerySpan {
- t.Helper()
- getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})
- span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{
- GetDatabaseMetadataFunc: getter,
- ListDatabaseNamesFunc: lister,
- }, base.Statement{Text: sql}, loaderTestDB, "", false)
- if err != nil {
- t.Fatalf("GetQuerySpan(%q): %v", sql, err)
- }
- if span == nil {
- t.Fatalf("GetQuerySpan(%q): nil span", sql)
- }
- return span
-}
-
-func mustHaveExactSource(t *testing.T, result base.QuerySpanResult, table, column string) {
- t.Helper()
- want := base.ColumnResource{
- Database: loaderTestDB,
- Schema: loaderTestSchema,
- Table: table,
- Column: column,
- }
- for src := range result.SourceColumns {
- if src == want {
- return
- }
- }
- t.Errorf("result %q missing source %+v; have %+v", result.Name, want, result.SourceColumns)
-}
diff --git a/backend/plugin/parser/pg/query_span_loader_poc_test.go b/backend/plugin/parser/pg/query_span_loader_poc_test.go
deleted file mode 100644
index 389dc9246f..0000000000
--- a/backend/plugin/parser/pg/query_span_loader_poc_test.go
+++ /dev/null
@@ -1,2527 +0,0 @@
-package pg
-
-// v5 PoC — AST-only catalog population.
-//
-// This PoC demonstrates that we can:
-// 1. Construct a *ast.CreateStmt directly in Go code (no SQL text, no parser).
-// 2. Call catalog.DefineRelation on it to install a table into an omni catalog.
-// 3. Parse a SELECT query against that table.
-// 4. Call AnalyzeSelectStmt and get correct column lineage.
-//
-// The entire "table install" half of the pipeline never touches pgparser.Parse,
-// which is the source of BYT-9215 / BYT-9261 class failures. Only the SELECT
-// statement itself is parsed.
-//
-// If this PoC passes, v5 ("lazy per-object AST install") is the recommended
-// architecture for PG query span and supersedes v2/v3/v4 plans.
-
-import (
- "fmt"
- "strings"
- "testing"
-
- "github.com/pkg/errors"
-
- "github.com/bytebase/omni/pg/ast"
- "github.com/bytebase/omni/pg/catalog"
-)
-
-// mustTypeName is a test-only shim around the production typeNameFromString.
-// It fails the test on any parse error; production code uses typeNameFromString
-// directly and handles the error.
-func mustTypeName(t *testing.T, typeStr string) *ast.TypeName {
- t.Helper()
- tn, err := typeNameFromString(typeStr)
- if err != nil {
- t.Fatalf("typeNameFromString(%q): %v", typeStr, err)
- }
- return tn
-}
-
-// typeName builds an unqualified PG type name AST node.
-// For built-in types, use the PG internal name: int -> int4, text -> text,
-// bigint -> int8, varchar -> varchar, timestamp -> timestamp.
-func typeName(name string) *ast.TypeName {
- return &ast.TypeName{
- Names: &ast.List{Items: []ast.Node{&ast.String{Str: name}}},
- Typemod: -1,
- }
-}
-
-// columnDef builds a minimal ColumnDef AST node.
-func columnDef(name string, typ *ast.TypeName, notNull bool) *ast.ColumnDef {
- return &ast.ColumnDef{
- Colname: name,
- TypeName: typ,
- IsNotNull: notNull,
- }
-}
-
-// buildCreateTableStmt constructs a *ast.CreateStmt for a simple table,
-// purely from Go code with no SQL text. This is the shape that a v5
-// catalog loader would produce from bytebase metadata.
-//
-// parameter signals intent and keeps the helper ready for cross-schema tests.
-//
-//nolint:unparam // schema is always "public" in these PoCs; keeping the
-func buildCreateTableStmt(schema, name string, cols []*ast.ColumnDef) *ast.CreateStmt {
- nodes := make([]ast.Node, len(cols))
- for i, c := range cols {
- nodes[i] = c
- }
- return &ast.CreateStmt{
- Relation: &ast.RangeVar{
- Schemaname: schema,
- Relname: name,
- Relpersistence: 'p',
- },
- TableElts: &ast.List{Items: nodes},
- }
-}
-
-// TestLoaderPoC_AstOnlyTableInstallAndAnalyze is the main PoC.
-//
-// Run with:
-//
-// go test -v -count=1 -run TestLoaderPoC github.com/bytebase/bytebase/backend/plugin/parser/pg
-func TestLoaderPoC_AstOnlyTableInstallAndAnalyze(t *testing.T) {
- // Step 1: fresh catalog. public / pg_catalog / pg_toast are preloaded.
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Step 2: hand-construct CREATE TABLE public.plan (
- // id int NOT NULL,
- // title text,
- // status text,
- // creator_id int
- // ) — no SQL string involved.
- planStmt := buildCreateTableStmt("public", "plan", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("title", typeName("text"), false),
- columnDef("status", typeName("text"), false),
- columnDef("creator_id", typeName("int4"), false),
- })
-
- // Step 3: install via DefineRelation. Second arg 'r' = regular table.
- if err := cat.DefineRelation(planStmt, 'r'); err != nil {
- t.Fatalf("DefineRelation(plan) failed: %v", err)
- }
-
- // Step 4: install a second table to test JOINs too.
- principalStmt := buildCreateTableStmt("public", "principal", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("name", typeName("text"), false),
- columnDef("email", typeName("text"), false),
- })
- if err := cat.DefineRelation(principalStmt, 'r'); err != nil {
- t.Fatalf("DefineRelation(principal) failed: %v", err)
- }
-
- // Step 5: verify catalog state — both tables resolvable by name.
- if rel := cat.GetRelation("public", "plan"); rel == nil {
- t.Fatal("GetRelation(public, plan) returned nil after DefineRelation")
- } else if len(rel.Columns) != 4 {
- t.Errorf("plan column count: got %d, want 4", len(rel.Columns))
- }
- if rel := cat.GetRelation("public", "principal"); rel == nil {
- t.Fatal("GetRelation(public, principal) returned nil after DefineRelation")
- }
-
- // Step 6: parse a SELECT query that touches both tables.
- // The SELECT parser is used here — that's expected and safe (it's not
- // the DDL parser that breaks in BYT-9215/9261).
- stmts, err := ParsePg(`
- SELECT p.id, p.title, principal.name
- FROM plan p
- JOIN principal ON principal.id = p.creator_id
- WHERE p.status = 'open'
- `)
- if err != nil {
- t.Fatalf("ParsePg failed: %v", err)
- }
- if len(stmts) != 1 {
- t.Fatalf("expected 1 statement, got %d", len(stmts))
- }
- selStmt, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- t.Fatalf("expected *ast.SelectStmt, got %T", stmts[0].AST)
- }
-
- // Step 7: THE KEY CALL — analyze the SELECT against the hand-installed
- // catalog. If this returns a non-nil Query with a TargetList matching
- // the three SELECT columns, v5 architecture is viable.
- query, err := cat.AnalyzeSelectStmt(selStmt)
- if err != nil {
- t.Fatalf("AnalyzeSelectStmt failed: %v", err)
- }
- if query == nil {
- t.Fatal("AnalyzeSelectStmt returned nil query")
- }
-
- // Step 8: assert lineage is sane — 3 target columns.
- if got := len(query.TargetList); got != 3 {
- t.Errorf("TargetList length: got %d, want 3", got)
- for i, te := range query.TargetList {
- t.Logf(" [%d] %s", i, te.ResName)
- }
- }
-
- // Step 9: assert each target is a VarExpr (plain column reference)
- // pointing at the expected column names.
- wantNames := []string{"id", "title", "name"}
- for i, want := range wantNames {
- if i >= len(query.TargetList) {
- break
- }
- te := query.TargetList[i]
- if te.ResName != want {
- t.Errorf("TargetList[%d].ResName: got %q, want %q", i, te.ResName, want)
- }
- if _, isVar := te.Expr.(*catalog.VarExpr); !isVar {
- t.Errorf("TargetList[%d].Expr: got %T, want *VarExpr", i, te.Expr)
- }
- }
-
- // Step 10: verify JoinTree.Quals is analyzed (WHERE was walked).
- // We don't assert exact shape here — just that analysis didn't bail.
- if query.JoinTree == nil {
- t.Error("JoinTree is nil — analyzer did not build a join tree")
- } else if query.JoinTree.Quals == nil {
- t.Error("JoinTree.Quals is nil — WHERE clause was not analyzed")
- }
-
- t.Logf("v5 PoC passed: hand-constructed CreateStmt → DefineRelation → " +
- "AnalyzeSelectStmt pipeline works end-to-end with no DDL text.")
-}
-
-// TestLoaderPoC_BrokenAstIsIsolated demonstrates blast-radius containment:
-// installing a table with an unresolvable type fails locally, and queries
-// against OTHER tables continue to work. This is the core fidelity claim
-// v5 makes over the current whole-schema DDL init path.
-func TestLoaderPoC_BrokenAstIsIsolated(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Install a good table.
- goodStmt := buildCreateTableStmt("public", "good", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("label", typeName("text"), false),
- })
- if err := cat.DefineRelation(goodStmt, 'r'); err != nil {
- t.Fatalf("DefineRelation(good) failed: %v", err)
- }
-
- // Try to install a broken table — column type does not exist.
- brokenStmt := buildCreateTableStmt("public", "broken", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("payload", typeName("nonexistent_type_xyz"), false),
- })
- err := cat.DefineRelation(brokenStmt, 'r')
- if err == nil {
- t.Log("note: DefineRelation accepted unresolvable type without error")
- t.Log(" this is informational — the analyzer may still reject it at query time")
- } else {
- t.Logf("DefineRelation(broken) rejected as expected: %v", err)
- }
-
- // Key assertion: the good table is still queryable, regardless of
- // whether the broken table was accepted.
- if rel := cat.GetRelation("public", "good"); rel == nil {
- t.Fatal("good table was lost after broken install attempt")
- }
-
- stmts, err := ParsePg(`SELECT id, label FROM good WHERE id = 1`)
- if err != nil {
- t.Fatalf("ParsePg failed: %v", err)
- }
- selStmt, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- t.Fatalf("expected *ast.SelectStmt, got %T", stmts[0].AST)
- }
-
- query, err := cat.AnalyzeSelectStmt(selStmt)
- if err != nil {
- t.Fatalf("AnalyzeSelectStmt on good table failed after broken install: %v", err)
- }
- if query == nil || len(query.TargetList) != 2 {
- t.Errorf("expected 2 target columns on good table query, got %v", query)
- }
-
- t.Log("v5 PoC blast-radius isolation: broken table did not break queries " +
- "against unrelated good table.")
-}
-
-// ---------- Risk scenario 1: complex typmods (numeric(10,2)) ----------
-
-func TestLoaderPoC_ComplexTypmod(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Build a column with numeric(10,2) via the cheat parser.
- stmt := buildCreateTableStmt("public", "orders", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),
- columnDef("ts", mustTypeName(t, "timestamp(3) with time zone"), false),
- })
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation with complex typmods failed: %v", err)
- }
-
- rel := cat.GetRelation("public", "orders")
- if rel == nil || len(rel.Columns) != 3 {
- t.Fatalf("orders install incomplete: %+v", rel)
- }
-
- // Query the table — analyzer must resolve each column.
- q := mustAnalyze(t, cat, `SELECT id, amount, ts FROM orders WHERE amount > 0`)
- if len(q.TargetList) != 3 {
- t.Errorf("TargetList: got %d, want 3", len(q.TargetList))
- }
- t.Log("complex typmod PoC: numeric(10,2) and timestamp(3) with time zone resolve via cheat parser")
-}
-
-// ---------- Risk scenario 2: array type ----------
-
-func TestLoaderPoC_ArrayType(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- stmt := buildCreateTableStmt("public", "tagged", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("tags", mustTypeName(t, "text[]"), false),
- columnDef("scores", mustTypeName(t, "int4[]"), false),
- })
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation with array columns failed: %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT id, tags FROM tagged`)
- if len(q.TargetList) != 2 {
- t.Errorf("TargetList: got %d, want 2", len(q.TargetList))
- }
- t.Log("array type PoC: text[] and int4[] resolve via cheat parser")
-}
-
-// ---------- Risk scenario 3: SELECT * expansion ----------
-
-func TestLoaderPoC_StarExpansion(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- if err := cat.DefineRelation(buildCreateTableStmt("public", "plan", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("title", typeName("text"), false),
- columnDef("status", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("install: %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT * FROM plan`)
- // With a 3-column table, SELECT * should expand to 3 target entries.
- if len(q.TargetList) != 3 {
- t.Errorf("SELECT * expansion: got %d targets, want 3", len(q.TargetList))
- }
- wantCols := []string{"id", "title", "status"}
- for i, want := range wantCols {
- if i >= len(q.TargetList) {
- break
- }
- if q.TargetList[i].ResName != want {
- t.Errorf("target[%d]: got %q, want %q", i, q.TargetList[i].ResName, want)
- }
- }
- t.Log("SELECT * expansion PoC: analyzer expands star against hand-installed columns")
-}
-
-// ---------- Risk scenario 4: CTE (WITH clause) ----------
-
-func TestLoaderPoC_CTE(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- if err := cat.DefineRelation(buildCreateTableStmt("public", "users", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("name", typeName("text"), false),
- columnDef("dept", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("install: %v", err)
- }
-
- q := mustAnalyze(t, cat, `
- WITH active AS (
- SELECT id, name FROM users WHERE dept = 'eng'
- )
- SELECT id, name FROM active
- `)
- if len(q.TargetList) != 2 {
- t.Errorf("CTE analyze: got %d targets, want 2", len(q.TargetList))
- }
- t.Log("CTE PoC: WITH clause analyzed against hand-installed table")
-}
-
-// ---------- Risk scenario 5: correlated subquery ----------
-
-func TestLoaderPoC_CorrelatedSubquery(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- if err := cat.DefineRelation(buildCreateTableStmt("public", "dept", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("name", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("install dept: %v", err)
- }
- if err := cat.DefineRelation(buildCreateTableStmt("public", "emp", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("dept_id", typeName("int4"), false),
- columnDef("salary", typeName("int4"), false),
- }), 'r'); err != nil {
- t.Fatalf("install emp: %v", err)
- }
-
- q := mustAnalyze(t, cat, `
- SELECT d.name,
- (SELECT MAX(e.salary) FROM emp e WHERE e.dept_id = d.id) AS max_sal
- FROM dept d
- `)
- if len(q.TargetList) != 2 {
- t.Errorf("correlated: got %d targets, want 2", len(q.TargetList))
- }
- t.Log("correlated subquery PoC: outer scope d.id resolved from inner subquery")
-}
-
-// ---------- Risk scenario 6: user enum type in column ----------
-
-func TestLoaderPoC_UserEnumType(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Step 1: install the enum via DefineEnum with hand-constructed stmt.
- enumStmt := &ast.CreateEnumStmt{
- TypeName: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "task_status"},
- }},
- Vals: &ast.List{Items: []ast.Node{
- &ast.String{Str: "pending"},
- &ast.String{Str: "running"},
- &ast.String{Str: "done"},
- }},
- }
- if err := cat.DefineEnum(enumStmt); err != nil {
- t.Fatalf("DefineEnum failed: %v", err)
- }
-
- // Step 2: install a table whose column uses the enum.
- // We go through the cheat parser so the TypeName carries the qualified
- // name format omni expects for user types.
- stmt := buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("status", mustTypeName(t, "public.task_status"), false),
- })
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation with enum column failed: %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT id, status FROM tasks WHERE status = 'pending'`)
- if len(q.TargetList) != 2 {
- t.Errorf("enum-col query: got %d targets, want 2", len(q.TargetList))
- }
- t.Log("user enum PoC: DefineEnum + DefineRelation with enum-typed column + WHERE filter works")
-}
-
-// ---------- Risk scenario 7: view via DefineView with parsed body ----------
-
-func TestLoaderPoC_DefineViewWithParsedBody(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Install an underlying table.
- if err := cat.DefineRelation(buildCreateTableStmt("public", "plan", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("title", typeName("text"), false),
- columnDef("creator_id", typeName("int4"), false),
- }), 'r'); err != nil {
- t.Fatalf("install plan: %v", err)
- }
-
- // View body is stored as text in bytebase metadata. Parse it once.
- viewBody := `SELECT id, title FROM plan WHERE creator_id = 1`
- stmts, err := ParsePg(viewBody)
- if err != nil {
- t.Fatalf("parse view body: %v", err)
- }
- selStmt, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- t.Fatalf("view body: expected SelectStmt, got %T", stmts[0].AST)
- }
-
- // Construct ViewStmt and install via DefineView.
- viewStmt := &ast.ViewStmt{
- View: &ast.RangeVar{
- Schemaname: "public",
- Relname: "plan_mine",
- Relpersistence: 'p',
- },
- Query: selStmt,
- }
- if err := cat.DefineView(viewStmt); err != nil {
- t.Fatalf("DefineView failed: %v", err)
- }
-
- rel := cat.GetRelation("public", "plan_mine")
- if rel == nil {
- t.Fatal("view not installed")
- }
- if rel.RelKind != 'v' {
- t.Errorf("RelKind: got %c, want 'v'", rel.RelKind)
- }
-
- // Query the view. analyzer should be able to expand the view.
- q := mustAnalyze(t, cat, `SELECT id, title FROM plan_mine`)
- if len(q.TargetList) != 2 {
- t.Errorf("query on view: got %d targets, want 2", len(q.TargetList))
- }
- t.Log("DefineView PoC: parsed-body ViewStmt + DefineView + query on view works")
-}
-
-// ---------- Risk scenario 8: nested view (view on view) ----------
-
-func TestLoaderPoC_NestedView(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("user_id", typeName("int4"), false),
- columnDef("amount", typeName("int4"), false),
- }), 'r'); err != nil {
- t.Fatalf("install orders: %v", err)
- }
-
- // View 1: orders_by_user
- body1 := `SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id`
- sel1 := mustParseSelect(t, body1)
- if err := cat.DefineView(&ast.ViewStmt{
- View: &ast.RangeVar{Schemaname: "public", Relname: "orders_by_user", Relpersistence: 'p'},
- Query: sel1,
- }); err != nil {
- t.Fatalf("DefineView v1: %v", err)
- }
-
- // View 2: top spenders — references view 1.
- body2 := `SELECT user_id, total FROM orders_by_user WHERE total > 100`
- sel2 := mustParseSelect(t, body2)
- if err := cat.DefineView(&ast.ViewStmt{
- View: &ast.RangeVar{Schemaname: "public", Relname: "top_spenders", Relpersistence: 'p'},
- Query: sel2,
- }); err != nil {
- t.Fatalf("DefineView v2 (nested): %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT user_id, total FROM top_spenders`)
- if len(q.TargetList) != 2 {
- t.Errorf("nested view query: got %d targets, want 2", len(q.TargetList))
- }
- t.Log("nested view PoC: view-on-view resolves correctly via DefineView chain")
-}
-
-// ---------- helpers ----------
-
-func mustAnalyze(t *testing.T, cat *catalog.Catalog, sql string) *catalog.Query {
- t.Helper()
- stmts, err := ParsePg(sql)
- if err != nil {
- t.Fatalf("parse %q: %v", sql, err)
- }
- selStmt, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- t.Fatalf("expected SelectStmt, got %T", stmts[0].AST)
- }
- q, err := cat.AnalyzeSelectStmt(selStmt)
- if err != nil {
- t.Fatalf("AnalyzeSelectStmt(%q): %v", sql, err)
- }
- return q
-}
-
-func mustParseSelect(t *testing.T, sql string) *ast.SelectStmt {
- t.Helper()
- stmts, err := ParsePg(sql)
- if err != nil {
- t.Fatalf("parse %q: %v", sql, err)
- }
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- t.Fatalf("expected SelectStmt, got %T", stmts[0].AST)
- }
- return sel
-}
-
-// ---------- Risk scenario 9: typeNameFromString with quoted/reserved identifier ----------
-
-func TestLoaderPoC_QuotedIdentifierType(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Create an enum with a reserved-ish name.
- enumStmt := &ast.CreateEnumStmt{
- TypeName: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "user"}, // "user" is a reserved word in PG
- }},
- Vals: &ast.List{Items: []ast.Node{
- &ast.String{Str: "admin"},
- &ast.String{Str: "member"},
- }},
- }
- if err := cat.DefineEnum(enumStmt); err != nil {
- t.Fatalf("DefineEnum for reserved-word enum failed: %v", err)
- }
-
- // The cheat parser should be able to resolve this through SELECT NULL::"public"."user".
- // Note: caller is responsible for quoting — this is documented in typeNameFromString.
- tn := mustTypeName(t, `"public"."user"`)
- if tn == nil {
- t.Fatal("typeNameFromString returned nil for quoted reserved-word type")
- }
-
- // And we can use it in a CREATE TABLE.
- stmt := buildCreateTableStmt("public", "accts", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("role", tn, false),
- })
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation with quoted-reserved-word enum column failed: %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT id, role FROM accts`)
- if len(q.TargetList) != 2 {
- t.Errorf("quoted-identifier enum query: got %d, want 2", len(q.TargetList))
- }
- t.Log("quoted identifier PoC: cheat parser handles reserved-word-adjacent type names")
-}
-
-// ---------- Risk scenario 10: partitioned table ----------
-
-func TestLoaderPoC_PartitionedTable(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Construct CREATE TABLE sales (region text, amount int4) PARTITION BY LIST (region).
- stmt := &ast.CreateStmt{
- Relation: &ast.RangeVar{
- Schemaname: "public",
- Relname: "sales",
- Relpersistence: 'p',
- },
- TableElts: &ast.List{Items: []ast.Node{
- columnDef("region", typeName("text"), false),
- columnDef("amount", typeName("int4"), false),
- }},
- Partspec: &ast.PartitionSpec{
- Strategy: "l", // list
- PartParams: &ast.List{Items: []ast.Node{
- &ast.PartitionElem{Name: "region"},
- }},
- },
- }
- // Note: omni test convention passes 'r' even for partitioned tables;
- // omni flips relkind internally based on Partspec presence.
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation partitioned failed: %v", err)
- }
-
- rel := cat.GetRelation("public", "sales")
- if rel == nil {
- t.Fatal("partitioned table not installed")
- }
-
- q := mustAnalyze(t, cat, `SELECT region, amount FROM sales WHERE region = 'us'`)
- if len(q.TargetList) != 2 {
- t.Errorf("partitioned table query: got %d, want 2", len(q.TargetList))
- }
- t.Logf("partitioned table PoC: DefineRelation with Partspec works; relation RelKind=%c", rel.RelKind)
-}
-
-// ---------- Risk scenario 11: inherited table ----------
-
-func TestLoaderPoC_InheritedTable(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Parent with 2 columns.
- parentStmt := buildCreateTableStmt("public", "parent", []*ast.ColumnDef{
- columnDef("a", typeName("int4"), false),
- columnDef("b", typeName("text"), false),
- })
- if err := cat.DefineRelation(parentStmt, 'r'); err != nil {
- t.Fatalf("install parent failed: %v", err)
- }
-
- // Child with 1 locally-defined column, inheriting from parent.
- childStmt := &ast.CreateStmt{
- Relation: &ast.RangeVar{
- Schemaname: "public",
- Relname: "child",
- Relpersistence: 'p',
- },
- InhRelations: &ast.List{Items: []ast.Node{
- &ast.RangeVar{Schemaname: "public", Relname: "parent"},
- }},
- TableElts: &ast.List{Items: []ast.Node{
- columnDef("c", typeName("int4"), false),
- }},
- }
- if err := cat.DefineRelation(childStmt, 'r'); err != nil {
- t.Fatalf("install child with InhRelations failed: %v", err)
- }
-
- child := cat.GetRelation("public", "child")
- if child == nil {
- t.Fatal("child relation not installed")
- }
- // Child should have inherited parent's columns plus its own local column.
- if len(child.Columns) < 3 {
- t.Errorf("child column count: got %d, want >= 3 (a, b inherited + c local)", len(child.Columns))
- for _, col := range child.Columns {
- t.Logf(" col: %s", col.Name)
- }
- }
-
- // Query should resolve all inherited + local columns.
- q := mustAnalyze(t, cat, `SELECT a, b, c FROM child`)
- if len(q.TargetList) != 3 {
- t.Errorf("inherited query: got %d targets, want 3", len(q.TargetList))
- }
- t.Log("inherited table PoC: InhRelations merges parent columns into child")
-}
-
-// ---------- Risk scenario 12: materialized view via ExecCreateTableAs ----------
-
-func TestLoaderPoC_MaterializedView(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("val", typeName("int4"), false),
- }), 'r'); err != nil {
- t.Fatalf("install t: %v", err)
- }
-
- // Parse the matview body — same pattern as DefineView.
- body := `SELECT id, val * 2 AS doubled FROM t WHERE val > 0`
- selStmt := mustParseSelect(t, body)
-
- stmt := &ast.CreateTableAsStmt{
- Query: selStmt,
- Objtype: ast.OBJECT_MATVIEW,
- Into: &ast.IntoClause{
- Rel: &ast.RangeVar{
- Schemaname: "public",
- Relname: "t_doubled",
- Relpersistence: 'p',
- },
- },
- }
- if err := cat.ExecCreateTableAs(stmt); err != nil {
- t.Fatalf("ExecCreateTableAs failed: %v", err)
- }
-
- rel := cat.GetRelation("public", "t_doubled")
- if rel == nil {
- t.Fatal("matview not installed")
- }
- if rel.RelKind != 'm' {
- t.Errorf("matview RelKind: got %c, want 'm'", rel.RelKind)
- }
-
- q := mustAnalyze(t, cat, `SELECT id, doubled FROM t_doubled`)
- if len(q.TargetList) != 2 {
- t.Errorf("matview query: got %d, want 2", len(q.TargetList))
- }
- t.Log("matview PoC: ExecCreateTableAs with OBJECT_MATVIEW works; query on matview analyzes")
-}
-
-// ---------- Risk scenario 13: function + overload via CreateFunctionStmt ----------
-
-func TestLoaderPoC_FunctionOverload(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Helper to build a CreateFunctionStmt — mirrors omni's makeCreateFuncStmt
- // but uses explicit AST nodes.
- makeFuncStmt := func(schema, name string, argTypeNames []string, retTypeName, language, body string) *ast.CreateFunctionStmt {
- var nameItems []ast.Node
- if schema != "" {
- nameItems = append(nameItems, &ast.String{Str: schema})
- }
- nameItems = append(nameItems, &ast.String{Str: name})
-
- var params []ast.Node
- for _, argTypeStr := range argTypeNames {
- params = append(params, &ast.FunctionParameter{
- ArgType: typeName(argTypeStr),
- Mode: ast.FUNC_PARAM_IN,
- })
- }
-
- stmt := &ast.CreateFunctionStmt{
- Funcname: &ast.List{Items: nameItems},
- ReturnType: typeName(retTypeName),
- }
- if len(params) > 0 {
- stmt.Parameters = &ast.List{Items: params}
- }
-
- var opts []ast.Node
- if language != "" {
- opts = append(opts, &ast.DefElem{Defname: "language", Arg: &ast.String{Str: language}})
- }
- if body != "" {
- opts = append(opts, &ast.DefElem{
- Defname: "as",
- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: body}}},
- })
- }
- if len(opts) > 0 {
- stmt.Options = &ast.List{Items: opts}
- }
- return stmt
- }
-
- // Install overload 1: my_double(int4) returns int4.
- err := cat.CreateFunctionStmt(makeFuncStmt("public", "my_double",
- []string{"int4"}, "int4", "sql", "SELECT $1 * 2"))
- if err != nil {
- t.Fatalf("install my_double(int4) failed: %v", err)
- }
-
- // Install overload 2: my_double(int8) returns int8. Different signature,
- // must coexist (this is codex's D3 verification — sequential
- // CreateFunctionStmt calls install all overloads).
- err = cat.CreateFunctionStmt(makeFuncStmt("public", "my_double",
- []string{"int8"}, "int8", "sql", "SELECT $1 * 2"))
- if err != nil {
- t.Fatalf("install my_double(int8) failed: %v", err)
- }
-
- // Both overloads must be visible.
- procs := cat.LookupProcByName("my_double")
- if len(procs) < 2 {
- t.Fatalf("overload count: got %d, want >= 2", len(procs))
- }
-
- // Install a table and query using the function.
- if err := cat.DefineRelation(buildCreateTableStmt("public", "nums", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("n", typeName("int4"), false),
- }), 'r'); err != nil {
- t.Fatalf("install nums: %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT id, my_double(n) AS doubled FROM nums`)
- if len(q.TargetList) != 2 {
- t.Errorf("function query: got %d, want 2", len(q.TargetList))
- }
- t.Logf("function overload PoC: 2 overloads installed sequentially, analyzer picks correctly. %d procs visible under 'my_double'", len(procs))
-}
-
-// ---------- Risk scenario 14: composite type + column referencing it ----------
-
-func TestLoaderPoC_CompositeType(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Install a composite type: address (street text, city text, zip text).
- compStmt := &ast.CompositeTypeStmt{
- Typevar: &ast.RangeVar{
- Schemaname: "public",
- Relname: "address",
- },
- Coldeflist: &ast.List{Items: []ast.Node{
- columnDef("street", typeName("text"), false),
- columnDef("city", typeName("text"), false),
- columnDef("zip", typeName("text"), false),
- }},
- }
- if err := cat.DefineCompositeType(compStmt); err != nil {
- t.Fatalf("DefineCompositeType failed: %v", err)
- }
-
- // Install a table with a column of the composite type.
- stmt := buildCreateTableStmt("public", "customers", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("addr", mustTypeName(t, "public.address"), false),
- })
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation with composite column failed: %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT id, addr FROM customers`)
- if len(q.TargetList) != 2 {
- t.Errorf("composite-col query: got %d, want 2", len(q.TargetList))
- }
- t.Log("composite type PoC: DefineCompositeType + DefineRelation with composite column works")
-}
-
-// ---------- Risk scenario 15: domain type + column referencing it ----------
-
-func TestLoaderPoC_DomainType(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Install a domain: positive_int, base int4, NOT NULL.
- domainStmt := &ast.CreateDomainStmt{
- Domainname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "positive_int"},
- }},
- Typname: typeName("int4"),
- Constraints: &ast.List{Items: []ast.Node{
- &ast.Constraint{Contype: ast.CONSTR_NOTNULL},
- }},
- }
- if err := cat.DefineDomain(domainStmt); err != nil {
- t.Fatalf("DefineDomain failed: %v", err)
- }
-
- // Install a table with a column of the domain type.
- stmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("count", mustTypeName(t, "public.positive_int"), true),
- })
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation with domain column failed: %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT id, count FROM counters WHERE count > 0`)
- if len(q.TargetList) != 2 {
- t.Errorf("domain-col query: got %d, want 2", len(q.TargetList))
- }
- t.Log("domain type PoC: DefineDomain + DefineRelation with domain column works")
-}
-
-// ---------- Exploration: does query span actually need real types? ----------
-//
-// Hypothesis: for query span purposes, we only need column NAMES, not real
-// column TYPES. Lineage is a name-resolution problem, not a type-resolution
-// problem. If we install every column as "text" (regardless of real metadata
-// type), most queries should still analyze correctly and produce identical
-// lineage output.
-//
-// Why this matters: if true, v5 can skip typeNameFromString entirely, skip
-// the user enum / domain / composite problem entirely, and reduce the
-// builder function library to a trivial "table with column names" shape.
-//
-// This test runs a battery of queries against tables where all columns are
-// declared as text, and asks: (a) does AnalyzeSelectStmt succeed, (b) is the
-// lineage output the same as with correct types? A query that fails here
-// would fall through to extractFallbackColumns today (same as the current
-// path does on any analyzer error).
-
-func TestLoaderPoC_TypesDontMatterForLineage(t *testing.T) {
- // Build two catalogs: one with "honest" types, one with everything as text.
- // Run the same queries against both and compare lineage.
-
- installHonest := func(cat *catalog.Catalog) {
- // orders (id int, amount numeric(10,2), status text, placed_at timestamptz, tags text[])
- if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),
- columnDef("status", typeName("text"), false),
- columnDef("placed_at", mustTypeName(t, "timestamp with time zone"), false),
- columnDef("tags", mustTypeName(t, "text[]"), false),
- }), 'r'); err != nil {
- t.Fatalf("honest install: %v", err)
- }
- // customers (id int, email text)
- if err := cat.DefineRelation(buildCreateTableStmt("public", "customers", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("email", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("honest install customers: %v", err)
- }
- }
-
- installLying := func(cat *catalog.Catalog) {
- // Everything text, everything. Even numeric/timestamp/array columns.
- // This is the minimum information the loader would need if we decided
- // bytebase metadata should not bother with per-column type strings.
- if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{
- columnDef("id", typeName("text"), true),
- columnDef("amount", typeName("text"), false),
- columnDef("status", typeName("text"), false),
- columnDef("placed_at", typeName("text"), false),
- columnDef("tags", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("lying install: %v", err)
- }
- if err := cat.DefineRelation(buildCreateTableStmt("public", "customers", []*ast.ColumnDef{
- columnDef("id", typeName("text"), true),
- columnDef("email", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("lying install customers: %v", err)
- }
- }
-
- catHonest := catalog.New()
- catHonest.SetSearchPath([]string{"public"})
- installHonest(catHonest)
-
- catLying := catalog.New()
- catLying.SetSearchPath([]string{"public"})
- installLying(catLying)
-
- // Each test case: a query, a description, an "honest expected to pass" flag.
- // We record (for each catalog) whether AnalyzeSelectStmt succeeded and what
- // the lineage looked like, then compare.
- cases := []struct {
- name string
- query string
- // expectedHonest is whether we expect the honest catalog to analyze successfully.
- // All our cases are expected to succeed on the honest catalog.
- }{
- // Category 1: Name resolution only — no operator semantics involved
- {"plain_select", `SELECT id, status FROM orders`},
- {"select_star", `SELECT * FROM orders`},
- {"join", `SELECT o.id, c.email FROM orders o JOIN customers c ON c.id = o.id`},
- {"alias", `SELECT o.id AS order_id FROM orders o`},
-
- // Category 2: text-friendly operators (=, LIKE, IS NULL)
- {"text_equality", `SELECT id FROM orders WHERE status = 'open'`},
- {"text_like", `SELECT id FROM orders WHERE status LIKE 'op%'`},
- {"is_null", `SELECT id FROM orders WHERE status IS NULL`},
-
- // Category 3: numeric operators against text — will omni coerce or fail?
- {"numeric_compare", `SELECT id FROM orders WHERE amount > 100`},
- {"numeric_arithmetic", `SELECT id, amount + 1 AS amount_plus FROM orders`},
- {"numeric_sum", `SELECT SUM(amount) FROM orders GROUP BY status`},
-
- // Category 4: date/time ops against text
- {"date_compare", `SELECT id FROM orders WHERE placed_at > '2024-01-01'::date`},
- {"date_extract", `SELECT EXTRACT(year FROM placed_at::timestamp) FROM orders`},
-
- // Category 5: array ops against text
- {"array_contains", `SELECT id FROM orders WHERE tags @> ARRAY['urgent']`},
- {"array_unnest", `SELECT id, unnest(tags) FROM orders`},
-
- // Category 6: subquery + CTE (name resolution heavy)
- {"cte", `WITH active AS (SELECT id, status FROM orders WHERE status = 'open') SELECT id FROM active`},
- {"subquery_where", `SELECT id FROM orders WHERE id IN (SELECT id FROM customers)`},
- {"correlated", `SELECT id, (SELECT email FROM customers c WHERE c.id = orders.id) AS email FROM orders`},
-
- // Category 7: aggregations
- {"count_star", `SELECT COUNT(*) FROM orders`},
- {"count_col", `SELECT COUNT(id) FROM orders`},
- {"group_by", `SELECT status, COUNT(*) FROM orders GROUP BY status`},
- }
-
- type result struct {
- ok bool
- err string
- nTargets int
- names []string
- }
-
- runOne := func(cat *catalog.Catalog, sql string) result {
- stmts, err := ParsePg(sql)
- if err != nil {
- return result{err: "parse: " + err.Error()}
- }
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- return result{err: "not a SelectStmt"}
- }
- q, err := cat.AnalyzeSelectStmt(sel)
- if err != nil {
- return result{err: "analyze: " + err.Error()}
- }
- r := result{ok: true, nTargets: len(q.TargetList)}
- for _, te := range q.TargetList {
- r.names = append(r.names, te.ResName)
- }
- return r
- }
-
- var bothOk, honestOnlyOk, lyingOnlyOk, bothFail, lineageDiverges int
- t.Log("---- query span type-free install exploration ----")
- t.Logf("%-22s %-8s %-8s %s", "case", "honest", "lying", "note")
- for _, tc := range cases {
- h := runOne(catHonest, tc.query)
- l := runOne(catLying, tc.query)
-
- honestMark := "FAIL"
- if h.ok {
- honestMark = "PASS"
- }
- lyingMark := "FAIL"
- if l.ok {
- lyingMark = "PASS"
- }
-
- note := ""
- if h.ok && l.ok {
- bothOk++
- // Compare lineage shape (target count + names).
- if h.nTargets != l.nTargets {
- note = fmt.Sprintf("TARGETS DIFFER h=%d l=%d", h.nTargets, l.nTargets)
- lineageDiverges++
- } else {
- same := true
- for i := range h.names {
- if h.names[i] != l.names[i] {
- same = false
- break
- }
- }
- if !same {
- note = fmt.Sprintf("NAMES DIFFER h=%v l=%v", h.names, l.names)
- lineageDiverges++
- } else {
- note = fmt.Sprintf("IDENTICAL (%d cols)", h.nTargets)
- }
- }
- } else if h.ok && !l.ok {
- honestOnlyOk++
- note = "lying fails: " + truncate(l.err, 60)
- } else if !h.ok && l.ok {
- lyingOnlyOk++
- note = "honest fails: " + truncate(h.err, 60)
- } else {
- bothFail++
- note = "both fail"
- }
-
- t.Logf("%-22s %-8s %-8s %s", tc.name, honestMark, lyingMark, note)
- }
-
- t.Log("")
- t.Logf("summary: both_pass=%d honest_only=%d lying_only=%d both_fail=%d diverge=%d",
- bothOk, honestOnlyOk, lyingOnlyOk, bothFail, lineageDiverges)
- t.Log("")
- t.Log("key question: for cases where both paths analyze successfully, does lineage match?")
- t.Log("key question: for cases where only honest analyzes, can fallback recover equal-or-better lineage?")
-}
-
-func truncate(s string, n int) string {
- if len(s) <= n {
- return s
- }
- return s[:n] + "..."
-}
-
-// TestLoaderPoC_UnknownPseudoType explores whether PG's 'unknown' pseudo-type
-// can be used as a universal column type that accepts all operators.
-func TestLoaderPoC_UnknownPseudoType(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- tries := []string{"unknown", "anyelement", "any"}
- for _, ty := range tries {
- stmt := buildCreateTableStmt("public", "t_"+ty, []*ast.ColumnDef{
- {Colname: "id", TypeName: typeName(ty)},
- })
- err := cat.DefineRelation(stmt, 'r')
- if err != nil {
- t.Logf("%-12s → DefineRelation rejected: %s", ty, truncate(err.Error(), 80))
- } else {
- t.Logf("%-12s → DefineRelation accepted", ty)
- }
- }
-}
-
-// TestLoaderPoC_UnknownTypeQueryBattery installs the same queries as
-// TestLoaderPoC_TypesDontMatterForLineage but with all columns declared as
-// 'unknown' instead of 'text'. The question: does the analyzer's overload
-// resolution treat unknown differently from text?
-func TestLoaderPoC_UnknownTypeQueryBattery(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- install := func(table string, cols []string) {
- colDefs := make([]*ast.ColumnDef, len(cols))
- for i, c := range cols {
- colDefs[i] = columnDef(c, typeName("unknown"), false)
- }
- if err := cat.DefineRelation(buildCreateTableStmt("public", table, colDefs), 'r'); err != nil {
- t.Fatalf("install %s: %v", table, err)
- }
- }
- install("orders", []string{"id", "amount", "status", "placed_at", "tags", "customer_id"})
- install("customers", []string{"id", "email", "name", "region"})
-
- cases := []struct {
- category, name, query string
- }{
- // ---- Baseline: name resolution only ----
- {"name_res", "plain_select", `SELECT id, status FROM orders`},
- {"name_res", "select_star", `SELECT * FROM orders`},
- {"name_res", "join", `SELECT o.id, c.email FROM orders o JOIN customers c ON c.id = o.customer_id`},
- {"name_res", "three_way_join", `SELECT o.id FROM orders o JOIN customers c ON c.id = o.customer_id JOIN orders o2 ON o2.id = o.id`},
- {"name_res", "alias_qualified", `SELECT o.id AS order_id, c.name AS customer FROM orders o, customers c WHERE c.id = o.customer_id`},
-
- // ---- Text-friendly operators ----
- {"text_ops", "text_equality", `SELECT id FROM orders WHERE status = 'open'`},
- {"text_ops", "text_like", `SELECT id FROM orders WHERE status LIKE 'op%'`},
- {"text_ops", "text_ilike", `SELECT id FROM orders WHERE status ILIKE 'OP%'`},
- {"text_ops", "text_in_list", `SELECT id FROM orders WHERE status IN ('open', 'closed')`},
- {"text_ops", "is_null", `SELECT id FROM orders WHERE status IS NULL`},
- {"text_ops", "is_not_null", `SELECT id FROM orders WHERE status IS NOT NULL`},
-
- // ---- Numeric operators (unknown auto-coerces) ----
- {"numeric_ops", "numeric_compare", `SELECT id FROM orders WHERE amount > 100`},
- {"numeric_ops", "numeric_arithmetic", `SELECT id, amount + 1 AS p FROM orders`},
- {"numeric_ops", "numeric_between", `SELECT id FROM orders WHERE amount BETWEEN 10 AND 100`},
- {"numeric_ops", "numeric_negation", `SELECT id, -amount FROM orders`},
-
- // ---- Date/time ops ----
- {"date_ops", "date_compare", `SELECT id FROM orders WHERE placed_at > '2024-01-01'::date`},
- {"date_ops", "date_extract", `SELECT EXTRACT(year FROM placed_at::timestamp) FROM orders`},
- {"date_ops", "interval_arith", `SELECT id FROM orders WHERE placed_at > now() - interval '7 days'`},
-
- // ---- Array ops ----
- {"array_ops", "array_contains", `SELECT id FROM orders WHERE tags @> ARRAY['urgent']`},
- {"array_ops", "array_any", `SELECT id FROM orders WHERE 'urgent' = ANY(tags)`},
- {"array_ops", "array_length", `SELECT id, array_length(tags, 1) FROM orders`},
- {"array_ops", "array_unnest", `SELECT id, unnest(tags) FROM orders`},
-
- // ---- Aggregates (the worry zone) ----
- {"aggregate", "count_star", `SELECT COUNT(*) FROM orders`},
- {"aggregate", "count_col", `SELECT COUNT(id) FROM orders`},
- {"aggregate", "count_distinct", `SELECT COUNT(DISTINCT customer_id) FROM orders`},
- {"aggregate", "sum", `SELECT SUM(amount) FROM orders`},
- {"aggregate", "avg", `SELECT AVG(amount) FROM orders`},
- {"aggregate", "min_max", `SELECT MIN(amount), MAX(amount) FROM orders`},
- {"aggregate", "string_agg", `SELECT STRING_AGG(status, ',') FROM orders`},
- {"aggregate", "array_agg", `SELECT ARRAY_AGG(id) FROM orders`},
- {"aggregate", "group_by", `SELECT status, COUNT(*) FROM orders GROUP BY status`},
- {"aggregate", "having", `SELECT status, COUNT(*) FROM orders GROUP BY status HAVING COUNT(*) > 5`},
-
- // ---- Aggregate with explicit cast (workaround) ----
- {"cast_workaround", "sum_with_cast", `SELECT SUM(amount::numeric) FROM orders`},
- {"cast_workaround", "avg_with_cast", `SELECT AVG(amount::numeric) FROM orders`},
- {"cast_workaround", "string_agg_cast", `SELECT STRING_AGG(status::text, ',') FROM orders`},
-
- // ---- Window functions ----
- {"window", "row_number", `SELECT id, ROW_NUMBER() OVER (ORDER BY placed_at) FROM orders`},
- {"window", "rank_partition", `SELECT id, RANK() OVER (PARTITION BY status ORDER BY amount) FROM orders`},
- {"window", "sum_over", `SELECT id, SUM(amount) OVER (PARTITION BY status) FROM orders`},
- {"window", "lag_col", `SELECT id, LAG(status) OVER (ORDER BY placed_at) FROM orders`},
-
- // ---- Control flow / null handling ----
- {"control", "case_when", `SELECT id, CASE WHEN status = 'open' THEN 1 ELSE 0 END FROM orders`},
- {"control", "coalesce", `SELECT COALESCE(status, 'unknown') FROM orders`},
- {"control", "nullif", `SELECT NULLIF(status, 'open') FROM orders`},
-
- // ---- Subqueries / set operations ----
- {"subquery", "exists", `SELECT id FROM orders o WHERE EXISTS (SELECT 1 FROM customers c WHERE c.id = o.customer_id)`},
- {"subquery", "scalar_subquery", `SELECT id, (SELECT MAX(amount) FROM orders) AS max_amt FROM orders`},
- {"subquery", "not_in", `SELECT id FROM orders WHERE customer_id NOT IN (SELECT id FROM customers WHERE region = 'us')`},
- {"subquery", "union", `SELECT id FROM orders UNION SELECT id FROM customers`},
- {"subquery", "intersect", `SELECT id FROM orders INTERSECT SELECT id FROM customers`},
- {"subquery", "except", `SELECT id FROM orders EXCEPT SELECT id FROM customers`},
-
- // ---- CTE + recursion ----
- {"cte", "simple_cte", `WITH active AS (SELECT id, status FROM orders WHERE status = 'open') SELECT id FROM active`},
- {"cte", "cte_with_agg", `WITH stats AS (SELECT status, COUNT(*) c FROM orders GROUP BY status) SELECT * FROM stats`},
-
- // ---- Sort / limit ----
- {"order_limit", "order_by_col", `SELECT id FROM orders ORDER BY placed_at DESC LIMIT 10`},
- {"order_limit", "order_by_expr", `SELECT id FROM orders ORDER BY amount + 1`},
- {"order_limit", "distinct_on", `SELECT DISTINCT ON (customer_id) id, customer_id FROM orders ORDER BY customer_id, placed_at DESC`},
- }
-
- type bucket struct{ pass, fail int }
- byCategory := map[string]*bucket{}
-
- pass := 0
- fail := 0
- var failed []string
- for _, tc := range cases {
- if byCategory[tc.category] == nil {
- byCategory[tc.category] = &bucket{}
- }
- stmts, err := ParsePg(tc.query)
- if err != nil {
- t.Logf("%-12s %-22s PARSE FAIL: %v", tc.category, tc.name, err)
- fail++
- byCategory[tc.category].fail++
- failed = append(failed, tc.category+"/"+tc.name+" (parse)")
- continue
- }
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- t.Logf("%-12s %-22s NOT SELECT", tc.category, tc.name)
- fail++
- byCategory[tc.category].fail++
- continue
- }
- _, err = cat.AnalyzeSelectStmt(sel)
- if err != nil {
- t.Logf("%-12s %-22s FAIL: %s", tc.category, tc.name, truncate(err.Error(), 90))
- fail++
- byCategory[tc.category].fail++
- failed = append(failed, tc.category+"/"+tc.name)
- continue
- }
- pass++
- byCategory[tc.category].pass++
- t.Logf("%-12s %-22s PASS", tc.category, tc.name)
- }
-
- t.Log("")
- t.Log("----- unknown-type battery: per-category breakdown -----")
- for _, cat := range []string{"name_res", "text_ops", "numeric_ops", "date_ops", "array_ops", "aggregate", "cast_workaround", "window", "control", "subquery", "cte", "order_limit"} {
- b := byCategory[cat]
- if b == nil {
- continue
- }
- t.Logf(" %-16s %d pass / %d fail", cat, b.pass, b.fail)
- }
- t.Log("")
- t.Logf("TOTAL: %d pass, %d fail (%.0f%% pass rate)", pass, fail, 100.0*float64(pass)/float64(pass+fail))
- if len(failed) > 0 {
- t.Log("failed:")
- for _, f := range failed {
- t.Logf(" - %s", f)
- }
- }
-}
-
-// TestLoaderPoC_HybridTypeResolution explores the pragmatic middle ground:
-// for each column, try to resolve the metadata type string; if it fails
-// (user type not in catalog, malformed type string), fall back to text.
-// This gives us "best of both" — real types where available, permissive
-// text where not.
-func TestLoaderPoC_HybridTypeResolution(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Hybrid column builder: try honest type, fall back to text.
- hybridCol := func(name string, typeStr string) *ast.ColumnDef {
- tn := mustTypeName(t, typeStr)
- // Check if catalog can resolve it at this moment.
- // Note: for user types to resolve, they must already be installed.
- // We're checking against the catalog AS-IS.
- resolved := false
- func() {
- defer func() { _ = recover() }() // defensive: ResolveType may not panic, but be safe
- _, _, err := cat.ResolveType(toCatalogTypeName(tn))
- resolved = (err == nil)
- }()
- if resolved {
- return &ast.ColumnDef{Colname: name, TypeName: tn}
- }
- t.Logf(" hybrid: column %q type %q unresolvable in catalog, falling back to text", name, typeStr)
- return &ast.ColumnDef{Colname: name, TypeName: typeName("text")}
- }
-
- // Simulate metadata where some types are resolvable, some aren't.
- // 'int4' is a builtin — resolvable.
- // 'numeric(10,2)' is a builtin — resolvable.
- // 'public.unknown_enum_xyz' is a user type we never installed — should fall back to text.
- stmt := buildCreateTableStmt("public", "mix", []*ast.ColumnDef{
- hybridCol("id", "int4"),
- hybridCol("amount", "numeric(10,2)"),
- hybridCol("tag", "public.unknown_enum_xyz"),
- })
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("hybrid install failed: %v", err)
- }
-
- // Queries that use the numeric column (known type) should still work
- // with real operator resolution.
- q1 := mustAnalyze(t, cat, `SELECT id, amount FROM mix WHERE amount > 100`)
- if len(q1.TargetList) != 2 {
- t.Errorf("q1 targets: got %d, want 2", len(q1.TargetList))
- }
-
- // Queries that touch the tag column only do name resolution — no operator
- // type-checking — should work even though tag is text-in-reality-but-
- // metadata-said-enum.
- q2 := mustAnalyze(t, cat, `SELECT id, tag FROM mix WHERE tag = 'a'`)
- if len(q2.TargetList) != 2 {
- t.Errorf("q2 targets: got %d, want 2", len(q2.TargetList))
- }
-
- t.Log("hybrid PoC: resolvable types use real TypeName, unresolvable types fall back to text")
- t.Log("effect: best of both — operator semantics work where types known, name resolution works where not")
-}
-
-// TestLoaderPoC_BvsCDivergence pinpoints the exact behavioral difference between
-// Option B (hybrid: builtins honest, user types unknown) and Option C (all
-// honest types, user types installed).
-//
-// Scenario: a table with ONE builtin-int column and ONE user-enum column.
-// Run the same queries against two catalogs:
-//
-// catC: enum installed via DefineEnum, table column is the real enum type
-// catB: enum NOT installed, table column is 'unknown'
-//
-// For each query, record (passes/fails, target count, lineage equivalent).
-// The output shows exactly where B and C diverge.
-func TestLoaderPoC_BvsCDivergence(t *testing.T) {
- // ----- catC: option C (all honest) -----
- catC := catalog.New()
- catC.SetSearchPath([]string{"public"})
-
- if err := catC.DefineEnum(&ast.CreateEnumStmt{
- TypeName: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "task_status"},
- }},
- Vals: &ast.List{Items: []ast.Node{
- &ast.String{Str: "open"},
- &ast.String{Str: "closed"},
- }},
- }); err != nil {
- t.Fatalf("catC install enum: %v", err)
- }
- if err := catC.DefineRelation(buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("status", mustTypeName(t, "public.task_status"), false),
- columnDef("created_at", mustTypeName(t, "timestamptz"), false),
- }), 'r'); err != nil {
- t.Fatalf("catC install table: %v", err)
- }
-
- // ----- catB: option B (hybrid — enum column as unknown, everything else honest) -----
- catB := catalog.New()
- catB.SetSearchPath([]string{"public"})
- // Note: no DefineEnum call. Enum "doesn't exist" from catB's perspective.
- if err := catB.DefineRelation(buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true), // builtin → honest
- columnDef("status", typeName("unknown"), false), // user type → unknown
- columnDef("created_at", typeName("timestamptz"), false), // builtin → honest
- }), 'r'); err != nil {
- t.Fatalf("catB install table: %v", err)
- }
-
- cases := []struct {
- name, query string
- }{
- // Queries that should work in BOTH (builtin columns untouched)
- {"select_id", `SELECT id FROM tasks`},
- {"select_star", `SELECT * FROM tasks`},
- {"count_star", `SELECT COUNT(*) FROM tasks`},
- {"sum_id", `SELECT SUM(id) FROM tasks`},
- {"avg_id", `SELECT AVG(id) FROM tasks`},
- {"max_created", `SELECT MAX(created_at) FROM tasks`},
-
- // Queries touching status column — B gets 'unknown', C gets real enum
- {"where_status_eq", `SELECT id FROM tasks WHERE status = 'open'`},
- {"where_status_in", `SELECT id FROM tasks WHERE status IN ('open', 'closed')`},
- {"select_status", `SELECT status FROM tasks`},
- {"group_by_status", `SELECT status, COUNT(*) FROM tasks GROUP BY status`},
- {"order_by_status", `SELECT id FROM tasks ORDER BY status`},
- {"distinct_status", `SELECT DISTINCT status FROM tasks`},
-
- // Aggregates directly on the enum column — the real test
- {"count_status", `SELECT COUNT(status) FROM tasks`},
- {"count_distinct_status", `SELECT COUNT(DISTINCT status) FROM tasks`},
- {"min_status", `SELECT MIN(status) FROM tasks`},
- {"max_status", `SELECT MAX(status) FROM tasks`},
- {"array_agg_status", `SELECT array_agg(status) FROM tasks`},
- {"string_agg_status", `SELECT string_agg(status::text, ',') FROM tasks`},
-
- // Mixed: status in WHERE, sum on builtin
- {"mixed_sum", `SELECT SUM(id) FROM tasks WHERE status = 'open'`},
- {"mixed_group", `SELECT status, SUM(id) FROM tasks GROUP BY status`},
- }
-
- type res struct {
- ok bool
- err string
- }
- runOne := func(cat *catalog.Catalog, sql string) res {
- stmts, err := ParsePg(sql)
- if err != nil {
- return res{err: "parse: " + truncate(err.Error(), 60)}
- }
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- return res{err: "not a SelectStmt"}
- }
- if _, err := cat.AnalyzeSelectStmt(sel); err != nil {
- return res{err: truncate(err.Error(), 70)}
- }
- return res{ok: true}
- }
-
- var bothOk, cOnlyOk, bOnlyOk, bothFail int
- t.Logf("%-24s %-6s %-6s %s", "case", "C", "B", "divergence note")
- t.Logf("%-24s %-6s %-6s %s", "----", "-", "-", "---------------")
- for _, tc := range cases {
- c := runOne(catC, tc.query)
- b := runOne(catB, tc.query)
- cMark := "FAIL"
- if c.ok {
- cMark = "PASS"
- }
- bMark := "FAIL"
- if b.ok {
- bMark = "PASS"
- }
-
- note := ""
- switch {
- case c.ok && b.ok:
- bothOk++
- note = "same"
- case c.ok && !b.ok:
- cOnlyOk++
- note = "C only — B: " + b.err
- case !c.ok && b.ok:
- bOnlyOk++
- note = "B only — C: " + c.err
- default:
- bothFail++
- note = "both fail"
- }
- t.Logf("%-24s %-6s %-6s %s", tc.name, cMark, bMark, note)
- }
-
- t.Log("")
- t.Logf("summary: both_ok=%d C_only=%d B_only=%d both_fail=%d", bothOk, cOnlyOk, bOnlyOk, bothFail)
- t.Logf("option B loss (queries where C works but B falls back): %d / %d = %.0f%%",
- cOnlyOk, len(cases), 100.0*float64(cOnlyOk)/float64(len(cases)))
-}
-
-// TestLoaderPoC_BvsCDomainColumn shows the case where B is STRICTLY BETTER than C:
-// a table with a domain-typed column where the domain is NOT in bytebase metadata
-// (which is the ACTUAL state of bytebase PG sync today — domain/composite/range
-// types are not synced). Option C fails at install time and blocks all queries
-// on the table. Option B uses unknown and only loses type-sensitive aggregates.
-func TestLoaderPoC_BvsCDomainColumn(t *testing.T) {
- // Simulate reality: bytebase sync does NOT capture the 'public.positive_int'
- // domain. The table metadata says column 'count' has type 'public.positive_int',
- // but bytebase has no DomainMetadata for it.
-
- // ----- catC: option C — try to honest-install everything -----
- // Closure expansion would try to load 'public.positive_int' from metadata,
- // find nothing, and either fail the table install or skip the column.
- // Let's simulate: C tries to install the table with the real type reference,
- // fails because the domain doesn't exist in the catalog.
- catC := catalog.New()
- catC.SetSearchPath([]string{"public"})
- cStmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("count", mustTypeName(t, "public.positive_int"), true),
- })
- cErr := catC.DefineRelation(cStmt, 'r')
- if cErr != nil {
- t.Logf("[C] DefineRelation failed as expected: %s", truncate(cErr.Error(), 80))
- } else {
- t.Log("[C] DefineRelation unexpectedly succeeded")
- }
-
- // ----- catB: option B — count column degrades to unknown -----
- catB := catalog.New()
- catB.SetSearchPath([]string{"public"})
- bStmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("count", typeName("unknown"), true),
- })
- if err := catB.DefineRelation(bStmt, 'r'); err != nil {
- t.Fatalf("[B] DefineRelation failed: %v", err)
- }
- t.Log("[B] DefineRelation succeeded")
-
- cases := []struct {
- name, query string
- }{
- {"simple_select", `SELECT id, count FROM counters`},
- {"where_id", `SELECT id FROM counters WHERE id > 0`},
- {"where_count", `SELECT id FROM counters WHERE count > 0`},
- {"sum_id", `SELECT SUM(id) FROM counters`},
- {"sum_count", `SELECT SUM(count) FROM counters`},
- {"sum_id_cast", `SELECT SUM(count::numeric) FROM counters`},
- }
-
- runOne := func(cat *catalog.Catalog, sql string) string {
- stmts, err := ParsePg(sql)
- if err != nil {
- return "parse: " + err.Error()
- }
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- return "not a SelectStmt"
- }
- if _, err := cat.AnalyzeSelectStmt(sel); err != nil {
- return truncate(err.Error(), 60)
- }
- return "PASS"
- }
-
- t.Log("")
- t.Log("Query availability on 'counters' table (domain column):")
- t.Logf("%-22s %-40s %s", "case", "C result", "B result")
- var cWorks, bWorks int
- for _, tc := range cases {
- var cResult string
- if cErr != nil {
- cResult = "TABLE NOT INSTALLED"
- } else {
- cResult = runOne(catC, tc.query)
- }
- bResult := runOne(catB, tc.query)
- if cResult == "PASS" {
- cWorks++
- }
- if bResult == "PASS" {
- bWorks++
- }
- t.Logf("%-22s %-40s %s", tc.name, truncate(cResult, 38), bResult)
- }
- t.Log("")
- t.Logf("domain column query survival: C=%d/%d, B=%d/%d", cWorks, len(cases), bWorks, len(cases))
-}
-
-// toCatalogTypeName converts ast.TypeName to catalog.TypeName for ResolveType.
-// catalog.TypeName is the lightweight shape used by the public ResolveType API.
-func toCatalogTypeName(tn *ast.TypeName) catalog.TypeName {
- var schema, name string
- if tn.Names != nil && len(tn.Names.Items) > 0 {
- items := tn.Names.Items
- if len(items) == 1 {
- if s, ok := items[0].(*ast.String); ok {
- name = s.Str
- }
- } else if len(items) >= 2 {
- if s, ok := items[0].(*ast.String); ok {
- schema = s.Str
- }
- if s, ok := items[len(items)-1].(*ast.String); ok {
- name = s.Str
- }
- }
- }
- return catalog.TypeName{Schema: schema, Name: name, TypeMod: tn.Typemod}
-}
-
-// Format helper for readable output.
-var _ = fmt.Sprintf // keep fmt imported when strings.Builder not used
-
-// ---------- Equivalence test: hand-built CreateStmt vs parsed DDL ----------
-//
-// Per codex review section 2B: the PoC must prove that hand-constructed AST
-// produces a catalog state equivalent to parsing the equivalent DDL text.
-// This test installs the same table two ways on two separate catalogs and
-// asserts the resulting Relation objects are structurally identical on all
-// fields reachable via public API.
-
-func TestLoaderPoC_HandBuiltVsDDLEquivalence(t *testing.T) {
- // Catalog A: install via hand-built CreateStmt.
- catA := catalog.New()
- catA.SetSearchPath([]string{"public"})
- handBuilt := buildCreateTableStmt("public", "orders", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("customer", typeName("text"), false),
- columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),
- columnDef("placed_at", mustTypeName(t, "timestamp with time zone"), false),
- })
- if err := catA.DefineRelation(handBuilt, 'r'); err != nil {
- t.Fatalf("catA install: %v", err)
- }
-
- // Catalog B: install via parsed DDL text.
- catB := catalog.New()
- catB.SetSearchPath([]string{"public"})
- ddl := `CREATE TABLE public.orders (
- id int NOT NULL,
- customer text,
- amount numeric(10,2),
- placed_at timestamp with time zone
- )`
- if _, err := catB.Exec(ddl, nil); err != nil {
- t.Fatalf("catB install via Exec: %v", err)
- }
-
- relA := catA.GetRelation("public", "orders")
- relB := catB.GetRelation("public", "orders")
- if relA == nil || relB == nil {
- t.Fatalf("relation install missing: A=%v B=%v", relA != nil, relB != nil)
- }
-
- // Compare structural fields. OIDs are not expected to match absolutely
- // (each catalog has its own oidGen starting point), but relative order
- // and per-column shape must match.
-
- if relA.RelKind != relB.RelKind {
- t.Errorf("RelKind differs: A=%c B=%c", relA.RelKind, relB.RelKind)
- }
- if len(relA.Columns) != len(relB.Columns) {
- t.Fatalf("column count: A=%d B=%d", len(relA.Columns), len(relB.Columns))
- }
- for i := range relA.Columns {
- ca, cb := relA.Columns[i], relB.Columns[i]
- if ca.Name != cb.Name {
- t.Errorf("col[%d].Name: A=%q B=%q", i, ca.Name, cb.Name)
- }
- if ca.TypeOID != cb.TypeOID {
- t.Errorf("col[%d] (%s).TypeOID: A=%d B=%d", i, ca.Name, ca.TypeOID, cb.TypeOID)
- }
- if ca.NotNull != cb.NotNull {
- t.Errorf("col[%d] (%s).NotNull: A=%v B=%v", i, ca.Name, ca.NotNull, cb.NotNull)
- }
- if ca.AttNum != cb.AttNum {
- t.Errorf("col[%d] (%s).AttNum: A=%d B=%d", i, ca.Name, ca.AttNum, cb.AttNum)
- }
- if ca.TypeMod != cb.TypeMod {
- t.Errorf("col[%d] (%s).TypeMod: A=%d B=%d", i, ca.Name, ca.TypeMod, cb.TypeMod)
- }
- }
-
- // Functional equivalence: analyze the same SELECT against both catalogs
- // and assert the same target list shape.
- qA := mustAnalyze(t, catA, `SELECT id, customer, amount, placed_at FROM orders WHERE amount > 0`)
- qB := mustAnalyze(t, catB, `SELECT id, customer, amount, placed_at FROM orders WHERE amount > 0`)
-
- if len(qA.TargetList) != len(qB.TargetList) {
- t.Errorf("query target count: A=%d B=%d", len(qA.TargetList), len(qB.TargetList))
- }
- for i := range qA.TargetList {
- if qA.TargetList[i].ResName != qB.TargetList[i].ResName {
- t.Errorf("target[%d].ResName: A=%q B=%q",
- i, qA.TargetList[i].ResName, qB.TargetList[i].ResName)
- }
- }
-
- t.Log("equivalence PoC: hand-built CreateStmt and parsed DDL produce structurally identical Relation + analyze the same way")
-}
-
-// ============================================================================
-// Loader pseudo-install verification
-// ============================================================================
-//
-// These tests probe whether omni accepts minimal "pseudo" forms of each schema
-// object kind, so the catalog loader strategy ("install everything; on failure install a
-// pseudo at the same slot to keep downstream installs cascading through") can
-// work inline during catalog init rather than as a query-time retry.
-//
-// Each test exercises one pseudo form in isolation, then the final
-// TestLoaderPoC_RootPseudoCascadePrevention combines them end-to-end.
-
-// ---- Pseudo enum: DefineEnum with minimal vals ----
-
-func TestLoaderPoC_PseudoEnum_EmptyVals(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Can omni accept an enum with zero values?
- stmt := &ast.CreateEnumStmt{
- TypeName: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "broken_enum"},
- }},
- Vals: &ast.List{Items: []ast.Node{}},
- }
- err := cat.DefineEnum(stmt)
- t.Logf("DefineEnum empty vals: err=%v", err)
-
- if err == nil {
- // If it accepted, can a table column reference it and can the query run?
- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("status", mustTypeName(t, "public.broken_enum"), false),
- })
- if err := cat.DefineRelation(tbl, 'r'); err != nil {
- t.Logf("DefineRelation with empty-enum column: err=%v", err)
- } else {
- q, err := analyzeSafe(cat, `SELECT id, status FROM t WHERE status = 'anything'`)
- t.Logf("analyze against empty-enum column: err=%v targets=%d", err, targetsOf(q))
- }
- }
-}
-
-func TestLoaderPoC_PseudoEnum_SingleDummyVal(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Fallback: enum with single dummy value.
- stmt := &ast.CreateEnumStmt{
- TypeName: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "broken_enum"},
- }},
- Vals: &ast.List{Items: []ast.Node{
- &ast.String{Str: "__pseudo__"},
- }},
- }
- if err := cat.DefineEnum(stmt); err != nil {
- t.Fatalf("DefineEnum single dummy val failed: %v", err)
- }
-
- // Install a table using the pseudo enum as a column type.
- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("status", mustTypeName(t, "public.broken_enum"), false),
- })
- if err := cat.DefineRelation(tbl, 'r'); err != nil {
- t.Fatalf("DefineRelation with pseudo-enum column failed: %v", err)
- }
-
- // Probe queries that would normally hit enum operators.
- queries := []string{
- `SELECT id, status FROM t`,
- `SELECT id FROM t WHERE status = 'running'`,
- `SELECT id FROM t WHERE status::text = 'running'`,
- `SELECT COUNT(*) FROM t GROUP BY status`,
- }
- for _, q := range queries {
- res, err := analyzeSafe(cat, q)
- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))
- }
-}
-
-// ---- Pseudo domain: DefineDomain with base type text ----
-
-func TestLoaderPoC_PseudoDomain_OverText(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- stmt := &ast.CreateDomainStmt{
- Domainname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "broken_domain"},
- }},
- Typname: typeName("text"),
- }
- if err := cat.DefineDomain(stmt); err != nil {
- t.Fatalf("DefineDomain over text failed: %v", err)
- }
-
- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("code", mustTypeName(t, "public.broken_domain"), false),
- })
- if err := cat.DefineRelation(tbl, 'r'); err != nil {
- t.Fatalf("DefineRelation with pseudo-domain column failed: %v", err)
- }
-
- queries := []string{
- `SELECT id, code FROM t`,
- `SELECT id FROM t WHERE code = 'anything'`,
- `SELECT id FROM t WHERE code LIKE 'A%'`,
- }
- for _, q := range queries {
- res, err := analyzeSafe(cat, q)
- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))
- }
-}
-
-// ---- Pseudo composite: DefineCompositeType with metadata field names all text ----
-
-func TestLoaderPoC_PseudoComposite_FieldsAllText(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Pretend metadata tells us this composite has fields {street, city, zip}.
- // We preserve the names but flatten every field to text.
- stmt := &ast.CompositeTypeStmt{
- Typevar: &ast.RangeVar{
- Schemaname: "public",
- Relname: "broken_addr",
- },
- Coldeflist: &ast.List{Items: []ast.Node{
- columnDef("street", typeName("text"), false),
- columnDef("city", typeName("text"), false),
- columnDef("zip", typeName("text"), false),
- }},
- }
- if err := cat.DefineCompositeType(stmt); err != nil {
- t.Fatalf("DefineCompositeType pseudo failed: %v", err)
- }
-
- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("addr", mustTypeName(t, "public.broken_addr"), false),
- })
- if err := cat.DefineRelation(tbl, 'r'); err != nil {
- t.Fatalf("DefineRelation with pseudo-composite column failed: %v", err)
- }
-
- queries := []string{
- `SELECT id, addr FROM t`,
- `SELECT id, (addr).city FROM t`,
- `SELECT id FROM t WHERE (addr).city = 'NYC'`,
- }
- for _, q := range queries {
- res, err := analyzeSafe(cat, q)
- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))
- }
-}
-
-// ---- Pseudo range: DefineRange with subtype text ----
-
-func TestLoaderPoC_PseudoRange_SubtypeText(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Try: range with subtype text. PG itself requires subtype to have btree
- // ordering, which text does. If omni accepts it, this is a viable pseudo.
- stmt := &ast.CreateRangeStmt{
- TypeName: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "broken_range"},
- }},
- Params: &ast.List{Items: []ast.Node{
- &ast.DefElem{
- Defname: "subtype",
- Arg: &ast.TypeName{
- Names: &ast.List{Items: []ast.Node{&ast.String{Str: "text"}}},
- Typemod: -1,
- },
- },
- }},
- }
- err := cat.DefineRange(stmt)
- t.Logf("DefineRange subtype=text: err=%v", err)
- if err != nil {
- t.Skip("range install failed; this is fine if bytebase metadata never has ranges")
- }
-
- tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("span", mustTypeName(t, "public.broken_range"), false),
- })
- if err := cat.DefineRelation(tbl, 'r'); err != nil {
- t.Logf("DefineRelation with pseudo-range column: err=%v", err)
- }
-
- q, err := analyzeSafe(cat, `SELECT id, span FROM t`)
- t.Logf("query on pseudo-range: err=%v targets=%d", err, targetsOf(q))
-}
-
-// ---- Pseudo view: constant SELECT body, no FROM ----
-
-func TestLoaderPoC_PseudoView_ConstantTargetList(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Pretend metadata tells us view "v" exposes columns {id, title, total},
- // but its body is broken/unparseable. Build a pseudo SELECT body with
- // NULL::text aliased to each name.
- pseudoBody := `SELECT NULL::text AS id, NULL::text AS title, NULL::text AS total`
- sel := mustParseSelect(t, pseudoBody)
-
- stmt := &ast.ViewStmt{
- View: &ast.RangeVar{
- Schemaname: "public",
- Relname: "broken_view",
- Relpersistence: 'p',
- },
- Query: sel,
- }
- if err := cat.DefineView(stmt); err != nil {
- t.Fatalf("DefineView with constant-target body failed: %v", err)
- }
-
- rel := cat.GetRelation("public", "broken_view")
- if rel == nil {
- t.Fatal("pseudo view not installed")
- }
- t.Logf("pseudo view installed: RelKind=%c cols=%d", rel.RelKind, len(rel.Columns))
- for _, c := range rel.Columns {
- t.Logf(" col: %s", c.Name)
- }
-
- queries := []string{
- `SELECT id, title, total FROM broken_view`,
- `SELECT id FROM broken_view WHERE total = '0'`,
- `SELECT COUNT(*) FROM broken_view`,
- }
- for _, q := range queries {
- res, err := analyzeSafe(cat, q)
- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))
- }
-}
-
-// ---- Pseudo function: all params/return as text ----
-
-func TestLoaderPoC_PseudoFunction_AllText(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Pretend metadata says the real function was broken_fn(some_broken_type)
- // returning some_other_broken_type. We install a pseudo with everything
- // collapsed to text.
- stmt := &ast.CreateFunctionStmt{
- Funcname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "broken_fn"},
- }},
- Parameters: &ast.List{Items: []ast.Node{
- &ast.FunctionParameter{ArgType: typeName("text"), Mode: ast.FUNC_PARAM_IN},
- }},
- ReturnType: typeName("text"),
- Options: &ast.List{Items: []ast.Node{
- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},
- &ast.DefElem{
- Defname: "as",
- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},
- },
- }},
- }
- if err := cat.CreateFunctionStmt(stmt); err != nil {
- t.Fatalf("pseudo function install failed: %v", err)
- }
-
- procs := cat.LookupProcByName("broken_fn")
- t.Logf("pseudo function installed: %d procs under 'broken_fn'", len(procs))
-
- // Install a minimal table to query against.
- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("name", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("install t: %v", err)
- }
-
- queries := []string{
- `SELECT id, broken_fn(name) AS result FROM t`,
- `SELECT id FROM t WHERE broken_fn(name) = 'x'`,
- // Analyzer should pick the only overload regardless of actual arg type.
- `SELECT broken_fn('literal') FROM t`,
- }
- for _, q := range queries {
- res, err := analyzeSafe(cat, q)
- t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))
- }
-}
-
-// ---- End-to-end: root-pseudo prevents cascade ----
-
-func TestLoaderPoC_RootPseudoCascadePrevention(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Scenario:
- // enum E — real install fails (simulated)
- // table T — has column col E
- // view V — SELECT col FROM T
- //
- // Phase 1: for each object in topo order:
- // 1. attempt real install
- // 2. on failure, install pseudo version in the same slot
- //
- // Expected: T and V install successfully because they see pseudo E in the
- // catalog. Query against V has correct lineage.
-
- // Step 1: simulate E real-install failure. In the real loader we would try
- // DefineEnum first and catch the error. Here we skip straight to the
- // pseudo form to simulate "real install failed".
- pseudoE := &ast.CreateEnumStmt{
- TypeName: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "task_status"},
- }},
- Vals: &ast.List{Items: []ast.Node{
- &ast.String{Str: "__pseudo__"},
- }},
- }
- if err := cat.DefineEnum(pseudoE); err != nil {
- t.Fatalf("pseudo enum install failed: %v", err)
- }
-
- // Step 2: install T pointing at the (pseudo) E. This is the real install,
- // not a pseudo — T's own metadata is fine.
- tStmt := buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("status", mustTypeName(t, "public.task_status"), false),
- columnDef("title", typeName("text"), false),
- })
- if err := cat.DefineRelation(tStmt, 'r'); err != nil {
- t.Fatalf("T real install failed (cascade not prevented): %v", err)
- }
-
- // Step 3: install V — real install, view body references T.
- viewBody := `SELECT id, title, status FROM tasks WHERE id > 0`
- sel := mustParseSelect(t, viewBody)
- vStmt := &ast.ViewStmt{
- View: &ast.RangeVar{
- Schemaname: "public",
- Relname: "open_tasks",
- Relpersistence: 'p',
- },
- Query: sel,
- }
- if err := cat.DefineView(vStmt); err != nil {
- t.Fatalf("V real install failed (cascade not prevented): %v", err)
- }
-
- // Step 4: query V — should analyze successfully and return lineage.
- q := mustAnalyze(t, cat, `SELECT id, title, status FROM open_tasks`)
- if len(q.TargetList) != 3 {
- t.Errorf("query on view: got %d targets, want 3", len(q.TargetList))
- }
-
- // Verify lineage reaches base tables, not just view columns.
- for i, te := range q.TargetList {
- t.Logf(" target[%d]: name=%s expr=%T", i, te.ResName, te.Expr)
- }
-
- // Step 5: try a query that uses enum-typed column with operators that
- // would normally need enum resolution.
- res, err := analyzeSafe(cat, `SELECT id FROM open_tasks WHERE status = 'pending'`)
- t.Logf("query with WHERE on pseudo-enum column: err=%v targets=%d", err, targetsOf(res))
-
- t.Log("cascade prevention PoC: pseudo-enum at root allowed T and V to install as real objects")
-}
-
-// ---- Corner case 1: mixed real + pseudo overloads of the same function name ----
-//
-// Scenario: function fn has two overloads in metadata:
-// fn(int4) → int4 (real install succeeds)
-// fn(my_broken_type) → my_broken_type (real install fails because type is broken)
-//
-// catalog loader strategy: keep fn(int4) real, install pseudo fn(text) for the broken one.
-// Verify: calls to fn(int_col) still pick fn(int4); calls to fn(text_col) pick
-// the pseudo fn(text); calls to fn(literal_string) pick the pseudo.
-
-func TestLoaderPoC_MixedRealAndPseudoOverloads(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- makeFuncStmt := func(name string, argType, retType string) *ast.CreateFunctionStmt {
- return &ast.CreateFunctionStmt{
- Funcname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: name},
- }},
- Parameters: &ast.List{Items: []ast.Node{
- &ast.FunctionParameter{ArgType: typeName(argType), Mode: ast.FUNC_PARAM_IN},
- }},
- ReturnType: typeName(retType),
- Options: &ast.List{Items: []ast.Node{
- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},
- &ast.DefElem{
- Defname: "as",
- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},
- },
- }},
- }
- }
-
- // Install real fn(int4).
- if err := cat.CreateFunctionStmt(makeFuncStmt("fn", "int4", "int4")); err != nil {
- t.Fatalf("real fn(int4) install failed: %v", err)
- }
-
- // Install pseudo fn(text) — pretending the real fn(broken_type) failed.
- if err := cat.CreateFunctionStmt(makeFuncStmt("fn", "text", "text")); err != nil {
- t.Fatalf("pseudo fn(text) install failed: %v", err)
- }
-
- procs := cat.LookupProcByName("fn")
- t.Logf("fn overloads after mixed install: %d", len(procs))
- if len(procs) < 2 {
- t.Errorf("expected >= 2 fn overloads, got %d", len(procs))
- }
-
- // Install a table with one int and one text column.
- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("n", typeName("int4"), false),
- columnDef("s", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("install t: %v", err)
- }
-
- // Each query exercises a different resolution path.
- queries := []struct {
- name string
- sql string
- }{
- {"fn(int_col) → should pick real fn(int4)", `SELECT id, fn(n) AS r FROM t`},
- {"fn(text_col) → should pick pseudo fn(text)", `SELECT id, fn(s) AS r FROM t`},
- {"fn(literal_int) → picks fn(int4) or via implicit cast", `SELECT fn(42) FROM t`},
- {"fn(literal_text) → picks fn(text)", `SELECT fn('hello') FROM t`},
- }
- for _, q := range queries {
- res, err := analyzeSafe(cat, q.sql)
- t.Logf("%s: err=%v targets=%d", q.name, err, targetsOf(res))
- }
-}
-
-// ---- Corner case 2: pseudo install after a real install has truly failed ----
-//
-// Scenario: real DefineDomain fails because the base type does not exist.
-// The loader must then install a pseudo in the same slot. Verify:
-// 1. real install actually returns an error (otherwise this test is vacuous)
-// 2. catalog does not hold partial state for the failed name
-// 3. pseudo install at the same name succeeds
-// 4. a table column using that pseudo name resolves correctly
-
-func TestLoaderPoC_PseudoAfterFailedReal(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- // Step 1: try a real DefineDomain with a non-existent base type.
- badStmt := &ast.CreateDomainStmt{
- Domainname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "status_code"},
- }},
- Typname: typeName("nonexistent_base_type_xyz"),
- }
- err := cat.DefineDomain(badStmt)
- t.Logf("real DefineDomain with bogus base type: err=%v", err)
-
- realFailed := err != nil
-
- // Step 2: check catalog state. If the real install created a partial entry,
- // the pseudo install at the same name may conflict.
- //
- // Probe by trying to look up the name via a constructed type reference.
- // There's no direct "is this type defined?" API, but we can try the pseudo
- // install and see if it errors with a duplicate.
-
- // Step 3: install pseudo domain over text at the same name.
- pseudoStmt := &ast.CreateDomainStmt{
- Domainname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "status_code"},
- }},
- Typname: typeName("text"),
- }
- err = cat.DefineDomain(pseudoStmt)
- t.Logf("pseudo DefineDomain (base=text) after real failure: err=%v", err)
-
- if err != nil {
- t.Errorf("pseudo install after real failure should succeed; got %v", err)
- return
- }
-
- // Step 4: use the pseudo in a table column.
- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("code", mustTypeName(t, "public.status_code"), false),
- }), 'r'); err != nil {
- t.Fatalf("table with pseudo-domain column install failed: %v", err)
- }
-
- q := mustAnalyze(t, cat, `SELECT id, code FROM t WHERE code = 'ok'`)
- if len(q.TargetList) != 2 {
- t.Errorf("query on pseudo-domain column: got %d targets, want 2", len(q.TargetList))
- }
-
- if !realFailed {
- t.Log("note: real DefineDomain with bogus base type DID NOT fail — " +
- "omni's validation is looser than expected. Test still exercises " +
- "the pseudo install path but the 'recovery after failure' premise " +
- "is weaker. Consider whether the loader needs to handle this case.")
- } else {
- t.Log("pseudo recovery PoC: real install failed cleanly, pseudo at same " +
- "name installed successfully, downstream table install works")
- }
-}
-
-// ---- helpers for loader tests ----
-
-// ---- Install cost benchmarks: catalog-loader hand-built vs Exec(DDL text) ----
-//
-// Validates the plan's "5-10× faster than current" claim empirically.
-// Generates synthetic schemas of varying size with representative shapes
-// (tables with mixed-type columns) and times both install paths.
-//
-// Run with:
-// go test -bench='BenchmarkLoader_Install' -benchmem -run='^$' -count=3 \
-// github.com/bytebase/bytebase/backend/plugin/parser/pg
-
-func makeSyntheticTables(n int) []*ast.CreateStmt {
- tables := make([]*ast.CreateStmt, n)
- for i := range n {
- tables[i] = buildCreateTableStmt("public", fmt.Sprintf("t_%d", i), []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("name", typeName("text"), false),
- columnDef("created_at", typeName("timestamp"), false),
- columnDef("amount", typeName("int8"), false),
- columnDef("status", typeName("text"), false),
- })
- }
- return tables
-}
-
-func makeSyntheticDDL(n int) string {
- var sb strings.Builder
- sb.Grow(n * 200)
- for i := range n {
- fmt.Fprintf(&sb, "CREATE TABLE public.t_%d (id int4 NOT NULL, name text, created_at timestamp, amount int8, status text);\n", i)
- }
- return sb.String()
-}
-
-func benchmarkE3Install(b *testing.B, n int) {
- tables := makeSyntheticTables(n)
- b.ResetTimer()
- for range b.N {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- for _, stmt := range tables {
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- b.Fatalf("DefineRelation: %v", err)
- }
- }
- }
-}
-
-func benchmarkDDLInstall(b *testing.B, n int) {
- ddl := makeSyntheticDDL(n)
- b.ResetTimer()
- for range b.N {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- if _, err := cat.Exec(ddl, &catalog.ExecOptions{ContinueOnError: true}); err != nil {
- b.Fatalf("Exec: %v", err)
- }
- }
-}
-
-func BenchmarkLoader_Install_E3_100(b *testing.B) { benchmarkE3Install(b, 100) }
-func BenchmarkLoader_Install_E3_500(b *testing.B) { benchmarkE3Install(b, 500) }
-func BenchmarkLoader_Install_E3_2000(b *testing.B) { benchmarkE3Install(b, 2000) }
-func BenchmarkLoader_Install_DDL_100(b *testing.B) { benchmarkDDLInstall(b, 100) }
-func BenchmarkLoader_Install_DDL_500(b *testing.B) { benchmarkDDLInstall(b, 500) }
-func BenchmarkLoader_Install_DDL_2000(b *testing.B) { benchmarkDDLInstall(b, 2000) }
-
-// ---- Overload selection assertions: prove analyzer picks the right overload ----
-//
-// Builds on TestLoaderPoC_MixedRealAndPseudoOverloads. That test logs outcomes
-// but does not assert which overload got picked. Here we install two overloads
-// with DISTINCT return types (int4 vs text) and assert that the analyzer picks
-// the right one by checking the result column's TypeOID.
-//
-// Well-known PG OIDs: int4=23, text=25 (stable; used throughout omni tests).
-
-func TestLoaderPoC_OverloadSelectionAsserted(t *testing.T) {
- const (
- oidInt4 uint32 = 23
- oidText uint32 = 25
- )
-
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- mkFn := func(arg, ret string) *ast.CreateFunctionStmt {
- return &ast.CreateFunctionStmt{
- Funcname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: "fn"},
- }},
- Parameters: &ast.List{Items: []ast.Node{
- &ast.FunctionParameter{ArgType: typeName(arg), Mode: ast.FUNC_PARAM_IN},
- }},
- ReturnType: typeName(ret),
- Options: &ast.List{Items: []ast.Node{
- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},
- &ast.DefElem{
- Defname: "as",
- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},
- },
- }},
- }
- }
-
- // Real fn(int4) → int4. Pseudo fn(text) → text (simulated: pretend the
- // real fn(broken_type) failed and we installed a text-backed pseudo).
- if err := cat.CreateFunctionStmt(mkFn("int4", "int4")); err != nil {
- t.Fatalf("fn(int4) install failed: %v", err)
- }
- if err := cat.CreateFunctionStmt(mkFn("text", "text")); err != nil {
- t.Fatalf("fn(text) install failed: %v", err)
- }
-
- if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("n", typeName("int4"), false),
- columnDef("s", typeName("text"), false),
- }), 'r'); err != nil {
- t.Fatalf("table install: %v", err)
- }
-
- exprResultType := func(q *catalog.Query, targetIdx int) uint32 {
- t.Helper()
- if targetIdx >= len(q.TargetList) {
- t.Fatalf("target index %d out of range (have %d)", targetIdx, len(q.TargetList))
- }
- te := q.TargetList[targetIdx]
- fce, ok := te.Expr.(*catalog.FuncCallExpr)
- if !ok {
- t.Fatalf("target[%d] expected FuncCallExpr, got %T", targetIdx, te.Expr)
- }
- return fce.ResultType
- }
-
- // Case 1: fn(int_col) must pick fn(int4) → returns int4
- q := mustAnalyze(t, cat, `SELECT fn(n) FROM t`)
- if got := exprResultType(q, 0); got != oidInt4 {
- t.Errorf("fn(int_col): expected ResultType=%d (int4), got %d", oidInt4, got)
- }
-
- // Case 2: fn(text_col) must pick fn(text) → returns text
- q = mustAnalyze(t, cat, `SELECT fn(s) FROM t`)
- if got := exprResultType(q, 0); got != oidText {
- t.Errorf("fn(text_col): expected ResultType=%d (text), got %d", oidText, got)
- }
-
- // Case 3: fn(42) — integer literal — must pick fn(int4)
- q = mustAnalyze(t, cat, `SELECT fn(42) FROM t`)
- if got := exprResultType(q, 0); got != oidInt4 {
- t.Errorf("fn(literal_int): expected ResultType=%d (int4), got %d", oidInt4, got)
- }
-
- // Case 4: fn('hello') — string literal — must pick fn(text)
- q = mustAnalyze(t, cat, `SELECT fn('hello') FROM t`)
- if got := exprResultType(q, 0); got != oidText {
- t.Errorf("fn(literal_text): expected ResultType=%d (text), got %d", oidText, got)
- }
-
- t.Log("overload selection asserted: analyzer picks the right overload by arg type; " +
- "mixed real+pseudo overloads resolve correctly")
-}
-
-// ---- REAL cascade prevention: force a real install failure, then install pseudo ----
-//
-// The previous cascade test simulated failure by skipping the real install
-// entirely. This test actually runs a real install that fails at the omni
-// level, then proves the loader pattern "catch error → install pseudo at
-// same slot" lets downstream installs use real AST.
-//
-// Scenario:
-// Domain D (base=nonexistent_type_xyz) — real DefineDomain ERRORS
-// Table T (col D, title text) — real DefineRelation must succeed against pseudo D
-// View V (SELECT col, title FROM T) — real DefineView must succeed against real T
-// Query SELECT col, title FROM V — must analyze with lineage to T.{col, title}
-
-func TestLoaderPoC_RealCascadeWithGenuineFailure(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
-
- installOne := func(kind, name string, installReal func() error, installPseudo func() error) (degraded bool) {
- if err := installReal(); err != nil {
- t.Logf("[%s %q] real install failed: %v", kind, name, err)
- if pErr := installPseudo(); pErr != nil {
- t.Fatalf("[%s %q] pseudo install also failed: %v", kind, name, pErr)
- }
- t.Logf("[%s %q] pseudo installed at same slot", kind, name)
- return true
- }
- t.Logf("[%s %q] real install succeeded", kind, name)
- return false
- }
-
- // Step 1: install D. Real fails (nonexistent base), pseudo succeeds.
- dName := "d_status"
- dDegraded := installOne("domain", dName,
- func() error {
- return cat.DefineDomain(&ast.CreateDomainStmt{
- Domainname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: dName},
- }},
- Typname: typeName("nonexistent_base_type_for_test_xyz"),
- })
- },
- func() error {
- return cat.DefineDomain(&ast.CreateDomainStmt{
- Domainname: &ast.List{Items: []ast.Node{
- &ast.String{Str: "public"},
- &ast.String{Str: dName},
- }},
- Typname: typeName("text"),
- })
- },
- )
- if !dDegraded {
- t.Fatal("expected D real install to fail (test premise broken)")
- }
-
- // Step 2: install T. This is a REAL install (T's metadata is fine).
- // The column references the pseudo D, so DefineRelation must find it.
- tName := "records"
- tDegraded := installOne("table", tName,
- func() error {
- return cat.DefineRelation(buildCreateTableStmt("public", tName, []*ast.ColumnDef{
- columnDef("id", typeName("int4"), true),
- columnDef("col", mustTypeName(t, "public."+dName), false),
- columnDef("title", typeName("text"), false),
- }), 'r')
- },
- func() error {
- return errors.Errorf("should not reach pseudo for table %s", tName)
- },
- )
- if tDegraded {
- t.Fatal("T should install as REAL against pseudo D — cascade prevention violated")
- }
-
- // Step 3: install V. REAL install. View body references T (which is real).
- vName := "records_view"
- vDegraded := installOne("view", vName,
- func() error {
- sel := mustParseSelect(t, `SELECT id, col, title FROM records WHERE id > 0`)
- return cat.DefineView(&ast.ViewStmt{
- View: &ast.RangeVar{
- Schemaname: "public",
- Relname: vName,
- Relpersistence: 'p',
- },
- Query: sel,
- })
- },
- func() error {
- return errors.Errorf("should not reach pseudo for view %s", vName)
- },
- )
- if vDegraded {
- t.Fatal("V should install as REAL against real T — cascade prevention violated")
- }
-
- // Step 4: query V. Verify lineage makes it all the way to base table columns.
- q := mustAnalyze(t, cat, `SELECT id, col, title FROM records_view`)
- if len(q.TargetList) != 3 {
- t.Fatalf("view query: got %d targets, want 3", len(q.TargetList))
- }
- for i, te := range q.TargetList {
- if _, isVar := te.Expr.(*catalog.VarExpr); !isVar {
- t.Errorf("target[%d] (%s): expected VarExpr (resolved lineage), got %T",
- i, te.ResName, te.Expr)
- }
- }
-
- // Step 5: query with WHERE on the pseudo-backed column — text semantics
- // should let this analyze successfully.
- q2, err := analyzeSafe(cat, `SELECT id FROM records_view WHERE col = 'active'`)
- if err != nil {
- t.Errorf("WHERE on pseudo-domain column should analyze; got err=%v", err)
- } else if len(q2.TargetList) != 1 {
- t.Errorf("WHERE query: got %d targets, want 1", len(q2.TargetList))
- }
-
- t.Log("REAL cascade PoC: genuine DefineDomain failure → pseudo at same slot → " +
- "downstream table and view install as real → query lineage intact")
-}
-
-// analyzeSafe runs AnalyzeSelectStmt but returns (nil, err) instead of failing
-// the test — probe tests need to record whether analyze succeeded.
-func analyzeSafe(cat *catalog.Catalog, sql string) (*catalog.Query, error) {
- stmts, err := ParsePg(sql)
- if err != nil {
- return nil, errors.Wrap(err, "parse")
- }
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("not a SelectStmt: %T", stmts[0].AST)
- }
- return cat.AnalyzeSelectStmt(sel)
-}
-
-func targetsOf(q *catalog.Query) int {
- if q == nil {
- return -1
- }
- return len(q.TargetList)
-}
diff --git a/backend/plugin/parser/pg/query_span_loader_pseudo.go b/backend/plugin/parser/pg/query_span_loader_pseudo.go
deleted file mode 100644
index 17d4d4109f..0000000000
--- a/backend/plugin/parser/pg/query_span_loader_pseudo.go
+++ /dev/null
@@ -1,273 +0,0 @@
-package pg
-
-import (
- "strings"
-
- "github.com/pkg/errors"
-
- "github.com/bytebase/omni/pg/ast"
-)
-
-// This file builds "pseudo" AST nodes used by the Catalog loader when a real
-// install fails. Every pseudo form is backed by built-in types only (text,
-// primarily) so pseudo install itself cannot cascade into further failures.
-//
-// Pseudo forms preserve object names and — where metadata allows — column
-// names. Type fidelity is intentionally dropped. For query span this is
-// sufficient in the vast majority of cases: lineage is a name-resolution
-// problem, not a type-resolution problem.
-//
-// Every form in this file corresponds to a PoC test in
-// query_span_v5_poc_test.go (TestLoaderPoC_Pseudo*).
-
-// pseudoTextTypeName is the single pseudo column/parameter/return type omni
-// accepts uniformly. Every pseudo form ends up routing to this shape.
-func pseudoTextTypeName() *ast.TypeName {
- return &ast.TypeName{
- Names: &ast.List{Items: []ast.Node{&ast.String{Str: "text"}}},
- Typemod: -1,
- }
-}
-
-// pseudoCreateEnumStmt builds a minimal enum with zero values. omni's
-// DefineEnum accepts an empty Vals list.
-func pseudoCreateEnumStmt(schema, name string) *ast.CreateEnumStmt {
- return &ast.CreateEnumStmt{
- TypeName: qualifiedNameList(schema, name),
- Vals: &ast.List{Items: []ast.Node{}},
- }
-}
-
-// pseudoCreateDomainStmt builds a domain over text with no constraints.
-// Query span does not inspect domain constraints, so dropping them is safe.
-func pseudoCreateDomainStmt(schema, name string) *ast.CreateDomainStmt {
- return &ast.CreateDomainStmt{
- Domainname: qualifiedNameList(schema, name),
- Typname: pseudoTextTypeName(),
- }
-}
-
-// pseudoCompositeTypeStmt builds a composite type with a single text field
-// named "_broken". It is the fallback when a composite type's real
-// definition is absent from metadata or fails to install, so (col).field
-// access against a pseudo composite will fall through to
-// extractFallbackColumns.
-func pseudoCompositeTypeStmt(schema, name string) *ast.CompositeTypeStmt {
- return &ast.CompositeTypeStmt{
- Typevar: &ast.RangeVar{
- Schemaname: schema,
- Relname: name,
- },
- Coldeflist: &ast.List{Items: []ast.Node{
- &ast.ColumnDef{
- Colname: "_broken",
- TypeName: pseudoTextTypeName(),
- },
- }},
- }
-}
-
-// pseudoCreateRangeStmt builds a range over text. text has btree ordering so
-// omni accepts it as a valid subtype.
-func pseudoCreateRangeStmt(schema, name string) *ast.CreateRangeStmt {
- return &ast.CreateRangeStmt{
- TypeName: qualifiedNameList(schema, name),
- Params: &ast.List{Items: []ast.Node{
- &ast.DefElem{
- Defname: "subtype",
- Arg: pseudoTextTypeName(),
- },
- }},
- }
-}
-
-// pseudoCreateTableStmt builds a table whose columns are all text-typed but
-// retain their metadata names. This is the fallback used when real install
-// fails because of unresolved column types (broken enum, unknown composite,
-// etc.).
-func pseudoCreateTableStmt(schema, name string, columnNames []string) *ast.CreateStmt {
- items := make([]ast.Node, 0, len(columnNames))
- for _, col := range columnNames {
- if col == "" {
- continue
- }
- items = append(items, &ast.ColumnDef{
- Colname: col,
- TypeName: pseudoTextTypeName(),
- })
- }
- return &ast.CreateStmt{
- Relation: &ast.RangeVar{
- Schemaname: schema,
- Relname: name,
- Relpersistence: 'p',
- },
- TableElts: &ast.List{Items: items},
- }
-}
-
-// pseudoViewStmt builds a view whose body is a constant SELECT target list
-// exposing each metadata column name as a nullable text expression. The body
-// has no FROM clause.
-//
-// Returns an error if the body cannot be parsed; callers should treat this
-// as a "truly broken" object.
-func pseudoViewStmt(schema, name string, columnNames []string) (*ast.ViewStmt, error) {
- sel, err := pseudoConstantSelect(columnNames)
- if err != nil {
- return nil, err
- }
- return &ast.ViewStmt{
- View: &ast.RangeVar{
- Schemaname: schema,
- Relname: name,
- Relpersistence: 'p',
- },
- Query: sel,
- }, nil
-}
-
-// pseudoCreateTableAsStmt builds a pseudo materialized view using the same
-// constant-target-list body as pseudoViewStmt.
-func pseudoCreateTableAsStmt(schema, name string, columnNames []string) (*ast.CreateTableAsStmt, error) {
- sel, err := pseudoConstantSelect(columnNames)
- if err != nil {
- return nil, err
- }
- return &ast.CreateTableAsStmt{
- Query: sel,
- Objtype: ast.OBJECT_MATVIEW,
- Into: &ast.IntoClause{
- Rel: &ast.RangeVar{
- Schemaname: schema,
- Relname: name,
- Relpersistence: 'p',
- },
- },
- }, nil
-}
-
-// pseudoCreateFunctionStmt builds a function whose signature collapses every
-// parameter to text and whose return type is text. argCount is taken from the
-// real function's signature so analyzer overload resolution by arity still
-// matches against the pseudo.
-//
-// The function body is `SELECT $1` when argCount > 0, `SELECT NULL::text`
-// otherwise — always a valid SQL-language body.
-func pseudoCreateFunctionStmt(schema, name string, argCount int) *ast.CreateFunctionStmt {
- stmt := &ast.CreateFunctionStmt{
- Funcname: qualifiedNameList(schema, name),
- ReturnType: pseudoTextTypeName(),
- }
- if argCount > 0 {
- params := make([]ast.Node, argCount)
- for i := range argCount {
- params[i] = &ast.FunctionParameter{
- ArgType: pseudoTextTypeName(),
- Mode: ast.FUNC_PARAM_IN,
- }
- }
- stmt.Parameters = &ast.List{Items: params}
- }
- body := "SELECT NULL::text"
- if argCount > 0 {
- body = "SELECT $1::text"
- }
- stmt.Options = &ast.List{Items: []ast.Node{
- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},
- &ast.DefElem{
- Defname: "as",
- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: body}}},
- },
- }}
- return stmt
-}
-
-// pseudoConstantSelect parses "SELECT NULL::text AS col1, NULL::text AS col2 ..."
-// and returns the resulting *ast.SelectStmt. Columns with empty names are
-// skipped; if the resulting target list is empty, a single anonymous column
-// is emitted so the SELECT is well-formed.
-func pseudoConstantSelect(columnNames []string) (*ast.SelectStmt, error) {
- var targets []string
- for _, col := range columnNames {
- if col == "" {
- continue
- }
- targets = append(targets, "NULL::text AS "+quoteIdent(col))
- }
- if len(targets) == 0 {
- targets = []string{"NULL::text"}
- }
- sql := "SELECT " + strings.Join(targets, ", ")
- stmts, err := ParsePg(sql)
- if err != nil {
- return nil, errors.Wrap(err, "parse pseudo constant select")
- }
- if len(stmts) != 1 {
- return nil, errors.Errorf("pseudo constant select: expected 1 statement, got %d", len(stmts))
- }
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("pseudo constant select: expected SelectStmt, got %T", stmts[0].AST)
- }
- return sel, nil
-}
-
-// functionArgCountFromSignature parses a signature string like
-// "my_func(integer, text)" and returns the number of arguments. A signature
-// with no parentheses or an empty argument list returns 0.
-func functionArgCountFromSignature(signature string) int {
- openIdx := strings.Index(signature, "(")
- if openIdx < 0 {
- return 0
- }
- closeIdx := strings.LastIndex(signature, ")")
- if closeIdx < 0 || closeIdx <= openIdx {
- return 0
- }
- inner := strings.TrimSpace(signature[openIdx+1 : closeIdx])
- if inner == "" {
- return 0
- }
- // Split on top-level commas. Type names inside parens (rare, e.g.
- // numeric(10,2)) must not count as separators.
- depth := 0
- count := 1
- for _, r := range inner {
- switch r {
- case '(':
- depth++
- case ')':
- if depth > 0 {
- depth--
- }
- case ',':
- if depth == 0 {
- count++
- }
- default:
- // other runes contribute nothing to arg-count accounting
- }
- }
- return count
-}
-
-// ---------------- small AST helpers shared with builders.go ----------------
-
-// qualifiedNameList builds an omni List of String nodes representing a
-// schema-qualified object name. An empty schema emits only the name.
-func qualifiedNameList(schema, name string) *ast.List {
- items := make([]ast.Node, 0, 2)
- if schema != "" {
- items = append(items, &ast.String{Str: schema})
- }
- items = append(items, &ast.String{Str: name})
- return &ast.List{Items: items}
-}
-
-// quoteIdent wraps an identifier in double quotes, doubling any internal
-// double quotes. Used when a metadata column name may contain special chars
-// or collide with SQL reserved words.
-func quoteIdent(name string) string {
- return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
-}
diff --git a/backend/plugin/parser/pg/query_span_loader_test.go b/backend/plugin/parser/pg/query_span_loader_test.go
deleted file mode 100644
index ce0d2a8773..0000000000
--- a/backend/plugin/parser/pg/query_span_loader_test.go
+++ /dev/null
@@ -1,911 +0,0 @@
-package pg
-
-import (
- "context"
- "strings"
- "testing"
-
- "github.com/pkg/errors"
-
- metadatapb "github.com/bytebase/omni/metadata"
- "github.com/bytebase/omni/pg/ast"
- "github.com/bytebase/omni/pg/catalog"
-)
-
-// ---------------- typeNameFromString ----------------
-
-func TestLoaderTypeNameFromString(t *testing.T) {
- cases := []struct {
- in string
- wantErr bool
- }{
- {"int4", false},
- {"text", false},
- {"numeric(10,2)", false},
- {"timestamp(3) with time zone", false},
- {"character varying(255)", false},
- {"text[]", false},
- {"public.task_status", false},
- {`"public"."task_status"`, false},
- {"", true}, // empty cannot parse as SELECT NULL::
- }
- for _, c := range cases {
- tn, err := typeNameFromString(c.in)
- if c.wantErr {
- if err == nil {
- t.Errorf("typeNameFromString(%q): expected error, got tn=%+v", c.in, tn)
- }
- continue
- }
- if err != nil {
- t.Errorf("typeNameFromString(%q): unexpected error: %v", c.in, err)
- continue
- }
- if tn == nil {
- t.Errorf("typeNameFromString(%q): nil TypeName with no error", c.in)
- }
- }
-}
-
-// ---------------- extractUserTypeRefs ----------------
-
-func TestLoaderExtractUserTypeRefs(t *testing.T) {
- cases := []struct {
- in string
- want []UserTypeRef
- }{
- // Built-in scalars.
- {"integer", nil},
- {"int4", nil},
- {"text", nil},
- {"bigint", nil},
- {"boolean", nil},
- {"json", nil},
- {"jsonb", nil},
- {"uuid", nil},
- {"date", nil},
- {"interval", nil},
-
- // Built-in with length / precision / timezone.
- {"numeric(10,2)", nil},
- {"numeric(8)", nil},
- {"decimal(10,2)", nil},
- {"character varying(255)", nil},
- {"character(10)", nil},
- {"varchar(255)", nil},
- {"bit(8)", nil},
- {"bit varying(8)", nil},
- {"timestamp(3) with time zone", nil},
- {"time(6) without time zone", nil},
- {"timestamp without time zone", nil},
-
- // USER-DEFINED.
- {"public.task_status", []UserTypeRef{{Schema: "public", Name: "task_status"}}},
- {"myschema.my_domain", []UserTypeRef{{Schema: "myschema", Name: "my_domain"}}},
-
- // Quoted qualified name.
- {`"MySchema"."MyType"`, []UserTypeRef{{Schema: "MySchema", Name: "MyType"}}},
-
- // PG internal array form.
- {"_text", nil},
- {"_int4", nil},
- {"_task_status", nil}, // sync.go:834 drops the schema; we do not topo-sort it.
-
- // System schemas.
- {"pg_catalog.int4", nil},
- {"information_schema.cardinal_number", nil},
- {"pg_toast.pg_toast_2619", nil},
-
- // Empty.
- {"", nil},
-
- // Whitespace.
- {" text ", nil},
- }
- for _, c := range cases {
- got := extractUserTypeRefs(c.in)
- if !userTypeRefsEqual(got, c.want) {
- t.Errorf("extractUserTypeRefs(%q): got %+v, want %+v", c.in, got, c.want)
- }
- }
-}
-
-func userTypeRefsEqual(a, b []UserTypeRef) bool {
- if len(a) != len(b) {
- return false
- }
- for i := range a {
- if a[i] != b[i] {
- return false
- }
- }
- return true
-}
-
-// ---------------- helpers ----------------
-
-func TestLoaderStripTypeModifiers(t *testing.T) {
- cases := []struct {
- in, want string
- }{
- {"numeric(10,2)", "numeric"},
- {"timestamp(3) with time zone", "timestamp"},
- {"timestamp without time zone", "timestamp"},
- {"character varying(255)", "character varying"},
- {" text ", "text"},
- {"public.task_status", "public.task_status"},
- {"", ""},
- }
- for _, c := range cases {
- if got := stripTypeModifiers(c.in); got != c.want {
- t.Errorf("stripTypeModifiers(%q): got %q, want %q", c.in, got, c.want)
- }
- }
-}
-
-func TestLoaderSplitQualifiedName(t *testing.T) {
- cases := []struct {
- in string
- wantSchema string
- wantName string
- wantOk bool
- }{
- {"public.foo", "public", "foo", true},
- {`"weird name"."another"`, "weird name", "another", true},
- {"foo", "", "", false},
- {"", "", "", false},
- {"a.b.c", "", "", false}, // too many parts
- }
- for _, c := range cases {
- s, n, ok := splitQualifiedName(c.in)
- if s != c.wantSchema || n != c.wantName || ok != c.wantOk {
- t.Errorf("splitQualifiedName(%q): got (%q,%q,%v), want (%q,%q,%v)",
- c.in, s, n, ok, c.wantSchema, c.wantName, c.wantOk)
- }
- }
-}
-
-func TestLoaderIsSystemSchema(t *testing.T) {
- // Sanity check that extractUserTypeRefs routes system schemas to nil via
- // IsSystemSchema. The underlying predicate is tested in system_objects.go.
- cases := map[string]bool{
- "pg_catalog.foo": true,
- "information_schema.bar": true,
- "pg_toast.baz": true,
- "pg_temp_1.qux": true,
- "public.foo": false,
- "myschema.bar": false,
- }
- for in, wantFiltered := range cases {
- got := extractUserTypeRefs(in)
- if wantFiltered && got != nil {
- t.Errorf("extractUserTypeRefs(%q): expected filtered, got %+v", in, got)
- }
- if !wantFiltered && got == nil {
- t.Errorf("extractUserTypeRefs(%q): expected non-nil, got nil", in)
- }
- }
-}
-
-// ---------------- classifyAnalyzeError ----------------
-
-func TestLoaderClassifyAnalyzeError(t *testing.T) {
- cases := []struct {
- name string
- err error
- want fallbackReason
- }{
- {"nil", nil, reasonNone},
- {
- "undefined function",
- &catalog.Error{Code: catalog.CodeUndefinedFunction, Message: "function fn(text) does not exist"},
- reasonExpectedPseudoSemantic,
- },
- {
- "ambiguous function",
- &catalog.Error{Code: catalog.CodeAmbiguousFunction, Message: "function fn is not unique"},
- reasonExpectedPseudoSemantic,
- },
- {
- "datatype mismatch",
- &catalog.Error{Code: catalog.CodeDatatypeMismatch, Message: "UNION types text and integer cannot be matched"},
- reasonExpectedPseudoSemantic,
- },
- {
- "feature not supported",
- &catalog.Error{Code: catalog.CodeFeatureNotSupported, Message: "not supported"},
- reasonExpectedPseudoSemantic,
- },
- {
- "ambiguous column",
- &catalog.Error{Code: catalog.CodeAmbiguousColumn, Message: "column x is ambiguous"},
- reasonExpectedPseudoSemantic,
- },
- {
- "undefined table",
- &catalog.Error{Code: catalog.CodeUndefinedTable, Message: `relation "foo" does not exist`},
- reasonUndefinedReference,
- },
- {
- "undefined column",
- &catalog.Error{Code: catalog.CodeUndefinedColumn, Message: `column "x" does not exist`},
- reasonUndefinedReference,
- },
- {
- "undefined object",
- &catalog.Error{Code: catalog.CodeUndefinedObject, Message: `type "t" does not exist`},
- reasonUndefinedReference,
- },
- {
- "undefined schema",
- &catalog.Error{Code: catalog.CodeUndefinedSchema, Message: `schema "s" does not exist`},
- reasonUndefinedReference,
- },
- {
- "other catalog error",
- &catalog.Error{Code: catalog.CodeDuplicateTable, Message: "duplicate"},
- reasonAnalyzerUnsupported,
- },
- {
- "plain error",
- errors.New("unsupported node type"),
- reasonAnalyzerUnsupported,
- },
- {
- "wrapped catalog error",
- errors.Wrap(&catalog.Error{Code: catalog.CodeDatatypeMismatch, Message: "mismatch"}, "ctx"),
- reasonExpectedPseudoSemantic,
- },
- }
- for _, c := range cases {
- if got := classifyAnalyzeError(c.err); got != c.want {
- t.Errorf("%s: classifyAnalyzeError got %v, want %v", c.name, got, c.want)
- }
- }
-}
-
-// ---------------- pseudo builders ----------------
-
-func TestLoaderPseudoEnum(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- if err := cat.DefineEnum(pseudoCreateEnumStmt("public", "broken")); err != nil {
- t.Fatalf("DefineEnum: %v", err)
- }
- // Reference it from a table column and analyze.
- tn, err := typeNameFromString("public.broken")
- if err != nil {
- t.Fatalf("typeNameFromString: %v", err)
- }
- tbl := &ast.CreateStmt{
- Relation: &ast.RangeVar{Schemaname: "public", Relname: "t", Relpersistence: 'p'},
- TableElts: &ast.List{Items: []ast.Node{
- &ast.ColumnDef{Colname: "s", TypeName: tn},
- }},
- }
- if err := cat.DefineRelation(tbl, 'r'); err != nil {
- t.Fatalf("DefineRelation with pseudo enum column: %v", err)
- }
-}
-
-func TestLoaderPseudoDomain(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- if err := cat.DefineDomain(pseudoCreateDomainStmt("public", "d")); err != nil {
- t.Fatalf("DefineDomain: %v", err)
- }
-}
-
-func TestLoaderPseudoComposite(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- if err := cat.DefineCompositeType(pseudoCompositeTypeStmt("public", "c")); err != nil {
- t.Fatalf("DefineCompositeType: %v", err)
- }
-}
-
-func TestLoaderPseudoRange(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- if err := cat.DefineRange(pseudoCreateRangeStmt("public", "r")); err != nil {
- t.Fatalf("DefineRange: %v", err)
- }
-}
-
-func TestLoaderPseudoTable(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- stmt := pseudoCreateTableStmt("public", "t", []string{"id", "name", "email"})
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation: %v", err)
- }
- rel := cat.GetRelation("public", "t")
- if rel == nil {
- t.Fatal("relation not installed")
- }
- if len(rel.Columns) != 3 {
- t.Errorf("column count: got %d, want 3", len(rel.Columns))
- }
-}
-
-func TestLoaderPseudoView(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- stmt, err := pseudoViewStmt("public", "v", []string{"id", "label"})
- if err != nil {
- t.Fatalf("build pseudo view: %v", err)
- }
- if err := cat.DefineView(stmt); err != nil {
- t.Fatalf("DefineView: %v", err)
- }
- rel := cat.GetRelation("public", "v")
- if rel == nil {
- t.Fatal("view not installed")
- }
- if len(rel.Columns) != 2 {
- t.Errorf("view column count: got %d, want 2", len(rel.Columns))
- }
-}
-
-func TestLoaderPseudoMatView(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- stmt, err := pseudoCreateTableAsStmt("public", "m", []string{"x", "y"})
- if err != nil {
- t.Fatalf("build pseudo matview: %v", err)
- }
- if err := cat.ExecCreateTableAs(stmt); err != nil {
- t.Fatalf("ExecCreateTableAs: %v", err)
- }
-}
-
-func TestLoaderPseudoFunction(t *testing.T) {
- cases := []struct {
- name string
- argCount int
- }{
- {"zero args", 0},
- {"one arg", 1},
- {"three args", 3},
- }
- for _, c := range cases {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- stmt := pseudoCreateFunctionStmt("public", "fn", c.argCount)
- if err := cat.CreateFunctionStmt(stmt); err != nil {
- t.Errorf("%s: CreateFunctionStmt: %v", c.name, err)
- continue
- }
- procs := cat.LookupProcByName("fn")
- if len(procs) == 0 {
- t.Errorf("%s: no procs installed", c.name)
- }
- }
-}
-
-func TestLoaderFunctionArgCountFromSignature(t *testing.T) {
- cases := map[string]int{
- "": 0,
- "fn()": 0,
- "fn(integer)": 1,
- "fn(integer, text)": 2,
- "fn(integer, text, uuid)": 3,
- "fn(numeric(10,2))": 1, // inner paren must not split
- "fn(numeric(10,2), int)": 2,
- }
- for in, want := range cases {
- if got := functionArgCountFromSignature(in); got != want {
- t.Errorf("functionArgCountFromSignature(%q): got %d, want %d", in, got, want)
- }
- }
-}
-
-func TestLoaderQualifiedNameList(t *testing.T) {
- list := qualifiedNameList("public", "foo")
- if list == nil || len(list.Items) != 2 {
- t.Fatalf("qualifiedNameList(public, foo): got %+v, want 2 items", list)
- }
- bare := qualifiedNameList("", "foo")
- if bare == nil || len(bare.Items) != 1 {
- t.Fatalf("qualifiedNameList(\"\", foo): got %+v, want 1 item", bare)
- }
-}
-
-// ---------------- real builders ----------------
-
-func TestLoaderBuildCreateEnumStmt(t *testing.T) {
- stmt := buildCreateEnumStmt("public", &metadatapb.EnumTypeMetadata{
- Name: "task_status",
- Values: []string{"pending", "running", "done"},
- })
- if stmt == nil || stmt.Vals == nil || len(stmt.Vals.Items) != 3 {
- t.Fatalf("unexpected: %+v", stmt)
- }
- cat := catalog.New()
- if err := cat.DefineEnum(stmt); err != nil {
- t.Fatalf("DefineEnum: %v", err)
- }
-}
-
-func TestLoaderBuildCreateStmt(t *testing.T) {
- stmt, err := buildCreateStmt("public", &metadatapb.TableMetadata{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4", Nullable: false},
- {Name: "name", Type: "text", Nullable: true},
- {Name: "amount", Type: "numeric(10,2)", Nullable: true},
- },
- })
- if err != nil {
- t.Fatalf("buildCreateStmt: %v", err)
- }
- if stmt == nil || stmt.TableElts == nil || len(stmt.TableElts.Items) != 3 {
- t.Fatalf("unexpected: %+v", stmt)
- }
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- if err := cat.DefineRelation(stmt, 'r'); err != nil {
- t.Fatalf("DefineRelation: %v", err)
- }
- rel := cat.GetRelation("public", "t")
- if rel == nil || len(rel.Columns) != 3 {
- t.Fatalf("rel: %+v", rel)
- }
-}
-
-func TestLoaderBuildCreateStmt_BadTypeErrors(t *testing.T) {
- _, err := buildCreateStmt("public", &metadatapb.TableMetadata{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "bad", Type: "this is not a type at all;;"},
- },
- })
- if err == nil {
- t.Fatal("expected error for bad column type")
- }
-}
-
-func TestLoaderBuildCreateStmt_EmptyTypeErrors(t *testing.T) {
- _, err := buildCreateStmt("public", &metadatapb.TableMetadata{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{{Name: "x", Type: ""}},
- })
- if err == nil {
- t.Fatal("expected error for empty column type")
- }
-}
-
-func TestLoaderBuildViewStmt(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- base, err := buildCreateStmt("public", &metadatapb.TableMetadata{
- Name: "orders",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- {Name: "total", Type: "int4"},
- },
- })
- if err != nil {
- t.Fatalf("build base: %v", err)
- }
- if err := cat.DefineRelation(base, 'r'); err != nil {
- t.Fatalf("install base: %v", err)
- }
- stmt, err := buildViewStmt("public", &metadatapb.ViewMetadata{
- Name: "v",
- Definition: "SELECT id, total FROM orders",
- })
- if err != nil {
- t.Fatalf("buildViewStmt: %v", err)
- }
- if err := cat.DefineView(stmt); err != nil {
- t.Fatalf("DefineView: %v", err)
- }
-}
-
-func TestLoaderBuildViewStmt_EmptyDefinition(t *testing.T) {
- _, err := buildViewStmt("public", &metadatapb.ViewMetadata{Name: "v"})
- if err == nil {
- t.Fatal("expected error for empty definition")
- }
-}
-
-func TestLoaderBuildCreateTableAsStmt(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- base, err := buildCreateStmt("public", &metadatapb.TableMetadata{
- Name: "orders",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- },
- })
- if err != nil {
- t.Fatalf("build base: %v", err)
- }
- if err := cat.DefineRelation(base, 'r'); err != nil {
- t.Fatalf("install base: %v", err)
- }
- stmt, err := buildCreateTableAsStmt("public", &metadatapb.MaterializedViewMetadata{
- Name: "m",
- Definition: "SELECT id FROM orders",
- })
- if err != nil {
- t.Fatalf("buildCreateTableAsStmt: %v", err)
- }
- if err := cat.ExecCreateTableAs(stmt); err != nil {
- t.Fatalf("ExecCreateTableAs: %v", err)
- }
-}
-
-func TestLoaderBuildCreateFunctionStmt(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- stmt, err := buildCreateFunctionStmt("public", &metadatapb.FunctionMetadata{
- Name: "fn",
- Signature: "fn(integer, text)",
- })
- if err != nil {
- t.Fatalf("buildCreateFunctionStmt: %v", err)
- }
- if stmt.Parameters == nil || len(stmt.Parameters.Items) != 2 {
- t.Fatalf("expected 2 params, got %+v", stmt.Parameters)
- }
- if err := cat.CreateFunctionStmt(stmt); err != nil {
- t.Fatalf("CreateFunctionStmt: %v", err)
- }
-}
-
-func TestLoaderBuildCreateFunctionStmt_ZeroArgs(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- stmt, err := buildCreateFunctionStmt("public", &metadatapb.FunctionMetadata{
- Name: "fn",
- Signature: "fn()",
- })
- if err != nil {
- t.Fatalf("buildCreateFunctionStmt: %v", err)
- }
- if err := cat.CreateFunctionStmt(stmt); err != nil {
- t.Fatalf("CreateFunctionStmt: %v", err)
- }
-}
-
-func TestLoaderParseFunctionSignatureArgTypes(t *testing.T) {
- cases := map[string][]string{
- "fn()": nil,
- "fn(integer)": {"integer"},
- "fn(integer, text)": {"integer", "text"},
- "fn(numeric(10,2), text)": {"numeric(10,2)", "text"},
- "fn( int4 , text )": {"int4", "text"},
- "fn": nil,
- }
- for in, want := range cases {
- got, err := parseFunctionSignatureArgTypes(in)
- if err != nil {
- t.Errorf("%q: unexpected error %v", in, err)
- continue
- }
- if !stringSlicesEqual(got, want) {
- t.Errorf("%q: got %v, want %v", in, got, want)
- }
- }
-}
-
-func stringSlicesEqual(a, b []string) bool {
- if len(a) != len(b) {
- return false
- }
- for i := range a {
- if a[i] != b[i] {
- return false
- }
- }
- return true
-}
-
-func TestLoaderParseSelectBody(t *testing.T) {
- sel, err := parseSelectBody("SELECT 1 AS x")
- if err != nil {
- t.Fatalf("parseSelectBody: %v", err)
- }
- if sel == nil {
- t.Fatal("nil sel")
- }
- if _, err := parseSelectBody(""); err == nil {
- t.Error("expected error for empty body")
- }
- if _, err := parseSelectBody("SELECT 1; SELECT 2"); err == nil {
- t.Error("expected error for multi-statement body")
- }
-}
-
-// ---------------- loader ----------------
-
-func TestLoader_HappyPath(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- EnumTypes: []*metadatapb.EnumTypeMetadata{
- {Name: "task_status", Values: []string{"pending", "running"}},
- },
- Tables: []*metadatapb.TableMetadata{{
- Name: "tasks",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- {Name: "status", Type: "public.task_status"},
- {Name: "title", Type: "text"},
- },
- }},
- Views: []*metadatapb.ViewMetadata{{
- Name: "open_tasks",
- Definition: "SELECT id, title, status FROM tasks",
- DependencyColumns: []*metadatapb.DependencyColumn{
- {Schema: "public", Table: "tasks", Column: "id"},
- {Schema: "public", Table: "tasks", Column: "title"},
- {Schema: "public", Table: "tasks", Column: "status"},
- },
- }},
- }},
- }
- loader := newCatalogLoader(cat, meta)
- if err := loader.Load(context.Background()); err != nil {
- t.Fatalf("Load: %v", err)
- }
- if len(loader.degraded) != 0 {
- t.Errorf("expected zero degraded, got %+v", loader.degraded)
- }
- if len(loader.trulyBroken) != 0 {
- t.Errorf("expected zero truly broken, got %+v", loader.trulyBroken)
- }
- if cat.GetRelation("public", "tasks") == nil {
- t.Error("tasks not installed")
- }
- if cat.GetRelation("public", "open_tasks") == nil {
- t.Error("open_tasks not installed")
- }
-}
-
-func TestLoader_BrokenEnumCascadesToPseudo(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Tables: []*metadatapb.TableMetadata{{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- // Reference a user type that is not in metadata. The
- // real install will fail; pseudo should take over.
- {Name: "status", Type: "public.nonexistent_enum"},
- },
- }},
- }},
- }
- loader := newCatalogLoader(cat, meta)
- if err := loader.Load(context.Background()); err != nil {
- t.Fatalf("Load: %v", err)
- }
- key := "rel:public.t"
- if _, ok := loader.degraded[key]; !ok {
- t.Errorf("expected public.t to be degraded, got %+v", loader.degraded)
- }
- if _, ok := loader.trulyBroken[key]; ok {
- t.Errorf("unexpected truly broken: %+v", loader.trulyBroken)
- }
- rel := cat.GetRelation("public", "t")
- if rel == nil {
- t.Fatal("pseudo t not installed")
- }
- if len(rel.Columns) != 2 {
- t.Errorf("pseudo t: got %d cols, want 2", len(rel.Columns))
- }
-}
-
-func TestLoader_TopoOrder_DependencyBeforeUse(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- // View is listed BEFORE its base table in metadata. The loader
- // must reorder so the table is installed first; otherwise the
- // view body analyzes against an empty catalog.
- Views: []*metadatapb.ViewMetadata{{
- Name: "v",
- Definition: "SELECT id FROM base",
- DependencyColumns: []*metadatapb.DependencyColumn{
- {Schema: "public", Table: "base", Column: "id"},
- },
- }},
- Tables: []*metadatapb.TableMetadata{{
- Name: "base",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int4"},
- },
- }},
- }},
- }
- loader := newCatalogLoader(cat, meta)
- if err := loader.Load(context.Background()); err != nil {
- t.Fatalf("Load: %v", err)
- }
- if loader.degraded["rel:public.v"] != nil {
- t.Errorf("view should install real, got degraded: %v", loader.degraded["rel:public.v"])
- }
- if loader.degraded["rel:public.base"] != nil {
- t.Errorf("base should install real, got degraded: %v", loader.degraded["rel:public.base"])
- }
-}
-
-func TestLoader_CycleBreaking(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- // Two views referencing each other — a metadata-level cycle.
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Views: []*metadatapb.ViewMetadata{
- {
- Name: "v_alpha",
- Definition: "SELECT id FROM v_beta",
- DependencyColumns: []*metadatapb.DependencyColumn{
- {Schema: "public", Table: "v_beta", Column: "id"},
- },
- },
- {
- Name: "v_beta",
- Definition: "SELECT id FROM v_alpha",
- DependencyColumns: []*metadatapb.DependencyColumn{
- {Schema: "public", Table: "v_alpha", Column: "id"},
- },
- },
- },
- }},
- }
- loader := newCatalogLoader(cat, meta)
- if err := loader.Load(context.Background()); err != nil {
- t.Fatalf("Load: %v", err)
- }
- // Both views should be in the catalog (real or pseudo). At least one
- // should be degraded (the lex-first member of the SCC).
- if cat.GetRelation("public", "v_alpha") == nil {
- t.Error("v_alpha missing")
- }
- if cat.GetRelation("public", "v_beta") == nil {
- t.Error("v_beta missing")
- }
- if len(loader.degraded) == 0 {
- t.Error("expected at least one degraded in SCC")
- }
- // v_alpha is lex-first by sortKey, so it should be the pseudo root.
- if _, ok := loader.degraded["rel:public.v_alpha"]; !ok {
- t.Errorf("lex-first v_alpha should be degraded, got %+v", loader.degraded)
- }
-}
-
-func TestLoader_LoaderObjectsTracksEverything(t *testing.T) {
- cat := catalog.New()
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- EnumTypes: []*metadatapb.EnumTypeMetadata{
- {Name: "e", Values: []string{"a"}},
- },
- Tables: []*metadatapb.TableMetadata{{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{{Name: "id", Type: "int4"}},
- }},
- }},
- }
- loader := newCatalogLoader(cat, meta)
- _ = loader.Load(context.Background())
- if !loader.loaderObjects["schema:public"] {
- t.Error("schema:public not tracked")
- }
- if !loader.loaderObjects["type:public.e"] {
- t.Error("type:public.e not tracked")
- }
- if !loader.loaderObjects["rel:public.t"] {
- t.Error("rel:public.t not tracked")
- }
-}
-
-func TestLoader_NilMetaIsNoop(t *testing.T) {
- cat := catalog.New()
- loader := newCatalogLoader(cat, nil)
- if err := loader.Load(context.Background()); err != nil {
- t.Errorf("Load(nil meta): %v", err)
- }
-}
-
-func TestLoader_NilCatalogErrors(t *testing.T) {
- loader := newCatalogLoader(nil, &metadatapb.DatabaseSchemaMetadata{})
- if err := loader.Load(context.Background()); err == nil {
- t.Error("expected error for nil catalog")
- }
-}
-
-// TestLoader_FunctionOverloadsAllInstalled guards against a regression where
-// multiple FunctionMetadata entries sharing a name collapsed into a single
-// loader object because the loader key didn't include the signature. Both
-// overloads must end up in the catalog under the same name.
-func TestLoader_FunctionOverloadsAllInstalled(t *testing.T) {
- cat := catalog.New()
- cat.SetSearchPath([]string{"public"})
- meta := &metadatapb.DatabaseSchemaMetadata{
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: "public",
- Functions: []*metadatapb.FunctionMetadata{
- {
- Name: "fn",
- Signature: "fn(integer)",
- Definition: "CREATE OR REPLACE FUNCTION public.fn(x int) RETURNS int LANGUAGE sql AS $$ SELECT $1 $$;",
- },
- {
- Name: "fn",
- Signature: "fn(text)",
- Definition: "CREATE OR REPLACE FUNCTION public.fn(x text) RETURNS text LANGUAGE sql AS $$ SELECT $1 $$;",
- },
- },
- }},
- }
- loader := newCatalogLoader(cat, meta)
- if err := loader.Load(context.Background()); err != nil {
- t.Fatalf("Load: %v", err)
- }
- if len(loader.degraded) != 0 {
- t.Errorf("expected zero degraded, got %+v", loader.degraded)
- }
- procs := cat.LookupProcByName("fn")
- if len(procs) < 2 {
- t.Errorf("expected both overloads installed, got %d procs", len(procs))
- }
- fnEntries := 0
- for k := range loader.loaderObjects {
- if strings.HasPrefix(k, "func:public.fn|") {
- fnEntries++
- }
- }
- if fnEntries != 2 {
- t.Errorf("expected 2 distinct fn loader-object keys, got %d: %+v",
- fnEntries, loader.loaderObjects)
- }
-}
-
-func TestLoaderTopoSort_Determinism(t *testing.T) {
- // Build a few fixtures and ensure topoSortObjects returns the same order
- // for the same input.
- objects := []*objectEntry{
- {kind: kindSchema, schema: "public", name: "public"},
- {kind: kindTable, schema: "public", name: "b", tableMeta: &metadatapb.TableMetadata{Name: "b"}},
- {kind: kindTable, schema: "public", name: "a", tableMeta: &metadatapb.TableMetadata{Name: "a"}},
- {kind: kindEnum, schema: "public", name: "e", enumMeta: &metadatapb.EnumTypeMetadata{Name: "e"}},
- }
- first := topoSortObjects(objects)
- second := topoSortObjects(objects)
- if len(first) != len(second) {
- t.Fatal("length mismatch")
- }
- for i := range first {
- if first[i].key() != second[i].key() {
- t.Errorf("non-deterministic: [%d] %s vs %s", i, first[i].key(), second[i].key())
- }
- }
-}
-
-func TestLoaderFallbackReasonString(t *testing.T) {
- cases := map[fallbackReason]string{
- reasonNone: "none",
- reasonExpectedPseudoSemantic: "expected_pseudo_semantic",
- reasonUndefinedReference: "undefined_reference",
- reasonAnalyzerUnsupported: "analyzer_unsupported",
- }
- for r, want := range cases {
- if got := r.String(); got != want {
- t.Errorf("fallbackReason(%d).String(): got %q, want %q", r, got, want)
- }
- }
-}
diff --git a/backend/plugin/parser/pg/query_span_loader_type_name.go b/backend/plugin/parser/pg/query_span_loader_type_name.go
deleted file mode 100644
index 191502df5a..0000000000
--- a/backend/plugin/parser/pg/query_span_loader_type_name.go
+++ /dev/null
@@ -1,285 +0,0 @@
-package pg
-
-import (
- "strings"
-
- "github.com/pkg/errors"
-
- "github.com/bytebase/omni/pg/ast"
-)
-
-// UserTypeRef names a user-defined type referenced by a column, function
-// parameter, or function return type. Extracted from type strings produced by
-// the PG sync layer (sync.go:820-872).
-type UserTypeRef struct {
- Schema string
- Name string
-}
-
-// typeNameFromString parses a PG type string into an *ast.TypeName by running
-// it through omni's SELECT parser via the "SELECT NULL::<type>" trick. This
-// reuses the stable SELECT parse path and avoids the DDL parse path that
-// breaks on BYT-9215 / BYT-9261 class inputs.
-//
-// Callers must only pass values originating from bytebase sync metadata. The
-// guardrails (single-statement, single-target) reject most malformed inputs
-// but are not a sanitization layer for arbitrary user input.
-func typeNameFromString(typeStr string) (*ast.TypeName, error) {
- stmts, err := ParsePg("SELECT NULL::" + typeStr)
- if err != nil {
- return nil, errors.Wrapf(err, "parse type %q", typeStr)
- }
- if len(stmts) != 1 {
- return nil, errors.Errorf("type %q: expected 1 statement, got %d", typeStr, len(stmts))
- }
- sel, ok := stmts[0].AST.(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("type %q: expected SelectStmt, got %T", typeStr, stmts[0].AST)
- }
- if sel.TargetList == nil || len(sel.TargetList.Items) != 1 {
- return nil, errors.Errorf("type %q: expected 1 target", typeStr)
- }
- rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)
- if !ok {
- return nil, errors.Errorf("type %q: expected ResTarget, got %T", typeStr, sel.TargetList.Items[0])
- }
- cast, ok := rt.Val.(*ast.TypeCast)
- if !ok {
- return nil, errors.Errorf("type %q: expected TypeCast, got %T", typeStr, rt.Val)
- }
- return cast.TypeName, nil
-}
-
-// extractUserTypeRefs returns the user-defined type references embedded in a
-// PG type string. Built-in types, PG internal array forms (_name), and
-// system-schema-qualified types return nil.
-//
-// Soundness rule (hard contract C5): false negatives are acceptable (the
-// loader's pseudo fallback catches them); false positives are not, because
-// they invent edges in the dependency graph.
-func extractUserTypeRefs(typeStr string) []UserTypeRef {
- if typeStr == "" {
- return nil
- }
- base := stripTypeModifiers(typeStr)
- if base == "" || isBuiltinType(base) {
- return nil
- }
- if strings.HasPrefix(base, "_") {
- // PG internal array form. The element may be a user type but the
- // schema is not recoverable from this representation — see the
- // sync.go:834 note in the plan.
- return nil
- }
- schema, name, ok := splitQualifiedName(base)
- if !ok {
- return nil
- }
- if IsSystemSchema(schema) {
- return nil
- }
- return []UserTypeRef{{Schema: schema, Name: name}}
-}
-
-// stripTypeModifiers normalizes a PG type string to its bare base-type form
-// for allow-list lookup. It strips:
-// - parenthesized type modifiers: "numeric(10,2)" -> "numeric"
-// - trailing time-zone suffixes: "timestamp(3) with time zone" -> "timestamp"
-// - surrounding whitespace
-//
-// The result may still be a qualified name like "public.task_status"; that is
-// the caller's job to split.
-func stripTypeModifiers(typeStr string) string {
- s := strings.TrimSpace(typeStr)
- // Array suffixes wrap the element type: "public.addr[]" refers to
- // public.addr. Quoted identifiers end with a quote, so a genuine name
- // containing "[]" is never clipped.
- for strings.HasSuffix(s, "[]") {
- s = strings.TrimSpace(strings.TrimSuffix(s, "[]"))
- }
- s = stripTimeZoneSuffix(s)
- s = stripParens(s)
- return strings.TrimSpace(s)
-}
-
-// stripTimeZoneSuffix removes a trailing "with time zone" / "without time
-// zone" clause. Idempotent on inputs that do not carry the suffix.
-func stripTimeZoneSuffix(s string) string {
- lower := strings.ToLower(s)
- for _, suffix := range []string{" with time zone", " without time zone"} {
- if strings.HasSuffix(lower, suffix) {
- return s[:len(s)-len(suffix)]
- }
- }
- return s
-}
-
-// stripParens removes the first parenthesized chunk from a type string.
-// "numeric(10,2)" -> "numeric"
-// "timestamp(3)" -> "timestamp"
-// "character varying(255)" -> "character varying"
-func stripParens(s string) string {
- before, _, found := strings.Cut(s, "(")
- if !found {
- return s
- }
- return strings.TrimSpace(before)
-}
-
-// splitQualifiedName splits a dotted type name into (schema, name). Quoted
-// identifiers are supported.
-// "public.task_status" -> ("public", "task_status", true)
-// "myschema.my_domain" -> ("myschema", "my_domain", true)
-// `"MySchema"."MyType"` -> ("MySchema", "MyType", true)
-// "task_status" -> ("", "", false)
-func splitQualifiedName(s string) (schema, name string, ok bool) {
- parts, err := splitDottedIdentifier(s)
- if err != nil || len(parts) != 2 {
- return "", "", false
- }
- return parts[0], parts[1], true
-}
-
-// splitDottedIdentifier splits "a.b.c" into ["a", "b", "c"], handling
-// double-quoted identifiers like `"weird name"."another"`.
-func splitDottedIdentifier(s string) ([]string, error) {
- var parts []string
- var cur strings.Builder
- inQuote := false
- for i := 0; i < len(s); i++ {
- c := s[i]
- switch {
- case c == '"':
- inQuote = !inQuote
- case c == '.' && !inQuote:
- parts = append(parts, cur.String())
- cur.Reset()
- default:
- cur.WriteByte(c)
- }
- }
- if inQuote {
- return nil, errors.Errorf("unbalanced quote in %q", s)
- }
- parts = append(parts, cur.String())
- return parts, nil
-}
-
-// isBuiltinType reports whether a bare (modifier-stripped) type name is a
-// PostgreSQL built-in type. Membership is exact, case-insensitive.
-//
-// The list is hand-maintained. New entries must come with a golden test in
-// query_span_e3_test.go.
-func isBuiltinType(s string) bool {
- _, ok := pgBuiltinTypes[strings.ToLower(s)]
- return ok
-}
-
-// pgBuiltinTypes is the allow-list of PostgreSQL built-in type base names
-// (without type modifiers, time-zone suffixes, or array brackets).
-//
-// Sources: PostgreSQL data types reference + the transforms in
-// backend/plugin/db/pg/sync.go:820-872 for the specific surface bytebase
-// currently emits.
-var pgBuiltinTypes = map[string]struct{}{
- // Numeric.
- "smallint": {},
- "integer": {},
- "int": {},
- "int2": {},
- "int4": {},
- "int8": {},
- "bigint": {},
- "decimal": {},
- "numeric": {},
- "real": {},
- "double precision": {},
- "float4": {},
- "float8": {},
- "smallserial": {},
- "serial": {},
- "bigserial": {},
- "money": {},
-
- // Character / binary.
- "character": {},
- "character varying": {},
- "char": {},
- "varchar": {},
- "text": {},
- "bytea": {},
- "bpchar": {},
- "name": {},
-
- // Bit.
- "bit": {},
- "bit varying": {},
- "varbit": {},
-
- // Date / time.
- "date": {},
- "time": {},
- "timetz": {},
- "timestamp": {},
- "timestamptz": {},
- "interval": {},
-
- // Boolean and friends.
- "boolean": {},
- "bool": {},
-
- // Structured.
- "json": {},
- "jsonb": {},
- "xml": {},
- "uuid": {},
-
- // Geometric.
- "point": {},
- "line": {},
- "lseg": {},
- "box": {},
- "path": {},
- "polygon": {},
- "circle": {},
-
- // Network.
- "cidr": {},
- "inet": {},
- "macaddr": {},
- "macaddr8": {},
-
- // Text search.
- "tsvector": {},
- "tsquery": {},
-
- // PG pseudo / system.
- "void": {},
- "record": {},
- "anyelement": {},
- "anyarray": {},
- "anynonarray": {},
- "anyenum": {},
- "anyrange": {},
- "any": {},
- "trigger": {},
- "event_trigger": {},
- "cstring": {},
- "internal": {},
- "language_handler": {},
- "fdw_handler": {},
- "index_am_handler": {},
- "tsm_handler": {},
- "pg_lsn": {},
- "oid": {},
- "regclass": {},
- "regproc": {},
- "regprocedure": {},
- "regoper": {},
- "regoperator": {},
- "regtype": {},
- "regconfig": {},
- "regdictionary": {},
- "regnamespace": {},
- "regrole": {},
-}
diff --git a/backend/plugin/parser/pg/query_span_plpgsql.go b/backend/plugin/parser/pg/query_span_plpgsql.go
index 1ed87bdfe3..34ed565fcf 100644
--- a/backend/plugin/parser/pg/query_span_plpgsql.go
+++ b/backend/plugin/parser/pg/query_span_plpgsql.go
@@ -160,10 +160,6 @@ func (e *omniQuerySpanExtractor) analyzeSQLBody(body string) ([]base.SourceColum
}
query, err := e.cat.AnalyzeSelectStmt(selStmt)
if err != nil {
- // Record the classifier verdict for test inspection; fallback
- // below always runs.
- e.lastFallbackReason = classifyAnalyzeError(err)
-
// Fall back to parse-tree extraction when analysis fails
// (e.g., nested function calls, parameter references).
fallbackResults := e.extractFallbackColumns(selStmt)
@@ -499,10 +495,6 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedSQL(sql string) []base.SourceColumnSet
query, err := a.extractor.cat.AnalyzeSelectStmt(selStmt)
if err != nil {
- // Record the classifier verdict for test inspection; fallback below
- // always runs.
- a.extractor.lastFallbackReason = classifyAnalyzeError(err)
-
// When full analysis fails (e.g., type mismatches with aggregate functions),
// fall back to extracting column references from the parse tree and resolving
// them against known tables in the catalog.
diff --git a/backend/plugin/parser/pg/query_span_test.go b/backend/plugin/parser/pg/query_span_test.go
index 5285f85042..cdfde7d715 100644
--- a/backend/plugin/parser/pg/query_span_test.go
+++ b/backend/plugin/parser/pg/query_span_test.go
@@ -4,9 +4,14 @@ import (
"context"
"io"
"os"
+ "os/exec"
+ "regexp"
+ "runtime/debug"
"testing"
metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/bytebase/omni/pg/ast"
+ "github.com/bytebase/omni/pg/catalog"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
@@ -104,3 +109,686 @@ func buildMockDatabaseMetadataGetter(databaseMetadata []*metadatapb.DatabaseSche
return names, nil
}
}
+
+func TestQuerySpanSurvivesABadQuotedIdentifier(t *testing.T) {
+ // A table name carrying a character sequence that a deparse-and-reparse
+ // loop chokes on (BYT-9215). Loading the snapshot never renders DDL, so the
+ // table installs and unrelated queries succeed either way.
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{
+ // A quoted identifier containing an apostrophe — the kind of
+ // input BYT-9215 reported failing under Exec(ddl).
+ {
+ Name: "'weird'table",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int4"},
+ },
+ },
+ // An unrelated healthy table.
+ {
+ Name: "accounts",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int4"},
+ {Name: "email", Type: "text"},
+ },
+ },
+ },
+ }},
+ }
+
+ span := mustGetQuerySpan(t, meta, `SELECT id, email FROM accounts`)
+ if len(span.Results) != 2 {
+ t.Fatalf("accounts query: got %d results, want 2", len(span.Results))
+ }
+ for _, r := range span.Results {
+ if len(r.SourceColumns) == 0 {
+ t.Errorf("result %q has empty sources (lineage lost)", r.Name)
+ }
+ }
+}
+
+func TestGetQuerySpanWithSelectedSchemaFallsBackToPublic(t *testing.T) {
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{
+ {
+ Name: "app",
+ },
+ {
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "customer",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "ssn", Type: "text"},
+ },
+ }},
+ },
+ },
+ }
+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})
+ span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{
+ GetDatabaseMetadataFunc: getter,
+ ListDatabaseNamesFunc: lister,
+ }, base.Statement{Text: `SELECT ssn FROM customer`}, "db", "app", false)
+ if err != nil {
+ t.Fatalf("GetQuerySpan: %v", err)
+ }
+ if len(span.Results) != 1 {
+ t.Fatalf("got %d results, want 1", len(span.Results))
+ }
+ want := base.ColumnResource{Database: "db", Schema: "public", Table: "customer", Column: "ssn"}
+ if _, ok := span.Results[0].SourceColumns[want]; !ok {
+ t.Fatalf("result %q missing public fallback source %+v; have %+v", span.Results[0].Name, want, span.Results[0].SourceColumns)
+ }
+ wantAccess := base.ColumnResource{Database: "db", Schema: "public", Table: "customer"}
+ if _, ok := span.SourceColumns[wantAccess]; !ok {
+ t.Fatalf("span missing public fallback access source %+v; have %+v", wantAccess, span.SourceColumns)
+ }
+}
+
+func TestGetQuerySpanSelectedSchemaTakesPrecedenceOverPublic(t *testing.T) {
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{
+ {
+ Name: "app",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "customer",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "email", Type: "text"},
+ },
+ }},
+ },
+ {
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "customer",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "ssn", Type: "text"},
+ },
+ }},
+ },
+ },
+ }
+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})
+ span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{
+ GetDatabaseMetadataFunc: getter,
+ ListDatabaseNamesFunc: lister,
+ }, base.Statement{Text: `SELECT email FROM customer`}, "db", "app", false)
+ if err != nil {
+ t.Fatalf("GetQuerySpan: %v", err)
+ }
+ if len(span.Results) != 1 {
+ t.Fatalf("got %d results, want 1", len(span.Results))
+ }
+ want := base.ColumnResource{Database: "db", Schema: "app", Table: "customer", Column: "email"}
+ if _, ok := span.Results[0].SourceColumns[want]; !ok {
+ t.Fatalf("result %q missing selected-schema source %+v; have %+v", span.Results[0].Name, want, span.Results[0].SourceColumns)
+ }
+}
+
+func TestQuerySpanSurvivesABrokenEnum(t *testing.T) {
+ // The table references a user-defined type the snapshot never declares, so
+ // it installs as a stand-in with text columns. A query against it still
+ // returns the column names the snapshot records.
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "tasks",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int4"},
+ // References an enum that is NOT declared in metadata —
+ // buildCreateStmt will succeed (typeNameFromString works)
+ // but DefineRelation fails (enum not in catalog).
+ {Name: "status", Type: "public.nonexistent_enum"},
+ {Name: "title", Type: "text"},
+ },
+ }},
+ }},
+ }
+
+ span := mustGetQuerySpan(t, meta, `SELECT id, status, title FROM tasks`)
+ if len(span.Results) != 3 {
+ t.Fatalf("tasks query: got %d results, want 3", len(span.Results))
+ }
+ wantNames := []string{"id", "status", "title"}
+ for i, want := range wantNames {
+ if i >= len(span.Results) {
+ break
+ }
+ if span.Results[i].Name != want {
+ t.Errorf("result[%d]: got %q, want %q", i, span.Results[i].Name, want)
+ }
+ if len(span.Results[i].SourceColumns) == 0 {
+ t.Errorf("result[%d] (%s): empty sources", i, span.Results[i].Name)
+ }
+ }
+}
+
+func TestQuerySpanKeepsAHealthyNeighborOfABrokenTable(t *testing.T) {
+ // A query against a healthy table must succeed even when an unrelated
+ // table in the same schema references a broken type. This is the core
+ // blast-radius claim: one bad object does not poison all queries.
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{
+ {
+ Name: "broken",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int4"},
+ {Name: "bad", Type: "public.nonexistent_type"},
+ },
+ },
+ {
+ Name: "healthy",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int4"},
+ {Name: "label", Type: "text"},
+ },
+ },
+ },
+ }},
+ }
+
+ span := mustGetQuerySpan(t, meta, `SELECT id, label FROM healthy`)
+ if len(span.Results) != 2 {
+ t.Fatalf("healthy query: got %d results, want 2", len(span.Results))
+ }
+ for _, r := range span.Results {
+ if len(r.SourceColumns) == 0 {
+ t.Errorf("result %q lost lineage because of unrelated broken table", r.Name)
+ }
+ }
+}
+
+func TestQuerySpanResolvesAViewOverABrokenTable(t *testing.T) {
+ // Chain: enum missing → table T references it, so T stands in → view V over
+ // T installs against that stand-in → a query on V still resolves.
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "records",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int4"},
+ {Name: "status", Type: "public.nonexistent_enum"},
+ {Name: "title", Type: "text"},
+ },
+ }},
+ Views: []*metadatapb.ViewMetadata{{
+ Name: "records_view",
+ Definition: "SELECT id, title, status FROM records",
+ DependencyColumns: []*metadatapb.DependencyColumn{
+ {Schema: "public", Table: "records", Column: "id"},
+ {Schema: "public", Table: "records", Column: "title"},
+ {Schema: "public", Table: "records", Column: "status"},
+ },
+ }},
+ }},
+ }
+
+ span := mustGetQuerySpan(t, meta, `SELECT id, title, status FROM records_view`)
+ if len(span.Results) != 3 {
+ t.Fatalf("view query: got %d results, want 3", len(span.Results))
+ }
+ for i, r := range span.Results {
+ if len(r.SourceColumns) == 0 {
+ t.Errorf("result[%d] (%s) lost lineage", i, r.Name)
+ }
+ }
+}
+
+func TestQuerySpanOnAHealthySchema(t *testing.T) {
+ // Baseline: a clean schema produces exact lineage down to
+ // (schema, table, column).
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "users",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int4"},
+ {Name: "email", Type: "text"},
+ {Name: "created_at", Type: "timestamp with time zone"},
+ },
+ }},
+ }},
+ }
+
+ span := mustGetQuerySpan(t, meta, `SELECT id, email FROM users WHERE id > 0`)
+ if len(span.Results) != 2 {
+ t.Fatalf("got %d results, want 2", len(span.Results))
+ }
+ mustHaveExactSource(t, span.Results[0], "users", "id")
+ mustHaveExactSource(t, span.Results[1], "users", "email")
+}
+
+func TestQuerySpanKeepsLineageThroughAPartition(t *testing.T) {
+ // Masking resolves a partition to its parent table, which needs the read to
+ // trace to the partition's columns.
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "orders",
+ Columns: []*metadatapb.ColumnMetadata{{Name: "id", Type: "int4"}, {Name: "ssn", Type: "text"}},
+ Partitions: []*metadatapb.TablePartitionMetadata{{Name: "orders_2024"}},
+ }},
+ }},
+ }
+
+ span := mustGetQuerySpan(t, meta, `SELECT ssn FROM orders_2024`)
+ if len(span.Results) != 1 {
+ t.Fatalf("got %d results, want 1", len(span.Results))
+ }
+ mustHaveExactSource(t, span.Results[0], "orders_2024", "ssn")
+}
+
+func TestQuerySpanResolvesADeclaredEnum(t *testing.T) {
+ // When an enum IS declared in metadata, the table installs as real and
+ // enum-typed columns resolve through their real type. This is the
+ // positive control for TestQuerySpanSurvivesABrokenEnum.
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ EnumTypes: []*metadatapb.EnumTypeMetadata{{
+ Name: "task_status",
+ Values: []string{"pending", "running", "done"},
+ }},
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "tasks",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int4"},
+ {Name: "status", Type: "public.task_status"},
+ },
+ }},
+ }},
+ }
+
+ span := mustGetQuerySpan(t, meta, `SELECT id, status FROM tasks WHERE status = 'pending'`)
+ if len(span.Results) != 2 {
+ t.Fatalf("got %d results, want 2", len(span.Results))
+ }
+ mustHaveExactSource(t, span.Results[0], "tasks", "id")
+ mustHaveExactSource(t, span.Results[1], "tasks", "status")
+}
+
+func TestQuerySpanOnACorrelatedRangeFunctionSubquery(t *testing.T) {
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{
+ {
+ Name: "compliance_case_record_audits",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "case_id", Type: "text"},
+ {Name: "entity_id", Type: "text"},
+ {Name: "reason", Type: "text"},
+ {Name: "remark", Type: "text"},
+ {Name: "case_record", Type: "jsonb"},
+ {Name: "reviewer_by", Type: "text"},
+ {Name: "deleted_at", Type: "timestamptz"},
+ },
+ },
+ {
+ Name: "compliance_cases",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "case_id", Type: "text"},
+ {Name: "case_info", Type: "jsonb"},
+ {Name: "deleted_at", Type: "timestamptz"},
+ },
+ },
+ },
+ }},
+ }
+ sql := `
+select
+ a.case_id,
+ a.entity_id as uid,
+ a.reason,
+ a.remark,
+ (
+ select elem->>'matchedDateTimeValue'
+ from jsonb_array_elements(a.case_record::jsonb->'secondaryFieldResults') elem
+ where elem->>'typeId' = '******'
+ limit 1
+ ) as wc_dob,
+ (
+ select elem->>'matchedValue'
+ from jsonb_array_elements(a.case_record::jsonb->'secondaryFieldResults') elem
+ where elem->>'typeId' = '******'
+ limit 1
+ ) as wc_citizenship,
+ c.case_info->>'birth_date' as kyc_dob,
+ c.case_info->>'nationality' as kyc_citizenship,
+ a.reviewer_by as review_by
+from compliance_case_record_audits a
+inner join compliance_cases c
+ on c.case_id = a.case_id and c.deleted_at is null
+where a.reviewer_by in ('****** ', '******')
+ and a.deleted_at is null;
+`
+ span := mustGetQuerySpan(t, meta, sql)
+ if len(span.Results) != 9 {
+ t.Fatalf("got %d results, want 9", len(span.Results))
+ }
+ mustHaveExactSource(t, span.Results[4], "compliance_case_record_audits", "case_record")
+ mustHaveExactSource(t, span.Results[5], "compliance_case_record_audits", "case_record")
+ mustHaveExactSource(t, span.Results[6], "compliance_cases", "case_info")
+ mustHaveExactSource(t, span.Results[7], "compliance_cases", "case_info")
+}
+
+func TestQuerySpanKeepsJSONBLineageThroughCTEsAndALateralJoin(t *testing.T) {
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "ai_conversation",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "text"},
+ {Name: "parts", Type: "jsonb"},
+ },
+ }},
+ }},
+ }
+
+ tests := []struct {
+ name string
+ sql string
+ }{
+ {
+ name: "single_cte",
+ sql: `
+WITH convs AS (
+ SELECT ac.id, ac.parts AS conv_parts
+ FROM ai_conversation ac
+ LIMIT 1
+)
+SELECT
+ LEFT(
+ COALESCE(
+ (SELECT string_agg(v::text, ' ')
+ FROM jsonb_array_elements_text(
+ jsonb_path_query_array(t.part->'bodyData', 'strict $.**.text', '{}'::jsonb, true)
+ ) AS v),
+ '<no text>'
+ ),
+ 2000
+ ) AS content
+FROM convs c
+CROSS JOIN LATERAL jsonb_array_elements(c.conv_parts) WITH ORDINALITY AS t(part, part_ord)
+WHERE t.part->>'type' IN ('prompt', 'text');
+`,
+ },
+ {
+ name: "multiple_ctes",
+ sql: `
+WITH cte1 AS (
+ SELECT ac.id, ac.parts AS conv_parts
+ FROM ai_conversation ac
+ LIMIT 1
+),
+cte2 AS (
+ SELECT id, conv_parts
+ FROM cte1
+)
+SELECT
+ LEFT(
+ COALESCE(
+ (SELECT string_agg(v::text, ' ')
+ FROM jsonb_array_elements_text(
+ jsonb_path_query_array(t.part->'bodyData', 'strict $.**.text', '{}'::jsonb, true)
+ ) AS v),
+ '<no text>'
+ ),
+ 2000
+ ) AS content
+FROM cte2 c
+CROSS JOIN LATERAL jsonb_array_elements(c.conv_parts) WITH ORDINALITY AS t(part, part_ord)
+WHERE t.part->>'type' IN ('prompt', 'text');
+`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ span := mustGetQuerySpan(t, meta, tt.sql)
+ if len(span.Results) != 1 {
+ t.Fatalf("got %d results, want 1", len(span.Results))
+ }
+ mustHaveExactSource(t, span.Results[0], "ai_conversation", "parts")
+ })
+ }
+}
+
+func TestAppendQueryDoesNotMutateSharedStack(t *testing.T) {
+ outer := &catalog.Query{}
+ parent := &catalog.Query{}
+ current := &catalog.Query{}
+ nested := &catalog.Query{}
+ queryStack := []*catalog.Query{outer, parent, current}
+
+ nextStack := appendQuery(queryStack[:2], nested)
+
+ if queryStack[2] != current {
+ t.Fatalf("appendQuery mutated caller stack: got %p, want %p", queryStack[2], current)
+ }
+ if len(nextStack) != 3 || nextStack[2] != nested {
+ t.Fatalf("appendQuery returned unexpected stack: %+v", nextStack)
+ }
+}
+
+func TestWalkExprStopsOnSelfReferentialFunction(t *testing.T) {
+ mustNotOverflow(t, func() {
+ varExpr := &catalog.VarExpr{RangeIdx: 0, AttNum: 1}
+ q := &catalog.Query{
+ TargetList: []*catalog.TargetEntry{{
+ Expr: varExpr,
+ }},
+ RangeTable: []*catalog.RangeTableEntry{{
+ Kind: catalog.RTEFunction,
+ FuncExprs: []catalog.AnalyzedExpr{&catalog.FuncCallExpr{
+ Args: []catalog.AnalyzedExpr{&catalog.OpExpr{
+ Left: varExpr,
+ Right: &catalog.ConstExpr{
+ Value: "items",
+ },
+ }},
+ }},
+ }},
+ }
+ extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})
+ extractor.cat = catalog.New()
+ extractor.walkExpr(q, q.TargetList[0].Expr, make(base.SourceColumnSet))
+ })
+}
+
+func TestIsUltimatelyPlainColumnStopsOnCyclicSubquery(t *testing.T) {
+ mustNotOverflow(t, func() {
+ varExpr := &catalog.VarExpr{RangeIdx: 0, AttNum: 1}
+ q := &catalog.Query{
+ TargetList: []*catalog.TargetEntry{{
+ Expr: varExpr,
+ }},
+ RangeTable: []*catalog.RangeTableEntry{{
+ Kind: catalog.RTESubquery,
+ }},
+ }
+ q.RangeTable[0].Subquery = q
+ _ = isUltimatelyPlainColumn(q, varExpr)
+ })
+}
+
+func TestCollectQueryPredicateColumnsStopsOnCyclicQuery(t *testing.T) {
+ mustNotOverflow(t, func() {
+ q := &catalog.Query{}
+ q.CTEList = []*catalog.CommonTableExprQ{{Query: q}}
+ extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})
+ analyzer := &plpgsqlAnalyzer{
+ extractor: extractor,
+ scope: newVariableScope(nil),
+ }
+ analyzer.collectQueryPredicateColumns(q)
+ })
+}
+
+func TestExtractLineageStopsOnCyclicSetOp(t *testing.T) {
+ mustNotOverflow(t, func() {
+ q := &catalog.Query{SetOp: catalog.SetOpUnion}
+ q.LArg = q
+ q.RArg = &catalog.Query{TargetList: []*catalog.TargetEntry{{
+ Expr: &catalog.ConstExpr{
+ Value: "1",
+ },
+ }}}
+ selStmt := &ast.SelectStmt{}
+ selStmt.Larg = selStmt
+ selStmt.Rarg = &ast.SelectStmt{}
+ extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})
+ extractor.extractLineage(q, selStmt)
+ })
+}
+
+func TestResolveThroughCTEStopsOnCyclicCTE(t *testing.T) {
+ mustNotOverflow(t, func() {
+ cteSel := &ast.SelectStmt{
+ TargetList: &ast.List{Items: []ast.Node{&ast.ResTarget{
+ Name: "x",
+ Val: &ast.ColumnRef{Fields: &ast.List{Items: []ast.Node{
+ &ast.String{Str: "c"},
+ &ast.String{Str: "x"},
+ }}},
+ }}},
+ FromClause: &ast.List{Items: []ast.Node{&ast.RangeVar{Relname: "c"}}},
+ }
+ extractor := newOmniQuerySpanExtractor(testDB, []string{testSchema}, base.GetQuerySpanContext{})
+ extractor.cat = catalog.New()
+ analyzer := &plpgsqlAnalyzer{
+ extractor: extractor,
+ scope: newVariableScope(nil),
+ cteMap: map[string]*ast.SelectStmt{"c": cteSel},
+ }
+ analyzer.resolveThroughCTE(cteSel, "x", nil, make(base.SourceColumnSet))
+ })
+}
+
+// ---------- helpers ----------
+
+// testDB / testSchema are the fixed database and schema names these tests use;
+// as constants they keep the call sites terse and satisfy the unparam linter.
+const (
+ testDB = "db"
+ testSchema = "public"
+)
+
+// noOverflowEnv marks the subprocess mustNotOverflow spawns.
+const noOverflowEnv = "BYTEBASE_TEST_NO_OVERFLOW"
+
+// mustNotOverflow runs fn in a subprocess with a 1 MB stack, so recursion that
+// does not terminate fails this test instead of taking the whole run down. The
+// subprocess re-runs this same test, which then takes the fn branch.
+func mustNotOverflow(t *testing.T, fn func()) {
+ t.Helper()
+ if os.Getenv(noOverflowEnv) == "1" {
+ debug.SetMaxStack(1 << 20)
+ fn()
+ return
+ }
+ cmd := exec.Command(os.Args[0], "-test.run=^"+regexp.QuoteMeta(t.Name())+"$")
+ cmd.Env = append(os.Environ(), noOverflowEnv+"=1")
+ if output, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("recursion did not terminate: %v\n%s", err, output)
+ }
+}
+
+func mustGetQuerySpan(t *testing.T, meta *metadatapb.DatabaseSchemaMetadata, sql string) *base.QuerySpan {
+ t.Helper()
+ getter, lister := buildMockDatabaseMetadataGetter([]*metadatapb.DatabaseSchemaMetadata{meta})
+ span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{
+ GetDatabaseMetadataFunc: getter,
+ ListDatabaseNamesFunc: lister,
+ }, base.Statement{Text: sql}, testDB, "", false)
+ if err != nil {
+ t.Fatalf("GetQuerySpan(%q): %v", sql, err)
+ }
+ if span == nil {
+ t.Fatalf("GetQuerySpan(%q): nil span", sql)
+ }
+ return span
+}
+
+func mustHaveExactSource(t *testing.T, result base.QuerySpanResult, table, column string) {
+ t.Helper()
+ want := base.ColumnResource{
+ Database: testDB,
+ Schema: testSchema,
+ Table: table,
+ Column: column,
+ }
+ for src := range result.SourceColumns {
+ if src == want {
+ return
+ }
+ }
+ t.Errorf("result %q missing source %+v; have %+v", result.Name, want, result.SourceColumns)
+}
+
+// TestRangeVarFallbackSkipsCompositeTypes proves the query-span fallback does
+// not expand a standalone composite type as a FROM source — PostgreSQL
+// rejects composite types as table sources.
+func TestRangeVarFallbackSkipsCompositeTypes(t *testing.T) {
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Schemas: []*metadatapb.SchemaMetadata{
+ {
+ Name: "public",
+ CompositeTypes: []*metadatapb.CompositeTypeMetadata{
+ {
+ Name: "addr",
+ Attributes: []*metadatapb.CompositeTypeAttribute{
+ {Name: "street", Type: "text"},
+ },
+ },
+ },
+ Tables: []*metadatapb.TableMetadata{
+ {
+ Name: "t",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "integer"},
+ },
+ },
+ },
+ },
+ },
+ }
+
+ cat := catalog.New()
+ cat.SetSearchPath([]string{"public"})
+ if _, err := cat.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{}); err != nil {
+ t.Fatalf("LoadMetadata: %v", err)
+ }
+
+ e := &omniQuerySpanExtractor{
+ cat: cat,
+ searchPath: []string{"public"},
+ defaultDatabase: "db",
+ }
+ if results := e.extractColumnsFromRangeVar(&ast.RangeVar{Schemaname: "public", Relname: "addr"}); results != nil {
+ t.Errorf("composite type must not expand as a FROM source, got %d columns", len(results))
+ }
+ if results := e.extractColumnsFromRangeVar(&ast.RangeVar{Schemaname: "public", Relname: "t"}); len(results) != 1 {
+ t.Errorf("real table must still expand, got %d columns", len(results))
+ }
+}
diff --git a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go
index 2987300059..28adc2e193 100644
--- a/backend/plugin/parser/pg/query_span_unresolved_columns_test.go
+++ b/backend/plugin/parser/pg/query_span_unresolved_columns_test.go
@@ -42,15 +42,11 @@ func healthySchema() *metadatapb.DatabaseSchemaMetadata {
// A table with no columns models a snapshot from a sync before #20581.
func degradedSchema() *metadatapb.DatabaseSchemaMetadata {
- return degradedSchemaNamed("t")
-}
-
-func degradedSchemaNamed(table string) *metadatapb.DatabaseSchemaMetadata {
return &metadatapb.DatabaseSchemaMetadata{
Name: "db",
Schemas: []*metadatapb.SchemaMetadata{{
Name: "public",
- Tables: []*metadatapb.TableMetadata{{Name: table}},
+ Tables: []*metadatapb.TableMetadata{{Name: "t"}},
}},
}
}
@@ -100,6 +96,8 @@ func TestUnresolvedColumnsSignalFiresOnDegradedSnapshot(t *testing.T) {
"WITH t AS (SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t)) AS c FROM (SELECT 1) x) SELECT * FROM t",
"WITH t AS (SELECT string_agg('x', ',' ORDER BY (SELECT count(*) FROM t)) AS c FROM (SELECT 1) x) SELECT * FROM t",
`WITH "T" AS (SELECT 1 AS n) SELECT count(*) FILTER (WHERE EXISTS (SELECT 1 FROM t)) FROM "T"`,
+ "SELECT * FROM generate_series(1, (SELECT count(*)::int FROM public.t))",
+ "SELECT * FROM (VALUES ((SELECT count(*) FROM public.t))) v(x)",
}
for _, statement := range statements {
t.Run(statement, func(t *testing.T) {
@@ -243,20 +241,3 @@ func TestUnresolvedColumnsSignalResolvesRelationsNotRoutines(t *testing.T) {
require.Nil(t, span.UnresolvedColumnsError,
"the sequence in a is the relation this query reads, and a sequence carries no column list to judge")
}
-
-// BYT-10076 tracks reads omitted from the access set; these cases record that gap.
-func TestUnresolvedColumnsSignalNotCoveredShapes(t *testing.T) {
- notCovered := map[string]string{
- "subquery in a FROM-clause function argument": "SELECT * FROM generate_series(1, (SELECT count(*)::int FROM public.d))",
- "subquery inside a VALUES list": "SELECT * FROM (VALUES ((SELECT count(*) FROM public.d))) v(x)",
- }
- for name, statement := range notCovered {
- t.Run(name, func(t *testing.T) {
- span := spanFor(t, statement, degradedSchemaNamed("d"))
- require.Empty(t, span.SourceColumns,
- "the premise of this gap is that the access set is empty; if this fails the gap may have been closed upstream")
- require.Nil(t, span.UnresolvedColumnsError,
- "documented gap: no access reported, so nothing to check (BYT-10076)")
- })
- }
-}
diff --git a/backend/plugin/parser/redshift/completion.go b/backend/plugin/parser/redshift/completion.go
index 1c7051caee..5eaa935c68 100644
--- a/backend/plugin/parser/redshift/completion.go
+++ b/backend/plugin/parser/redshift/completion.go
@@ -2,13 +2,11 @@ package redshift
import (
"context"
- "fmt"
"slices"
"strings"
"unicode"
"unicode/utf8"
- metadatapb "github.com/bytebase/omni/metadata"
redshiftcatalog "github.com/bytebase/omni/redshift/catalog"
redshiftcompletion "github.com/bytebase/omni/redshift/completion"
@@ -21,9 +19,9 @@ func init() {
}
// Completion returns Redshift auto-completion candidates backed by omni's
-// parser-native completer. Bytebase metadata is copied into an omni Redshift
-// catalog with minimal DDL so the completer can resolve schemas, relations,
-// columns, sequences, and Redshift-specific grammar slots.
+// parser-native completer. Bytebase metadata is loaded into an omni Redshift
+// catalog so the completer can resolve schemas, relations, columns, sequences,
+// and Redshift-specific grammar slots.
func Completion(ctx context.Context, cCtx base.CompletionContext, statement string, caretLine int, caretOffset int) ([]base.Candidate, error) {
cat := buildCompletionCatalog(ctx, cCtx)
@@ -102,48 +100,7 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext) *r
}
cat := redshiftcatalog.New()
- for _, schemaName := range metadata.ListSchemaNames() {
- schemaMeta := metadata.GetSchemaMetadata(schemaName)
- if schemaMeta == nil {
- continue
- }
- execCompletionDDL(cat, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s;", quoteIdent(schemaName)))
-
- for _, tableName := range schemaMeta.ListTableNames() {
- tableMeta := schemaMeta.GetTable(tableName)
- if tableMeta == nil {
- continue
- }
- execCompletionDDL(cat, createTableDDL(schemaName, tableName, tableMeta.GetProto().GetColumns()))
- }
- for _, tableName := range schemaMeta.ListForeignTableNames() {
- tableMeta := schemaMeta.GetExternalTable(tableName)
- if tableMeta == nil {
- continue
- }
- execCompletionDDL(cat, createTableDDL(schemaName, tableName, tableMeta.GetProto().GetColumns()))
- }
- for _, viewName := range schemaMeta.ListViewNames() {
- viewMeta := schemaMeta.GetView(viewName)
- if viewMeta == nil {
- continue
- }
- execCompletionDDL(cat, createViewDDL("VIEW", schemaName, viewName, viewMeta.GetColumns()))
- }
- for _, viewName := range schemaMeta.ListMaterializedViewNames() {
- viewMeta := schemaMeta.GetMaterializedView(viewName)
- if viewMeta == nil {
- continue
- }
- cat.SetSearchPath([]string{schemaName, "public"})
- if viewMeta.GetDefinition() == "" || !execCompletionDDL(cat, createMaterializedViewDDL(schemaName, viewName, viewMeta.GetDefinition())) {
- execCompletionDDL(cat, createViewDDL("MATERIALIZED VIEW", schemaName, viewName, nil))
- }
- }
- for _, sequenceName := range schemaMeta.ListSequenceNames() {
- execCompletionDDL(cat, fmt.Sprintf("CREATE SEQUENCE %s.%s;", quoteIdent(schemaName), quoteIdent(sequenceName)))
- }
- }
+ cat.LoadMetadata(metadata.GetProto())
searchPath := []string{"public"}
if cCtx.DefaultSchema != "" {
@@ -153,100 +110,6 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext) *r
return cat
}
-func execCompletionDDL(cat *redshiftcatalog.Catalog, sql string) bool {
- _, err := cat.Exec(sql, nil)
- return err == nil
-}
-
-func createTableDDL(schemaName, tableName string, columns []*metadatapb.ColumnMetadata) string {
- return fmt.Sprintf(
- "CREATE TABLE %s.%s (%s);",
- quoteIdent(schemaName),
- quoteIdent(tableName),
- columnListDDL(columns),
- )
-}
-
-func createViewDDL(kind, schemaName, viewName string, columns []*metadatapb.ColumnMetadata) string {
- selectItems := make([]string, 0, len(columns))
- for _, column := range columns {
- if column == nil || column.GetName() == "" {
- continue
- }
- selectItems = append(selectItems, fmt.Sprintf(
- "CAST(NULL AS %s) AS %s",
- normalizeCompletionType(column.GetType()),
- quoteIdent(column.GetName()),
- ))
- }
- if len(selectItems) == 0 {
- selectItems = append(selectItems, "1 AS __bytebase_completion_placeholder")
- }
- return fmt.Sprintf(
- "CREATE %s %s.%s AS SELECT %s;",
- kind,
- quoteIdent(schemaName),
- quoteIdent(viewName),
- strings.Join(selectItems, ", "),
- )
-}
-
-func createMaterializedViewDDL(schemaName, viewName, definition string) string {
- definition = strings.TrimSpace(definition)
- definition = strings.TrimSuffix(definition, ";")
- if strings.HasPrefix(strings.ToUpper(definition), "CREATE ") {
- return definition + ";"
- }
- return fmt.Sprintf(
- "CREATE MATERIALIZED VIEW %s.%s AS %s;",
- quoteIdent(schemaName),
- quoteIdent(viewName),
- definition,
- )
-}
-
-func columnListDDL(columns []*metadatapb.ColumnMetadata) string {
- columnDefs := make([]string, 0, len(columns))
- for _, column := range columns {
- if column == nil || column.GetName() == "" {
- continue
- }
- columnDefs = append(columnDefs, fmt.Sprintf("%s %s", quoteIdent(column.GetName()), normalizeCompletionType(column.GetType())))
- }
- if len(columnDefs) == 0 {
- columnDefs = append(columnDefs, "__bytebase_completion_placeholder text")
- }
- return strings.Join(columnDefs, ", ")
-}
-
-func normalizeCompletionType(typ string) string {
- lower := strings.ToLower(strings.TrimSpace(typ))
- switch {
- case lower == "":
- return "text"
- case strings.Contains(lower, "bigint"):
- return "bigint"
- case strings.Contains(lower, "int"):
- return "integer"
- case strings.Contains(lower, "bool"):
- return "boolean"
- case strings.Contains(lower, "char"), strings.Contains(lower, "text"), strings.Contains(lower, "string"):
- return "text"
- case strings.Contains(lower, "numeric"), strings.Contains(lower, "decimal"):
- return "numeric"
- case strings.Contains(lower, "double"):
- return "double precision"
- case strings.Contains(lower, "float"), strings.Contains(lower, "real"):
- return "real"
- case strings.Contains(lower, "date"):
- return "date"
- case strings.Contains(lower, "time"):
- return "timestamp"
- default:
- return "text"
- }
-}
-
func quoteIdent(name string) string {
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
}
diff --git a/backend/plugin/parser/redshift/query_span_extractor.go b/backend/plugin/parser/redshift/query_span_extractor.go
index 5b157e6a21..d4b5c4bdcb 100644
--- a/backend/plugin/parser/redshift/query_span_extractor.go
+++ b/backend/plugin/parser/redshift/query_span_extractor.go
@@ -2,7 +2,6 @@ package redshift
import (
"context"
- "slices"
"strings"
metadatapb "github.com/bytebase/omni/metadata"
@@ -12,7 +11,6 @@ import (
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
- "github.com/bytebase/bytebase/backend/store/model"
)
const redshiftGetDatabaseMetadataError = "failed to get database metadata for database: %s"
@@ -87,9 +85,6 @@ func (q *omniQuerySpanExtractor) getOmniQuerySpanWithFunctionStack(ctx context.C
if err != nil {
return nil, err
}
- if err := q.installOmniQuerySpanFunctions(ctx, cat); err != nil {
- return nil, err
- }
omniSpan, err := redshiftanalysis.GetQuerySpan(cat, statement)
if err != nil {
if isOmniQuerySpanNotFound(err) {
@@ -211,37 +206,10 @@ func (q *omniQuerySpanExtractor) newOmniQuerySpanCatalog(ctx context.Context) (*
}
cat := redshiftcatalog.New()
cat.SetSearchPath(q.searchPath)
- cat.SetRelationResolver(&redshiftQuerySpanRelationResolver{
- metadata: metadata,
- })
+ cat.UseMetadata(metadata.GetProto())
return cat, nil
}
-func (q *omniQuerySpanExtractor) installOmniQuerySpanFunctions(ctx context.Context, cat *redshiftcatalog.Catalog) error {
- if q.gCtx.GetDatabaseMetadataFunc == nil {
- return nil
- }
- _, metadata, err := q.gCtx.GetDatabaseMetadataFunc(ctx, q.gCtx.InstanceID, q.defaultDatabase)
- if err != nil {
- return errors.Wrapf(err, redshiftGetDatabaseMetadataError, q.defaultDatabase)
- }
- if metadata == nil {
- return nil
- }
- for _, schema := range metadata.GetProto().GetSchemas() {
- for _, function := range schema.GetFunctions() {
- definition := strings.TrimSpace(function.GetDefinition())
- if definition == "" {
- continue
- }
- if _, err := cat.Exec(definition, nil); err != nil {
- continue
- }
- }
- }
- return nil
-}
-
type omniFunctionBodySources struct {
accessTables base.SourceColumnSet
results []base.QuerySpanResult
@@ -269,256 +237,6 @@ type omniFunctionSourceProjection struct {
type omniFunctionSourceScope map[string]omniFunctionSourceProjection
-type redshiftQuerySpanRelationResolver struct {
- metadata *model.DatabaseMetadata
-}
-
-func (r *redshiftQuerySpanRelationResolver) ResolveRelation(schemaName, relationName string, searchPath []string) (*redshiftcatalog.RelationSpec, error) {
- if r.metadata == nil {
- return nil, nil
- }
-
- resolvedSchemaName, resolvedRelationName := schemaName, relationName
- if resolvedSchemaName == "" {
- resolvedSchemaName, resolvedRelationName = r.metadata.SearchRelation(searchPath, relationName)
- if resolvedSchemaName == "" && resolvedRelationName == "" {
- return nil, nil
- }
- }
-
- schema := r.metadata.GetSchemaMetadata(resolvedSchemaName)
- if schema == nil {
- return nil, nil
- }
- return redshiftRelationSpecFromMetadata(r.metadata, schema, resolvedSchemaName, resolvedRelationName), nil
-}
-
-func redshiftRelationSpecFromMetadata(metadata *model.DatabaseMetadata, schema *model.SchemaMetadata, schemaName, relationName string) *redshiftcatalog.RelationSpec {
- if table := schema.GetTable(relationName); table != nil {
- return redshiftTableRelationSpec(schemaName, table.GetProto().GetName(), table.GetProto().GetColumns())
- }
- if table := schema.GetExternalTable(relationName); table != nil {
- return redshiftTableRelationSpec(schemaName, table.GetProto().GetName(), table.GetProto().GetColumns())
- }
- if view := schema.GetView(relationName); view != nil {
- if definition := strings.TrimSpace(view.GetDefinition()); definition != "" {
- return &redshiftcatalog.RelationSpec{
- SchemaName: schemaName,
- Name: view.GetName(),
- Kind: redshiftcatalog.RelationKindView,
- Definition: qualifyRedshiftViewDefinition(definition, schemaName, metadata),
- }
- }
- return redshiftTableRelationSpec(schemaName, view.GetName(), view.GetColumns())
- }
- if view := schema.GetMaterializedView(relationName); view != nil {
- if definition := strings.TrimSpace(view.GetDefinition()); definition != "" {
- return &redshiftcatalog.RelationSpec{
- SchemaName: schemaName,
- Name: view.GetName(),
- Kind: redshiftcatalog.RelationKindMaterializedView,
- Definition: qualifyRedshiftViewDefinition(definition, schemaName, metadata),
- }
- }
- return redshiftTableRelationSpec(schemaName, view.GetName(), nil)
- }
- if sequence := schema.GetSequence(relationName); sequence != nil {
- return redshiftSequenceRelationSpec(schemaName, sequence.GetName())
- }
- return nil
-}
-
-func redshiftTableRelationSpec(schemaName, tableName string, columns []*metadatapb.ColumnMetadata) *redshiftcatalog.RelationSpec {
- return &redshiftcatalog.RelationSpec{
- SchemaName: schemaName,
- Name: tableName,
- Kind: redshiftcatalog.RelationKindTable,
- Columns: redshiftRelationColumnSpecs(columns),
- }
-}
-
-func redshiftRelationColumnSpecs(columns []*metadatapb.ColumnMetadata) []redshiftcatalog.RelationColumnSpec {
- result := make([]redshiftcatalog.RelationColumnSpec, 0, len(columns))
- for _, column := range columns {
- if column == nil || column.GetName() == "" {
- continue
- }
- result = append(result, redshiftcatalog.RelationColumnSpec{
- Name: column.GetName(),
- Type: normalizeCompletionType(column.GetType()),
- })
- }
- if len(result) == 0 {
- result = append(result, redshiftcatalog.RelationColumnSpec{
- Name: "__bytebase_query_span_placeholder",
- Type: "text",
- })
- }
- return result
-}
-
-func redshiftSequenceRelationSpec(schemaName, sequenceName string) *redshiftcatalog.RelationSpec {
- return &redshiftcatalog.RelationSpec{
- SchemaName: schemaName,
- Name: sequenceName,
- Kind: redshiftcatalog.RelationKindTable,
- Columns: []redshiftcatalog.RelationColumnSpec{
- {Name: "last_value", Type: "bigint"},
- {Name: "log_cnt", Type: "bigint"},
- {Name: "is_called", Type: "boolean"},
- },
- }
-}
-
-type redshiftSchemaQualificationEdit struct {
- offset int
- schema string
-}
-
-func qualifyRedshiftViewDefinition(definition, schemaName string, metadata *model.DatabaseMetadata) string {
- stmts, err := ParseRedshift(definition)
- if err != nil || len(stmts) != 1 {
- return definition
- }
-
- var query redshiftast.Node
- switch stmt := stmts[0].AST.(type) {
- case *redshiftast.ViewStmt:
- query = stmt.Query
- case *redshiftast.CreateTableAsStmt:
- query = stmt.Query
- case *redshiftast.SelectStmt:
- query = stmt
- default:
- return definition
- }
- if query == nil {
- return definition
- }
-
- searchPath := normalizeOmniSearchPath([]string{schemaName, "public"})
- var edits []redshiftSchemaQualificationEdit
- collectRedshiftSchemaQualificationEdits(metadata, searchPath, query, map[string]bool{}, &edits)
- if len(edits) == 0 {
- return definition
- }
-
- slices.SortFunc(edits, func(a, b redshiftSchemaQualificationEdit) int {
- return b.offset - a.offset
- })
- result := definition
- for _, edit := range edits {
- if edit.offset < 0 || edit.offset > len(result) {
- continue
- }
- result = result[:edit.offset] + quoteIdent(edit.schema) + "." + result[edit.offset:]
- }
- return result
-}
-
-func collectRedshiftSchemaQualificationEdits(metadata *model.DatabaseMetadata, searchPath []string, node redshiftast.Node, ctes map[string]bool, edits *[]redshiftSchemaQualificationEdit) {
- if node == nil {
- return
- }
- if selectStmt, ok := node.(*redshiftast.SelectStmt); ok {
- collectRedshiftSchemaQualificationEditsFromSelect(metadata, searchPath, selectStmt, ctes, edits)
- return
- }
-
- redshiftast.Inspect(node, func(n redshiftast.Node) bool {
- if n != node {
- if selectStmt, ok := n.(*redshiftast.SelectStmt); ok {
- collectRedshiftSchemaQualificationEditsFromSelect(metadata, searchPath, selectStmt, ctes, edits)
- return false
- }
- }
- rangeVar, ok := n.(*redshiftast.RangeVar)
- if !ok || rangeVar == nil || rangeVar.Relname == "" || rangeVar.Schemaname != "" || rangeVar.Catalogname != "" {
- return true
- }
- if isOmniCTEReference(rangeVar, ctes) || rangeVar.Loc.Start < 0 {
- return true
- }
- schemaName, _ := metadata.SearchRelation(searchPath, rangeVar.Relname)
- if schemaName == "" {
- return true
- }
- *edits = append(*edits, redshiftSchemaQualificationEdit{
- offset: rangeVar.Loc.Start,
- schema: schemaName,
- })
- return true
- })
-}
-
-func collectRedshiftSchemaQualificationEditsFromSelect(metadata *model.DatabaseMetadata, searchPath []string, selectStmt *redshiftast.SelectStmt, ctes map[string]bool, edits *[]redshiftSchemaQualificationEdit) {
- localCTEs := cloneOmniCTENameSet(ctes)
- if selectStmt.WithClause != nil && selectStmt.WithClause.Ctes != nil {
- cteScope := cloneOmniCTENameSet(ctes)
- if selectStmt.WithClause.Recursive {
- for _, item := range selectStmt.WithClause.Ctes.Items {
- cte, ok := item.(*redshiftast.CommonTableExpr)
- if ok && cte.Ctename != "" {
- cteScope[strings.ToLower(cte.Ctename)] = true
- }
- }
- }
- for _, item := range selectStmt.WithClause.Ctes.Items {
- cte, ok := item.(*redshiftast.CommonTableExpr)
- if !ok || cte.Ctename == "" {
- continue
- }
- collectRedshiftSchemaQualificationEdits(metadata, searchPath, cte.Ctequery, cteScope, edits)
- name := strings.ToLower(cte.Ctename)
- cteScope[name] = true
- localCTEs[name] = true
- }
- }
-
- for _, node := range []redshiftast.Node{
- selectStmt.WhereClause,
- selectStmt.HavingClause,
- selectStmt.QualifyClause,
- selectStmt.LimitOffset,
- selectStmt.LimitCount,
- } {
- if node != nil {
- collectRedshiftSchemaQualificationEdits(metadata, searchPath, node, localCTEs, edits)
- }
- }
- if selectStmt.IntoClause != nil {
- collectRedshiftSchemaQualificationEdits(metadata, searchPath, selectStmt.IntoClause, localCTEs, edits)
- }
- if selectStmt.Larg != nil {
- collectRedshiftSchemaQualificationEdits(metadata, searchPath, selectStmt.Larg, localCTEs, edits)
- }
- if selectStmt.Rarg != nil {
- collectRedshiftSchemaQualificationEdits(metadata, searchPath, selectStmt.Rarg, localCTEs, edits)
- }
- for _, list := range []*redshiftast.List{
- selectStmt.DistinctClause,
- selectStmt.TargetList,
- selectStmt.FromClause,
- selectStmt.GroupClause,
- selectStmt.WindowClause,
- selectStmt.ExcludeList,
- selectStmt.ValuesLists,
- selectStmt.SortClause,
- selectStmt.LockingClause,
- } {
- collectRedshiftSchemaQualificationEditsFromList(metadata, searchPath, list, localCTEs, edits)
- }
-}
-
-func collectRedshiftSchemaQualificationEditsFromList(metadata *model.DatabaseMetadata, searchPath []string, list *redshiftast.List, ctes map[string]bool, edits *[]redshiftSchemaQualificationEdit) {
- if list == nil {
- return
- }
- for _, item := range list.Items {
- collectRedshiftSchemaQualificationEdits(metadata, searchPath, item, ctes, edits)
- }
-}
-
func (q *omniQuerySpanExtractor) collectOmniAccessTables(ctx context.Context, node redshiftast.Node) (base.SourceColumnSet, error) {
result := make(base.SourceColumnSet)
if err := q.collectOmniAccessTablesFromNode(ctx, node, map[string]bool{}, result); err != nil {
diff --git a/backend/plugin/parser/redshift/query_span_test.go b/backend/plugin/parser/redshift/query_span_test.go
index 37e3e56fe5..c24f0cd265 100644
--- a/backend/plugin/parser/redshift/query_span_test.go
+++ b/backend/plugin/parser/redshift/query_span_test.go
@@ -108,15 +108,12 @@ func TestRedshiftQuerySpanEntrypointDoesNotDependOnANTLR(t *testing.T) {
}
}
-func TestRedshiftOmniQuerySpanUsesLazyRelationResolver(t *testing.T) {
+// Query span must resolve relations lazily through UseMetadata rather than
+// installing the whole snapshot up front.
+func TestRedshiftOmniQuerySpanUsesUseMetadata(t *testing.T) {
content, err := os.ReadFile("query_span_extractor.go")
require.NoError(t, err)
- source := string(content)
- require.Contains(t, source, "SetRelationResolver")
- require.NotContains(t, source, "pendingOmniCatalogView")
- require.NotContains(t, source, "buildOmniQuerySpanCatalog")
- require.NotContains(t, source, "orderPendingOmniCatalogViews")
- require.NotContains(t, source, "createQuerySpanViewDDL")
+ require.Contains(t, string(content), "UseMetadata")
}
func TestRedshiftOmniQuerySpanNonSelectTypes(t *testing.T) {
diff --git a/backend/plugin/parser/tidb/completion.go b/backend/plugin/parser/tidb/completion.go
index 533368873f..38b6c070c0 100644
--- a/backend/plugin/parser/tidb/completion.go
+++ b/backend/plugin/parser/tidb/completion.go
@@ -2,13 +2,11 @@ package tidb
import (
"context"
- "fmt"
"slices"
"strings"
"unicode"
"unicode/utf8"
- metadatapb "github.com/bytebase/omni/metadata"
"github.com/bytebase/omni/tidb/catalog"
omnicompletion "github.com/bytebase/omni/tidb/completion"
tidbparser "github.com/bytebase/omni/tidb/parser"
@@ -17,11 +15,6 @@ import (
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
-// genericColumnType is a syntactically valid fallback type used when a column's
-// real type is empty or fails to parse, so the column name still surfaces as a
-// completion candidate.
-const genericColumnType = "int"
-
// writeStatementKeywords are statement-initiating keywords for write, DDL,
// transaction-control, and admin statements. They are dropped from completion
// in the read-only query scene, leaving only read statements (SELECT, WITH,
@@ -112,13 +105,11 @@ func Completion(ctx context.Context, cCtx base.CompletionContext, statement stri
return result, nil
}
-// buildCatalog constructs an omni TiDB catalog from Bytebase metadata by
-// replaying minimal DDL. It fully loads the default database plus any database
-// referenced as a qualifier in the statement (so cross-database qualified
-// completion like `other_db.tbl.col` resolves), and registers every other known
-// database name so database-name candidates still surface. Every identifier is
-// backticked so reserved words and special characters parse; tables are created
-// one at a time so a single unparseable column type cannot empty the catalog.
+// buildCatalog constructs an omni TiDB catalog from Bytebase metadata. It
+// fully loads the default database plus any database referenced as a qualifier
+// in the statement (so cross-database qualified completion like
+// `other_db.tbl.col` resolves), and registers every other known database name
+// so database-name candidates still surface.
func buildCatalog(ctx context.Context, cCtx base.CompletionContext, statement string) *catalog.Catalog {
cat := catalog.New()
allNames := listAllDatabaseNames(ctx, cCtx)
@@ -166,33 +157,15 @@ func buildCatalog(ctx context.Context, cCtx base.CompletionContext, statement st
return cat
}
-// loadDatabaseObjects fully loads one database's tables and views into the
-// catalog, under that database's namespace.
+// loadDatabaseObjects loads one database's tables and views into the catalog.
func loadDatabaseObjects(ctx context.Context, cCtx base.CompletionContext, cat *catalog.Catalog, dbName string) {
- db := backtickIdentifier(dbName)
_, dbMeta, err := cCtx.Metadata(ctx, cCtx.InstanceID, dbName)
if err != nil || dbMeta == nil {
// Still register the name so it can be a database candidate.
- _, _ = cat.Exec("CREATE DATABASE "+db+";", &catalog.ExecOptions{ContinueOnError: true})
- return
- }
- if _, err := cat.Exec("CREATE DATABASE "+db+"; USE "+db+";", &catalog.ExecOptions{ContinueOnError: true}); err != nil {
- return
- }
- schema := dbMeta.GetSchemaMetadata("")
- if schema == nil {
+ _, _ = cat.Exec("CREATE DATABASE "+backtickIdentifier(dbName)+";", &catalog.ExecOptions{ContinueOnError: true})
return
}
- for _, tableName := range schema.ListTableNames() {
- if table := schema.GetTable(tableName); table != nil {
- defineTable(cat, tableName, table.GetProto().GetColumns())
- }
- }
- for _, viewName := range schema.ListViewNames() {
- if view := schema.GetView(viewName); view != nil {
- defineView(cat, viewName, view.GetDefinition())
- }
- }
+ _, _ = cat.LoadMetadata(ctx, dbMeta.GetProto())
}
// listAllDatabaseNames returns the instance's database names, or nil if the
@@ -252,76 +225,6 @@ func isIdentByte(b byte) bool {
(b >= '0' && b <= '9')
}
-// defineTable installs a table in the catalog, retrying with generic column
-// types if the real types fail to parse so that column names still surface.
-func defineTable(cat *catalog.Catalog, name string, columns []*metadatapb.ColumnMetadata) {
- if len(columns) == 0 {
- return
- }
- if execTableDDL(cat, name, columns, false) {
- return
- }
- execTableDDL(cat, name, columns, true)
-}
-
-func execTableDDL(cat *catalog.Catalog, name string, columns []*metadatapb.ColumnMetadata, generic bool) bool {
- defs := make([]string, 0, len(columns))
- for _, col := range columns {
- colType := col.GetType()
- if generic || colType == "" {
- colType = genericColumnType
- }
- defs = append(defs, backtickIdentifier(col.GetName())+" "+colType)
- }
- ddl := fmt.Sprintf("CREATE TABLE %s (%s);", backtickIdentifier(name), strings.Join(defs, ", "))
- return execOK(cat, ddl)
-}
-
-// defineView installs a view in the catalog. Unlike tables, the only structured
-// input is the view's definition SQL, whose exact shape varies (full
-// "CREATE VIEW ..." vs. a bare SELECT). We try the definition as-is, then
-// wrapped in CREATE VIEW, and finally fall back to a trivial view so the view
-// name still surfaces as a candidate (matching mysql, which reads view names
-// straight from metadata) even when the definition cannot be parsed.
-func defineView(cat *catalog.Catalog, name, definition string) {
- var attempts []string
- if definition != "" {
- attempts = append(attempts,
- definition,
- fmt.Sprintf("CREATE VIEW %s AS %s", backtickIdentifier(name), definition),
- )
- }
- attempts = append(attempts, fmt.Sprintf("CREATE VIEW %s AS SELECT 1", backtickIdentifier(name)))
- for _, ddl := range attempts {
- if execOK(cat, ddl) {
- return
- }
- }
-}
-
-// execOK runs DDL against the catalog and reports whether it actually installed
-// a schema object: every statement parsed without error AND at least one was a
-// non-DML (utility) statement that mutated the catalog. A definition that
-// reduces to only skipped DML — e.g. a bare SELECT view body, which TiDB sync
-// stores in information_schema.VIEWS.VIEW_DEFINITION — is NOT a success, so the
-// caller falls through to the wrapped CREATE VIEW form.
-func execOK(cat *catalog.Catalog, ddl string) bool {
- results, err := cat.Exec(ddl, &catalog.ExecOptions{ContinueOnError: true})
- if err != nil {
- return false
- }
- applied := false
- for _, r := range results {
- if r.Error != nil {
- return false
- }
- if !r.Skipped {
- applied = true
- }
- }
- return applied
-}
-
// backtickIdentifier quotes an identifier for TiDB DDL, escaping embedded
// backticks by doubling them.
func backtickIdentifier(name string) string {
diff --git a/backend/plugin/parser/tidb/completion_test.go b/backend/plugin/parser/tidb/completion_test.go
index 5f4335e041..f075ea67c7 100644
--- a/backend/plugin/parser/tidb/completion_test.go
+++ b/backend/plugin/parser/tidb/completion_test.go
@@ -128,10 +128,10 @@ func TestCompletion_ReservedWordIdentifiersSurface(t *testing.T) {
}
// A column whose stored type cannot be parsed must not silently vanish:
-// buildCatalog retries the whole table with a generic column type, so every
+// LoadMetadata stands the table in with text columns, so every
// column name still surfaces. Without that retry the failing CREATE TABLE would
// drop the entire table from the catalog.
-func TestCompletion_UnparseableColumnTypeFallsBackToGeneric(t *testing.T) {
+func TestCompletion_UnparseableColumnTypeFallsBackToStandIn(t *testing.T) {
meta := metadataFunc(&metadatapb.DatabaseSchemaMetadata{
Name: "db",
Schemas: []*metadatapb.SchemaMetadata{
@@ -140,7 +140,7 @@ func TestCompletion_UnparseableColumnTypeFallsBackToGeneric(t *testing.T) {
Tables: []*metadatapb.TableMetadata{
{Name: "t", Columns: []*metadatapb.ColumnMetadata{
{Name: "good", Type: "int"},
- {Name: "weird", Type: ")"}, // unparseable type — forces the generic retry
+ {Name: "weird", Type: ")"}, // unparseable type — forces the stand-in
}},
},
},
@@ -154,7 +154,7 @@ func TestCompletion_UnparseableColumnTypeFallsBackToGeneric(t *testing.T) {
require.True(t, hasCandidate(got, base.CandidateTypeColumn, "good"),
"column 'good' should surface; got %v", got)
require.True(t, hasCandidate(got, base.CandidateTypeColumn, "weird"),
- "column 'weird' with an unparseable type should still surface via the generic-type retry; got %v", got)
+ "column 'weird' with an unparseable type should still surface via the stand-in; got %v", got)
}
// In the read-only query scene, write statements (DML/DDL incl. TiDB BATCH)
diff --git a/backend/plugin/parser/trino/completion.go b/backend/plugin/parser/trino/completion.go
index 116afedd49..c843435bbc 100644
--- a/backend/plugin/parser/trino/completion.go
+++ b/backend/plugin/parser/trino/completion.go
@@ -11,7 +11,6 @@ import (
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
- "github.com/bytebase/bytebase/backend/store/model"
)
func init() {
@@ -90,7 +89,7 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext, st
if err != nil || meta == nil {
continue
}
- loadCatalogMetadata(cat, norm, meta)
+ cat.LoadMetadata(meta.GetProto())
}
if cCtx.DefaultDatabase != "" {
@@ -102,36 +101,14 @@ func buildCompletionCatalog(ctx context.Context, cCtx base.CompletionContext, st
return cat
}
-// loadCatalogMetadata copies one database's schemas, tables and views (with
-// their column lists and view definitions) into the omni catalog under the
-// given normalized catalog name, returning the view definitions it loaded.
-// Views carry their defining query so omni's analysis can resolve lineage
-// through them (GetQuerySpanWithCatalog); an empty definition leaves the view
-// opaque. Shared by the completion and query-span catalog builders.
-func loadCatalogMetadata(cat *catalog.Catalog, norm string, meta *model.DatabaseMetadata) []string {
+// viewDefinitions returns the defining query of every view in the snapshot.
+// Query span feeds them back as a corpus so it can tell which catalogs a view
+// reaches into; an empty definition leaves the view opaque.
+func viewDefinitions(meta *metadatapb.DatabaseSchemaMetadata) []string {
var definitions []string
- database := cat.EnsureCatalog(norm)
- for _, schemaName := range meta.ListSchemaNames() {
- schemaMeta := meta.GetSchemaMetadata(schemaName)
- if schemaMeta == nil {
- continue
- }
- sc := database.EnsureSchema(catalog.Normalize(schemaName))
- for _, tableName := range schemaMeta.ListTableNames() {
- tableMeta := schemaMeta.GetTable(tableName)
- if tableMeta == nil {
- continue
- }
- sc.AddTable(catalog.Normalize(tableName), columnsOf(tableMeta.GetProto().GetColumns())...)
- }
- for _, viewName := range schemaMeta.ListViewNames() {
- viewMeta := schemaMeta.GetView(viewName)
- if viewMeta == nil {
- continue
- }
- v := sc.AddView(catalog.Normalize(viewName), columnsOf(viewMeta.GetColumns())...)
- v.Definition = viewMeta.GetDefinition()
- if def := viewMeta.GetDefinition(); def != "" {
+ for _, schema := range meta.GetSchemas() {
+ for _, view := range schema.GetViews() {
+ if def := view.GetDefinition(); def != "" {
definitions = append(definitions, def)
}
}
@@ -164,19 +141,6 @@ func catalogNeeded(normName, defaultDB, lowerStmt string) bool {
return strings.Contains(lowerStmt, strings.ToLower(normName))
}
-// columnsOf converts storepb columns into omni catalog columns (names
-// normalized).
-func columnsOf(columns []*metadatapb.ColumnMetadata) []*catalog.Column {
- out := make([]*catalog.Column, 0, len(columns))
- for _, c := range columns {
- if c == nil {
- continue
- }
- out = append(out, catalog.NewColumn(catalog.Normalize(c.Name), c.Type, c.Nullable))
- }
- return out
-}
-
// convertCandidateType maps an omni completion candidate type onto the bytebase
// base.CandidateType. A Trino catalog corresponds to a bytebase DATABASE.
func convertCandidateType(t completion.CandidateType) base.CandidateType {
diff --git a/backend/plugin/parser/trino/query_span_extractor.go b/backend/plugin/parser/trino/query_span_extractor.go
index 3c57fcc3c8..5a9fa69d32 100644
--- a/backend/plugin/parser/trino/query_span_extractor.go
+++ b/backend/plugin/parser/trino/query_span_extractor.go
@@ -220,7 +220,8 @@ func (q *querySpanExtractor) buildSpanCatalog(ctx context.Context, statement str
if err != nil || meta == nil {
continue
}
- for _, def := range loadCatalogMetadata(cat, norm, meta) {
+ cat.LoadMetadata(meta.GetProto())
+ for _, def := range viewDefinitions(meta.GetProto()) {
corpus += "\n" + strings.ToLower(def)
}
}
diff --git a/backend/plugin/schema/mysql/walk_through.go b/backend/plugin/schema/mysql/walk_through.go
index bf459e6599..fabde580fe 100644
--- a/backend/plugin/schema/mysql/walk_through.go
+++ b/backend/plugin/schema/mysql/walk_through.go
@@ -27,27 +27,26 @@ func init() {
// WalkThroughWithContext performs DDL simulation using the omni MySQL catalog.
// Flow:
-// 1. Create the catalog and select the target database.
-// 2. loadWalkThroughCatalog: install each object individually with per-object
-// pseudo fallback, so one broken CREATE TABLE can't disable the whole
-// simulation.
+// 1. Create the catalog.
+// 2. catalog.LoadMetadata: install the snapshot, standing in for what fails.
// 3. catalog.Exec(userSQL) → execute user DDL
// 4. Map errors → *storepb.Advice
// 5. Convert updated catalog → DatabaseMetadata (for downstream rules)
-func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMetadata, asts []base.AST) *storepb.Advice {
- if ctx.RawSQL == "" {
+func WalkThroughWithContext(ctx context.Context, wtCtx schema.WalkThroughContext, d *model.DatabaseMetadata, asts []base.AST) *storepb.Advice {
+ if wtCtx.RawSQL == "" {
return nil
}
dbName := d.GetProto().GetName()
precheck := firstMySQLWalkThroughAdvice(
precheckMySQLOmniWalkThrough(d, asts),
- precheckMySQLRawWalkThrough(d, ctx.RawSQL, asts),
+ precheckMySQLRawWalkThrough(d, wtCtx.RawSQL, asts),
)
- // Step 1: Create the catalog and the target database.
+ // Step 1: Create the catalog. LoadMetadata creates and selects the target
+ // database itself; only a RENAME destination needs one made up front.
c := catalog.New()
- initSQL := fmt.Sprintf("SET foreign_key_checks = 0;\nCREATE DATABASE IF NOT EXISTS %s;\nUSE %s;", mysqlQuoteIdentifier(dbName), mysqlQuoteIdentifier(dbName))
+ initSQL := "SET foreign_key_checks = 0;"
for _, targetDB := range mysqlRenameTargetDatabases(d, asts) {
initSQL += fmt.Sprintf("\nCREATE DATABASE IF NOT EXISTS %s;", mysqlQuoteIdentifier(targetDB))
}
@@ -61,10 +60,8 @@ func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMeta
}
}
- // Step 2: Install every schema object individually with pseudo fallback.
- // TODO: thread a real context.Context through WalkThroughContext; for now the
- // loader only uses it for early cancellation during catalog bulk-load.
- if err := loadWalkThroughCatalog(context.Background(), c, dbName, d.GetProto()); err != nil {
+ // Step 2: Install every schema object, with stand-ins for the ones that fail.
+ if _, err := c.LoadMetadata(ctx, d.GetProto()); err != nil {
return &storepb.Advice{
Status: storepb.Advice_ERROR,
Code: code.DDLSimulationFailed.Int32(),
@@ -75,7 +72,7 @@ func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMeta
}
// Step 3: Execute user SQL.
- results, execErr := c.Exec(ctx.RawSQL, &catalog.ExecOptions{ContinueOnError: true})
+ results, execErr := c.Exec(wtCtx.RawSQL, &catalog.ExecOptions{ContinueOnError: true})
if execErr != nil {
errCode := code.DDLSimulationFailed
content := execErr.Error()
@@ -780,7 +777,11 @@ func tableToProto(t *catalog.Table) *metadatapb.TableMetadata {
for _, idx := range t.Indexes {
indexType := strings.ToUpper(idx.IndexType)
if indexType == "" {
+ // A key without USING gets the engine's default access method.
indexType = "BTREE"
+ if strings.EqualFold(t.Engine, "MEMORY") || strings.EqualFold(t.Engine, "HEAP") {
+ indexType = "HASH"
+ }
}
idxMeta := &metadatapb.IndexMetadata{
Name: idx.Name,
diff --git a/backend/plugin/schema/mysql/walk_through_loader.go b/backend/plugin/schema/mysql/walk_through_loader.go
deleted file mode 100644
index cbbd4116e3..0000000000
--- a/backend/plugin/schema/mysql/walk_through_loader.go
+++ /dev/null
@@ -1,1083 +0,0 @@
-package mysql
-
-import (
- "cmp"
- "context"
- "slices"
- "strings"
-
- "github.com/pkg/errors"
-
- metadatapb "github.com/bytebase/omni/metadata"
- "github.com/bytebase/omni/mysql/ast"
- "github.com/bytebase/omni/mysql/catalog"
- mysqlparser "github.com/bytebase/omni/mysql/parser"
-)
-
-// autoIncrementSentinel matches the value that MySQL sync writes into
-// ColumnMetadata.Default when the column is AUTO_INCREMENT. See
-// get_database_definition.go:isAutoIncrement.
-const autoIncrementSentinel = "AUTO_INCREMENT"
-
-// loadWalkThroughCatalog installs every schema object from DatabaseSchemaMetadata
-// into the omni MySQL catalog with per-object isolation and pseudo fallback.
-//
-// Rationale: the prior implementation emitted a single concatenated DDL and
-// executed it as one Exec call. A single failing CREATE TABLE would leave the
-// catalog missing that table, and every downstream view / user query that
-// referenced it would also fail. This loader restricts each failure to its
-// owning object: if a table cannot be installed as-is, a pseudo table with
-// the right column names (but TEXT types, no constraints) is installed in its
-// place so that references still resolve.
-//
-// Preconditions: the caller must already have created and selected the target
-// database. This loader mutates foreign_key_checks to allow forward FK
-// references during bulk load.
-func loadWalkThroughCatalog(ctx context.Context, cat *catalog.Catalog, dbName string, meta *metadatapb.DatabaseSchemaMetadata) error {
- if cat == nil {
- return errors.New("loadWalkThroughCatalog: nil catalog")
- }
- if meta == nil {
- return nil
- }
-
- // Disable FK checks for the duration of the bulk load. MySQL tolerates
- // forward FK references only when this flag is off.
- prevFKChecks := cat.ForeignKeyChecks()
- cat.SetForeignKeyChecks(false)
- defer cat.SetForeignKeyChecks(prevFKChecks)
-
- cat.SetCurrentDatabase(dbName)
-
- objects := wtCollectObjects(dbName, meta)
- sorted := wtTopoSort(objects)
- for _, obj := range sorted {
- if err := ctx.Err(); err != nil {
- return err
- }
- wtInstallOne(cat, dbName, obj)
- }
- return nil
-}
-
-// wtObjectKind identifies the kind of a walk-through loader object.
-type wtObjectKind int
-
-const (
- kindWTTable wtObjectKind = iota
- kindWTView
- kindWTFunction
- kindWTProcedure
- kindWTTrigger
- kindWTEvent
-)
-
-// wtObjectEntry is one unit of work for the walk-through loader.
-type wtObjectEntry struct {
- kind wtObjectKind
- name string
- // parentTable is set for kindWTTrigger — triggers must attach to an
- // existing table in the catalog, so we carry the owning table name
- // separately from the trigger name.
- parentTable string
-
- // Exactly one of the following is set based on kind.
- tableMeta *metadatapb.TableMetadata
- viewMeta *metadatapb.ViewMetadata
- funcMeta *metadatapb.FunctionMetadata
- procMeta *metadatapb.ProcedureMetadata
- triggerMeta *metadatapb.TriggerMetadata
- eventMeta *metadatapb.EventMetadata
-}
-
-func (e *wtObjectEntry) key() string {
- switch e.kind {
- case kindWTTable:
- return "table:" + e.name
- case kindWTView:
- return "view:" + e.name
- case kindWTFunction:
- return "func:" + e.name
- case kindWTProcedure:
- return "proc:" + e.name
- case kindWTTrigger:
- return "trigger:" + e.parentTable + "." + e.name
- case kindWTEvent:
- return "event:" + e.name
- }
- return "unknown:" + e.name
-}
-
-func (e *wtObjectEntry) sortKey() string {
- base := wtKindLabel(e.kind) + "\x00" + e.name
- if e.kind == kindWTTrigger {
- // Keep triggers grouped by their parent table so stable ordering
- // within a file doesn't depend on trigger name alone.
- base += "\x00" + e.parentTable
- }
- return base
-}
-
-func wtKindLabel(k wtObjectKind) string {
- switch k {
- case kindWTTable:
- return "1table"
- case kindWTView:
- return "2view"
- case kindWTFunction:
- return "3function"
- case kindWTProcedure:
- return "4procedure"
- case kindWTTrigger:
- // Triggers must install after their target table. The ordering here
- // alone does the trick because kindWTTable sorts before kindWTTrigger.
- return "5trigger"
- case kindWTEvent:
- return "6event"
- }
- return "9unknown"
-}
-
-// wtCollectObjects flattens DatabaseSchemaMetadata into wtObjectEntry values.
-// MySQL metadata uses a single empty-named schema, so we look only at the first.
-func wtCollectObjects(_ string, meta *metadatapb.DatabaseSchemaMetadata) []*wtObjectEntry {
- var out []*wtObjectEntry
- for _, sm := range meta.Schemas {
- for _, tbl := range sm.Tables {
- if tbl == nil || tbl.Name == "" {
- continue
- }
- out = append(out, &wtObjectEntry{
- kind: kindWTTable,
- name: tbl.Name,
- tableMeta: tbl,
- })
- for _, trg := range tbl.Triggers {
- if trg == nil || trg.Name == "" {
- continue
- }
- out = append(out, &wtObjectEntry{
- kind: kindWTTrigger,
- name: trg.Name,
- parentTable: tbl.Name,
- triggerMeta: trg,
- })
- }
- }
- for _, view := range sm.Views {
- if view == nil || view.Name == "" {
- continue
- }
- out = append(out, &wtObjectEntry{
- kind: kindWTView,
- name: view.Name,
- viewMeta: view,
- })
- }
- for _, fn := range sm.Functions {
- if fn == nil || fn.Name == "" {
- continue
- }
- out = append(out, &wtObjectEntry{
- kind: kindWTFunction,
- name: fn.Name,
- funcMeta: fn,
- })
- }
- for _, proc := range sm.Procedures {
- if proc == nil || proc.Name == "" {
- continue
- }
- out = append(out, &wtObjectEntry{
- kind: kindWTProcedure,
- name: proc.Name,
- procMeta: proc,
- })
- }
- for _, ev := range sm.Events {
- if ev == nil || ev.Name == "" {
- continue
- }
- out = append(out, &wtObjectEntry{
- kind: kindWTEvent,
- name: ev.Name,
- eventMeta: ev,
- })
- }
- }
- return out
-}
-
-// wtTopoSort orders objects by dependency using Tarjan SCC.
-//
-// For MySQL the dependency graph is extremely thin: tables have no cross-edges
-// (FKs are tolerated by SetForeignKeyChecks(false)), views tolerate forward
-// refs via DefineView semantics, and functions/procedures reference objects
-// only in their body text (opaque to us). So the graph reduces to a fixed
-// per-kind ordering: tables → views → functions → procedures. We still run
-// Tarjan so the shape of this file mirrors the pg walk-through loader, and so
-// the ordering remains deterministic if dependency edges are added later.
-func wtTopoSort(objects []*wtObjectEntry) []*wtObjectEntry {
- if len(objects) == 0 {
- return nil
- }
- index := make(map[string]*wtObjectEntry, len(objects))
- for _, obj := range objects {
- index[obj.key()] = obj
- }
-
- edges := wtBuildEdges(objects, index)
- sccs := wtTarjanSCC(objects, edges)
-
- sccOf := make(map[string]int, len(objects))
- for i, scc := range sccs {
- for _, obj := range scc {
- sccOf[obj.key()] = i
- }
- }
-
- condensedEdges := make([][]int, len(sccs))
- inDegree := make([]int, len(sccs))
- seenEdge := make(map[[2]int]bool)
- for src, dsts := range edges {
- srcSCC, ok := sccOf[src]
- if !ok {
- continue
- }
- for _, dst := range dsts {
- dstSCC, ok2 := sccOf[dst]
- if !ok2 || dstSCC == srcSCC {
- continue
- }
- edge := [2]int{dstSCC, srcSCC}
- if seenEdge[edge] {
- continue
- }
- seenEdge[edge] = true
- condensedEdges[dstSCC] = append(condensedEdges[dstSCC], srcSCC)
- inDegree[srcSCC]++
- }
- }
-
- ready := make([]int, 0)
- for i := range sccs {
- if inDegree[i] == 0 {
- ready = append(ready, i)
- }
- }
- wtSortSCCsByMin(ready, sccs)
-
- var flat []*wtObjectEntry
- for len(ready) > 0 {
- next := ready[0]
- ready = ready[1:]
- flat = append(flat, wtSortedSCCMembers(sccs[next])...)
- for _, nb := range condensedEdges[next] {
- inDegree[nb]--
- if inDegree[nb] == 0 {
- ready = append(ready, nb)
- }
- }
- wtSortSCCsByMin(ready, sccs)
- }
-
- if len(flat) != len(objects) {
- emitted := make(map[string]bool, len(flat))
- for _, o := range flat {
- emitted[o.key()] = true
- }
- var missed []*wtObjectEntry
- for _, o := range objects {
- if !emitted[o.key()] {
- missed = append(missed, o)
- }
- }
- slices.SortStableFunc(missed, func(a, b *wtObjectEntry) int {
- return cmp.Compare(a.sortKey(), b.sortKey())
- })
- flat = append(flat, missed...)
- }
- return flat
-}
-
-// wtBuildEdges is intentionally thin today. See wtTopoSort for the rationale.
-func wtBuildEdges(_ []*wtObjectEntry, _ map[string]*wtObjectEntry) map[string][]string {
- return map[string][]string{}
-}
-
-func wtTarjanSCC(objects []*wtObjectEntry, edges map[string][]string) [][]*wtObjectEntry {
- type state struct {
- index, low int
- onStack bool
- }
- st := make(map[string]*state, len(objects))
- byKey := make(map[string]*wtObjectEntry, len(objects))
- for _, obj := range objects {
- byKey[obj.key()] = obj
- }
-
- keys := make([]string, 0, len(objects))
- for _, obj := range objects {
- keys = append(keys, obj.key())
- }
- slices.Sort(keys)
-
- var (
- idx int
- stack []string
- result [][]*wtObjectEntry
- )
-
- var strongconnect func(v string)
- strongconnect = func(v string) {
- st[v] = &state{index: idx, low: idx, onStack: true}
- idx++
- stack = append(stack, v)
- for _, w := range edges[v] {
- if _, ok := byKey[w]; !ok {
- continue
- }
- if _, seen := st[w]; !seen {
- strongconnect(w)
- st[v].low = min(st[v].low, st[w].low)
- } else if st[w].onStack {
- st[v].low = min(st[v].low, st[w].index)
- }
- }
- if st[v].low == st[v].index {
- var scc []*wtObjectEntry
- for {
- w := stack[len(stack)-1]
- stack = stack[:len(stack)-1]
- st[w].onStack = false
- scc = append(scc, byKey[w])
- if w == v {
- break
- }
- }
- result = append(result, scc)
- }
- }
-
- for _, k := range keys {
- if _, seen := st[k]; !seen {
- strongconnect(k)
- }
- }
- return result
-}
-
-func wtSortedSCCMembers(scc []*wtObjectEntry) []*wtObjectEntry {
- out := make([]*wtObjectEntry, len(scc))
- copy(out, scc)
- slices.SortStableFunc(out, func(a, b *wtObjectEntry) int {
- return cmp.Compare(a.sortKey(), b.sortKey())
- })
- return out
-}
-
-func wtSortSCCsByMin(indices []int, sccs [][]*wtObjectEntry) {
- slices.SortStableFunc(indices, func(a, b int) int {
- return cmp.Compare(wtMinSortKey(sccs[a]), wtMinSortKey(sccs[b]))
- })
-}
-
-func wtMinSortKey(scc []*wtObjectEntry) string {
- if len(scc) == 0 {
- return ""
- }
- best := scc[0].sortKey()
- for _, o := range scc[1:] {
- if k := o.sortKey(); k < best {
- best = k
- }
- }
- return best
-}
-
-// wtInstallOne attempts a real install and falls back to pseudo for objects
-// that have a pseudo form. For objects without one (functions, procedures)
-// failure silently drops the object from the catalog.
-func wtInstallOne(cat *catalog.Catalog, dbName string, obj *wtObjectEntry) {
- if err := wtInstallReal(cat, obj); err == nil {
- return
- }
- switch obj.kind {
- case kindWTTable:
- _ = wtInstallPseudoTable(cat, dbName, obj.tableMeta)
- case kindWTView:
- _ = wtInstallPseudoView(cat, dbName, obj.viewMeta)
- case kindWTTrigger:
- _ = wtInstallPseudoTrigger(cat, obj.triggerMeta, obj.parentTable)
- case kindWTEvent:
- _ = wtInstallPseudoEvent(cat, obj.eventMeta)
- default:
- // Functions, procedures: no pseudo form. Leave uninstalled.
- }
-}
-
-// wtInstallReal builds the object's AST directly from metadata and calls the
-// corresponding catalog Define* API. This bypasses the bytebase deparser and
-// the omni parser on the critical path — both of which can have bugs that
-// would otherwise cause an object to fail real install when the metadata
-// itself is fine.
-//
-// Parser use is deliberately confined to cases where the input string comes
-// from MySQL itself (COLUMN_TYPE, VIEW_DEFINITION, SHOW CREATE output,
-// information_schema expression columns). Parsing those is parsing reality,
-// not parsing our own deparse, and that's the distinction that matters.
-func wtInstallReal(cat *catalog.Catalog, obj *wtObjectEntry) error {
- switch obj.kind {
- case kindWTTable:
- stmt, err := wtBuildCreateTableStmt(obj.tableMeta)
- if err != nil {
- return err
- }
- return cat.DefineTable(stmt)
- case kindWTView:
- stmt, err := wtBuildCreateViewStmt(obj.viewMeta)
- if err != nil {
- return err
- }
- return cat.DefineView(stmt)
- case kindWTFunction:
- stmt, err := wtParseCreateRoutineStmt(obj.funcMeta.Definition, false)
- if err != nil {
- return err
- }
- return cat.DefineFunction(stmt)
- case kindWTProcedure:
- stmt, err := wtParseCreateRoutineStmt(obj.procMeta.Definition, true)
- if err != nil {
- return err
- }
- return cat.DefineProcedure(stmt)
- case kindWTTrigger:
- tm := obj.triggerMeta
- if tm == nil || tm.Timing == "" || tm.Event == "" {
- // Missing routing metadata — let pseudo fallback synthesize
- // defaults rather than storing a semantically broken trigger.
- return errors.New("trigger: missing Timing or Event")
- }
- return cat.DefineTrigger(wtBuildCreateTriggerStmt(tm, obj.parentTable))
- case kindWTEvent:
- stmt, err := wtParseCreateEventStmt(obj.eventMeta.Definition)
- if err != nil {
- return err
- }
- return cat.DefineEvent(stmt)
- }
- return errors.Errorf("wtInstallReal: unknown kind %d", obj.kind)
-}
-
-// wtBuildCreateTableStmt maps TableMetadata directly onto *ast.CreateTableStmt.
-// Expression-bearing fields (DEFAULT, ON UPDATE, GENERATED, CHECK) go through
-// wtParseExpr, which tolerates parse failures by silently dropping the
-// affected feature rather than failing the whole table.
-func wtBuildCreateTableStmt(tbl *metadatapb.TableMetadata) (*ast.CreateTableStmt, error) {
- if tbl == nil || tbl.Name == "" {
- return nil, errors.New("wtBuildCreateTableStmt: empty table")
- }
- stmt := &ast.CreateTableStmt{
- Table: &ast.TableRef{Name: tbl.Name},
- }
-
- for _, col := range tbl.Columns {
- if col == nil || col.Name == "" {
- continue
- }
- def, err := wtBuildColumnDef(col)
- if err != nil {
- return nil, errors.Wrapf(err, "column %q", col.Name)
- }
- stmt.Columns = append(stmt.Columns, def)
- }
-
- for _, idx := range tbl.Indexes {
- if idx == nil || len(idx.Expressions) == 0 {
- continue
- }
- if c := wtBuildIndexConstraint(idx); c != nil {
- stmt.Constraints = append(stmt.Constraints, c)
- }
- }
-
- for _, fk := range tbl.ForeignKeys {
- if fk == nil {
- continue
- }
- stmt.Constraints = append(stmt.Constraints, wtBuildFKConstraint(fk))
- }
-
- for _, chk := range tbl.CheckConstraints {
- if chk == nil {
- continue
- }
- if c := wtBuildCheckConstraint(chk); c != nil {
- stmt.Constraints = append(stmt.Constraints, c)
- }
- }
-
- if tbl.Engine != "" {
- stmt.Options = append(stmt.Options, &ast.TableOption{Name: "ENGINE", Value: tbl.Engine})
- }
- if tbl.Charset != "" {
- stmt.Options = append(stmt.Options, &ast.TableOption{Name: "CHARSET", Value: tbl.Charset})
- }
- if tbl.Collation != "" {
- stmt.Options = append(stmt.Options, &ast.TableOption{Name: "COLLATE", Value: tbl.Collation})
- }
- if tbl.Comment != "" {
- stmt.Options = append(stmt.Options, &ast.TableOption{Name: "COMMENT", Value: tbl.Comment})
- }
-
- // Partitions deliberately skipped: PartitionClause requires parsed
- // expression trees that we don't have metadata for in decomposed form.
- // Losing partition info does not affect column resolution or query shape.
-
- return stmt, nil
-}
-
-func wtBuildColumnDef(col *metadatapb.ColumnMetadata) (*ast.ColumnDef, error) {
- typeName, err := wtParseTypeName(col.Type)
- if err != nil {
- return nil, err
- }
- // Column-level charset/collation can come in via the type string or via
- // separate metadata fields. Prefer type-embedded if present, else take
- // the separate fields.
- if typeName.Charset == "" && col.CharacterSet != "" {
- typeName.Charset = col.CharacterSet
- }
- if typeName.Collate == "" && col.Collation != "" {
- typeName.Collate = col.Collation
- }
- // Spatial SRID (MySQL 8.0): carry the explicit SRID by presence (including the valid
- // SRID 0) so a pre-synced `POINT SRID n` column seeds the catalog with its SRID and
- // tableToProto reads it back unchanged (no phantom SRID diff).
- if col.Srid != nil {
- typeName.SRID = int(col.GetSrid())
- typeName.HasSRID = true
- }
-
- def := &ast.ColumnDef{
- Name: col.Name,
- TypeName: typeName,
- Comment: col.Comment,
- }
- if !col.Nullable {
- def.Constraints = append(def.Constraints, &ast.ColumnConstraint{Type: ast.ColConstrNotNull})
- }
- // INVISIBLE column (MySQL 8.0.23+): seed the catalog as invisible so a pre-synced
- // INVISIBLE column round-trips (tableToProto re-reads it) instead of surfacing as a
- // phantom visibility change.
- if col.IsInvisible {
- def.Constraints = append(def.Constraints, &ast.ColumnConstraint{Type: ast.ColConstrInvisible})
- }
-
- // MySQL sync encodes AUTO_INCREMENT as the literal string "AUTO_INCREMENT"
- // in ColumnMetadata.Default. Detect it here and set AutoIncrement instead
- // of treating it as a real default expression.
- isAutoInc := strings.EqualFold(col.Default, autoIncrementSentinel)
- if isAutoInc {
- def.AutoIncrement = true
- }
-
- if !isAutoInc && col.Default != "" && col.Generation == nil {
- // MySQL sync fills "NULL" as the default for all nullable columns,
- // including TEXT/BLOB/JSON/GEOMETRY — types that MySQL's grammar
- // does NOT accept a DEFAULT clause on. Emitting `DEFAULT NULL` for
- // these would cause DefineTable to reject the whole table and force
- // pseudo fallback. Mirror the filter in
- // get_database_definition.go:typeSupportsDefaultValue.
- if !strings.EqualFold(col.Default, "NULL") || wtTypeSupportsDefault(col.Type) {
- if expr, err := wtParseExpr(col.Default); err == nil {
- def.DefaultValue = expr
- }
- // Silent drop on parse failure — a missing DEFAULT is a closer
- // approximation of reality than a whole-column failure.
- }
- }
- if col.OnUpdate != "" {
- if expr, err := wtParseExpr(col.OnUpdate); err == nil {
- def.OnUpdate = expr
- }
- }
- if col.Generation != nil && col.Generation.Expression != "" {
- if expr, err := wtParseExpr(col.Generation.Expression); err == nil {
- def.Generated = &ast.GeneratedColumn{
- Expr: expr,
- Stored: col.Generation.Type == metadatapb.GenerationMetadata_TYPE_STORED,
- }
- }
- }
- return def, nil
-}
-
-func wtBuildIndexConstraint(idx *metadatapb.IndexMetadata) *ast.Constraint {
- // IndexMetadata.Type mixes two axes: the constraint kind (FULLTEXT /
- // SPATIAL) and the access method (BTREE / HASH). Pick them apart so
- // the resulting ast.Constraint sets Type and IndexType correctly.
- idxTypeUpper := strings.ToUpper(strings.TrimSpace(idx.Type))
-
- c := &ast.Constraint{
- Name: idx.Name,
- }
- switch {
- case idx.Primary:
- c.Type = ast.ConstrPrimaryKey
- c.IndexType = idxTypeUpper
- case idx.Unique:
- c.Type = ast.ConstrUnique
- c.IndexType = idxTypeUpper
- case idxTypeUpper == "FULLTEXT":
- c.Type = ast.ConstrFulltextIndex
- case idxTypeUpper == "SPATIAL":
- c.Type = ast.ConstrSpatialIndex
- default:
- c.Type = ast.ConstrIndex
- c.IndexType = idxTypeUpper
- }
- // Functional key parts (MySQL 8.0+) arrive as expressions in
- // idx.Expressions — either parenthesized `(lower(name))` or
- // function-call `lower(name)` form. Those MUST live in
- // IndexColumns[*].Expr, not in the bare-identifier Columns slice.
- // Per-part length or DESC also force IndexColumns.
- needIC := false
- for i, expr := range idx.Expressions {
- if wtIsExpressionIndexKey(expr) ||
- (i < len(idx.KeyLength) && idx.KeyLength[i] > 0) ||
- (i < len(idx.Descending) && idx.Descending[i]) {
- needIC = true
- break
- }
- }
- if needIC {
- for i, expr := range idx.Expressions {
- ic := &ast.IndexColumn{Expr: wtIndexKeyExpr(expr)}
- if i < len(idx.KeyLength) && idx.KeyLength[i] > 0 {
- ic.Length = int(idx.KeyLength[i])
- }
- if i < len(idx.Descending) && idx.Descending[i] {
- ic.Desc = true
- }
- c.IndexColumns = append(c.IndexColumns, ic)
- }
- } else {
- for _, expr := range idx.Expressions {
- c.Columns = append(c.Columns, wtUnquoteIdent(expr))
- }
- }
- return c
-}
-
-// wtIsExpressionIndexKey returns true if the key-part string looks like a
-// functional index expression rather than a bare identifier. Heuristic:
-// presence of a paren implies either a function call or a parenthesized
-// expression. Backticked identifiers don't normally contain parens.
-func wtIsExpressionIndexKey(s string) bool {
- return strings.Contains(s, "(")
-}
-
-// wtIndexKeyExpr converts one idx.Expressions entry into an ExprNode. For
-// functional keys we parse the expression text (source is MySQL's own
-// information_schema output, not our deparse); bare identifiers become a
-// plain ColumnRef.
-func wtIndexKeyExpr(s string) ast.ExprNode {
- if wtIsExpressionIndexKey(s) {
- if parsed, err := wtParseExpr(s); err == nil {
- return parsed
- }
- // Parse failed — best-effort: fall through to ColumnRef so at least
- // the constraint installs with some key part instead of an empty
- // IndexColumn.
- }
- return &ast.ColumnRef{Column: wtUnquoteIdent(s)}
-}
-
-// wtTypeSupportsDefault returns false for MySQL column types that disallow a
-// literal DEFAULT clause (BLOB family, JSON, GEOMETRY). Mirrors the filter
-// used by the bytebase writer — see get_database_definition.go:
-// typeSupportsDefaultValue — so the loader refuses defaults for the same
-// types the writer refuses to emit them for.
-func wtTypeSupportsDefault(typeStr string) bool {
- head := strings.ToLower(strings.TrimSpace(typeStr))
- if i := strings.IndexAny(head, " ("); i >= 0 {
- head = head[:i]
- }
- switch head {
- case "blob", "tinyblob", "mediumblob", "longblob", "json", "geometry":
- return false
- }
- return true
-}
-
-func wtBuildFKConstraint(fk *metadatapb.ForeignKeyMetadata) *ast.Constraint {
- return &ast.Constraint{
- Type: ast.ConstrForeignKey,
- Name: fk.Name,
- Columns: fk.Columns,
- RefTable: &ast.TableRef{Schema: fk.ReferencedSchema, Name: fk.ReferencedTable},
- RefColumns: fk.ReferencedColumns,
- OnUpdate: wtFKAction(fk.OnUpdate),
- OnDelete: wtFKAction(fk.OnDelete),
- Match: strings.ToUpper(fk.MatchType),
- }
-}
-
-func wtFKAction(s string) ast.ReferenceAction {
- switch strings.ToUpper(strings.TrimSpace(s)) {
- case "RESTRICT":
- return ast.RefActRestrict
- case "CASCADE":
- return ast.RefActCascade
- case "SET NULL":
- return ast.RefActSetNull
- case "SET DEFAULT":
- return ast.RefActSetDefault
- case "NO ACTION":
- return ast.RefActNoAction
- default:
- return ast.RefActNone
- }
-}
-
-func wtBuildCheckConstraint(chk *metadatapb.CheckConstraintMetadata) *ast.Constraint {
- if chk.Expression == "" {
- return nil
- }
- expr, err := wtParseExpr(chk.Expression)
- if err != nil {
- return nil
- }
- return &ast.Constraint{
- Type: ast.ConstrCheck,
- Name: chk.Name,
- Expr: expr,
- }
-}
-
-// wtBuildCreateViewStmt builds the CreateViewStmt directly; the SELECT body is
-// parsed from VIEW_DEFINITION (a MySQL-produced string, not our deparse).
-// DefineView tolerates a nil Select, so forward references, cyclic views and
-// body-parse failures are all survivable.
-func wtBuildCreateViewStmt(view *metadatapb.ViewMetadata) (*ast.CreateViewStmt, error) {
- if view == nil || view.Name == "" {
- return nil, errors.New("wtBuildCreateViewStmt: empty view")
- }
- stmt := &ast.CreateViewStmt{
- OrReplace: true,
- Name: &ast.TableRef{Name: view.Name},
- SelectText: view.Definition,
- }
- for _, col := range view.Columns {
- if col != nil && col.Name != "" {
- stmt.Columns = append(stmt.Columns, col.Name)
- }
- }
- if view.Definition != "" {
- if sel, err := wtParseSelect(view.Definition); err == nil {
- stmt.Select = sel
- }
- }
- return stmt, nil
-}
-
-// wtParseCreateRoutineStmt parses the full SHOW CREATE FUNCTION / PROCEDURE
-// text that MySQL sync stored in FunctionMetadata.Definition /
-// ProcedureMetadata.Definition. The result is the AST form DefineFunction and
-// DefineProcedure consume.
-func wtParseCreateRoutineStmt(definition string, isProcedure bool) (*ast.CreateFunctionStmt, error) {
- if strings.TrimSpace(definition) == "" {
- return nil, errors.New("empty definition")
- }
- list, err := mysqlparser.Parse(definition)
- if err != nil {
- return nil, err
- }
- if list == nil {
- return nil, errors.New("nil parse result")
- }
- for _, n := range list.Items {
- if stmt, ok := n.(*ast.CreateFunctionStmt); ok {
- stmt.IsProcedure = isProcedure
- return stmt, nil
- }
- }
- return nil, errors.New("no CreateFunctionStmt in parse result")
-}
-
-// wtParseTypeName parses a MySQL column-type string (the output of
-// information_schema.COLUMNS.COLUMN_TYPE, e.g. "varchar(255)",
-// "int unsigned", "enum('a','b')") into *ast.DataType by wrapping it in a
-// throwaway CREATE TABLE and letting the omni parser do the work.
-func wtParseTypeName(typeStr string) (*ast.DataType, error) {
- typeStr = strings.TrimSpace(typeStr)
- if typeStr == "" {
- return nil, errors.New("empty type string")
- }
- sql := "CREATE TABLE `__bb_wt_type_probe` (`__bb_c` " + typeStr + ")"
- list, err := mysqlparser.Parse(sql)
- if err != nil {
- return nil, errors.Wrapf(err, "parse type %q", typeStr)
- }
- if list == nil || len(list.Items) == 0 {
- return nil, errors.Errorf("parse type %q: empty result", typeStr)
- }
- ct, ok := list.Items[0].(*ast.CreateTableStmt)
- if !ok {
- return nil, errors.Errorf("parse type %q: expected CreateTableStmt, got %T", typeStr, list.Items[0])
- }
- if len(ct.Columns) == 0 || ct.Columns[0].TypeName == nil {
- return nil, errors.Errorf("parse type %q: no type in parsed column", typeStr)
- }
- return ct.Columns[0].TypeName, nil
-}
-
-// wtParseExpr parses an expression string via a SELECT wrapper. Used for
-// DEFAULT, ON UPDATE, generated-column and CHECK expressions — all of which
-// originate in MySQL's information_schema or SHOW CREATE output (reality,
-// not our deparse).
-func wtParseExpr(exprStr string) (ast.ExprNode, error) {
- exprStr = strings.TrimSpace(exprStr)
- if exprStr == "" {
- return nil, errors.New("empty expression")
- }
- sql := "SELECT (" + exprStr + ") AS `__bb_wt_expr_probe`"
- list, err := mysqlparser.Parse(sql)
- if err != nil {
- return nil, errors.Wrapf(err, "parse expr %q", exprStr)
- }
- if list == nil || len(list.Items) == 0 {
- return nil, errors.Errorf("parse expr %q: empty result", exprStr)
- }
- sel, ok := list.Items[0].(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("parse expr %q: expected SelectStmt, got %T", exprStr, list.Items[0])
- }
- if len(sel.TargetList) == 0 {
- return nil, errors.Errorf("parse expr %q: empty target list", exprStr)
- }
- rt, ok := sel.TargetList[0].(*ast.ResTarget)
- if !ok {
- return nil, errors.Errorf("parse expr %q: unexpected target %T", exprStr, sel.TargetList[0])
- }
- return rt.Val, nil
-}
-
-// wtParseSelect parses a SELECT body (as returned by VIEW_DEFINITION) into
-// *ast.SelectStmt.
-func wtParseSelect(body string) (*ast.SelectStmt, error) {
- list, err := mysqlparser.Parse(body)
- if err != nil {
- return nil, err
- }
- if list == nil || len(list.Items) == 0 {
- return nil, errors.New("empty parse result")
- }
- sel, ok := list.Items[0].(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("expected SelectStmt, got %T", list.Items[0])
- }
- return sel, nil
-}
-
-func wtUnquoteIdent(s string) string {
- s = strings.TrimSpace(s)
- if len(s) >= 2 && s[0] == '`' && s[len(s)-1] == '`' {
- return strings.ReplaceAll(s[1:len(s)-1], "``", "`")
- }
- return s
-}
-
-// wtInstallPseudoTable installs a degraded table with all-TEXT columns and no
-// constraints. Built as an AST and handed to DefineTable directly — no SQL
-// string and no parser on the critical path.
-func wtInstallPseudoTable(cat *catalog.Catalog, _ string, tbl *metadatapb.TableMetadata) error {
- if tbl == nil || tbl.Name == "" {
- return errors.New("pseudo table: missing name")
- }
-
- stmt := &ast.CreateTableStmt{
- Table: &ast.TableRef{Name: tbl.Name},
- }
-
- seen := make(map[string]bool, len(tbl.Columns))
- for _, col := range tbl.Columns {
- if col == nil || col.Name == "" || seen[col.Name] {
- continue
- }
- seen[col.Name] = true
- stmt.Columns = append(stmt.Columns, &ast.ColumnDef{
- Name: col.Name,
- TypeName: wtPseudoTextType(),
- })
- }
- if len(stmt.Columns) == 0 {
- // MySQL requires at least one column — give it a placeholder so the
- // object still exists and blocks future duplicate-name installs.
- stmt.Columns = append(stmt.Columns, &ast.ColumnDef{
- Name: "__bb_placeholder",
- TypeName: wtPseudoTextType(),
- })
- }
- return cat.DefineTable(stmt)
-}
-
-// wtInstallPseudoView installs a degraded view whose SELECT list is a series
-// of NULL literals aliased to the original view's column names. Built in AST
-// form and installed via DefineView directly.
-func wtInstallPseudoView(cat *catalog.Catalog, _ string, view *metadatapb.ViewMetadata) error {
- if view == nil || view.Name == "" {
- return errors.New("pseudo view: missing name")
- }
-
- var targets []ast.ExprNode
- seen := make(map[string]bool)
- addTarget := func(name string) {
- if name == "" || seen[name] {
- return
- }
- seen[name] = true
- targets = append(targets, &ast.ResTarget{
- Name: name,
- Val: &ast.NullLit{},
- })
- }
- for _, col := range view.Columns {
- if col != nil {
- addTarget(col.Name)
- }
- }
- if len(targets) == 0 {
- for _, dc := range view.DependencyColumns {
- if dc != nil {
- addTarget(dc.Column)
- }
- }
- }
- if len(targets) == 0 {
- addTarget("__bb_placeholder")
- }
-
- return cat.DefineView(&ast.CreateViewStmt{
- OrReplace: true,
- Name: &ast.TableRef{Name: view.Name},
- Select: &ast.SelectStmt{TargetList: targets},
- })
-}
-
-func wtPseudoTextType() *ast.DataType {
- return &ast.DataType{Name: "TEXT"}
-}
-
-// wtBuildCreateTriggerStmt hand-constructs *ast.CreateTriggerStmt from
-// TriggerMetadata. Sync fills only the body (information_schema
-// TRIGGERS.ACTION_STATEMENT) without the CREATE TRIGGER header, so we wire
-// Name/Timing/Event/Table/BodyText directly. Body is a parsed sp_proc_stmt —
-// best-effort only; on parse failure we still hand DefineTrigger a usable
-// stmt with BodyText populated.
-func wtBuildCreateTriggerStmt(tm *metadatapb.TriggerMetadata, tableName string) *ast.CreateTriggerStmt {
- stmt := &ast.CreateTriggerStmt{
- Name: tm.Name,
- Timing: tm.Timing,
- Event: tm.Event,
- Table: &ast.TableRef{Name: tableName},
- BodyText: tm.Body,
- }
- if body := wtParseTriggerBody(tableName, tm.Timing, tm.Event, tm.Body); body != nil {
- stmt.Body = body
- }
- return stmt
-}
-
-// wtParseTriggerBody wraps the raw trigger body in a known-good CREATE TRIGGER
-// template so the omni parser can produce a Body node. Source is MySQL's
-// information_schema output, not our own deparse.
-func wtParseTriggerBody(tableName, timing, event, body string) ast.Node {
- if strings.TrimSpace(body) == "" {
- return nil
- }
- if tableName == "" {
- tableName = "__bb_probe_table"
- }
- if timing == "" {
- timing = "BEFORE"
- }
- if event == "" {
- event = "INSERT"
- }
- sql := "CREATE TRIGGER `__bb_probe_trigger` " + timing + " " + event +
- " ON `" + tableName + "` FOR EACH ROW " + body
- list, err := mysqlparser.Parse(sql)
- if err != nil || list == nil {
- return nil
- }
- for _, n := range list.Items {
- if ct, ok := n.(*ast.CreateTriggerStmt); ok {
- return ct.Body
- }
- }
- return nil
-}
-
-// wtInstallPseudoTrigger installs a trigger stub when the real install path
-// fails (e.g. body node construction tripped up DefineTrigger, or the parent
-// table name was empty in metadata). The stub preserves Name and routing
-// (Timing/Event/Table) so references still resolve; Body collapses to the
-// trivial no-op "BEGIN END".
-func wtInstallPseudoTrigger(cat *catalog.Catalog, tm *metadatapb.TriggerMetadata, tableName string) error {
- if tm == nil || tm.Name == "" {
- return errors.New("pseudo trigger: missing name")
- }
- timing := tm.Timing
- if timing == "" {
- timing = "BEFORE"
- }
- event := tm.Event
- if event == "" {
- event = "INSERT"
- }
- if tableName == "" {
- tableName = "__bb_placeholder"
- }
- return cat.DefineTrigger(&ast.CreateTriggerStmt{
- Name: tm.Name,
- Timing: timing,
- Event: event,
- Table: &ast.TableRef{Name: tableName},
- BodyText: "BEGIN END",
- })
-}
-
-// wtParseCreateEventStmt parses the full SHOW CREATE EVENT text that sync
-// stored in EventMetadata.Definition into *ast.CreateEventStmt.
-func wtParseCreateEventStmt(definition string) (*ast.CreateEventStmt, error) {
- if strings.TrimSpace(definition) == "" {
- return nil, errors.New("empty event definition")
- }
- list, err := mysqlparser.Parse(definition)
- if err != nil {
- return nil, err
- }
- if list == nil {
- return nil, errors.New("nil parse result")
- }
- for _, n := range list.Items {
- if ev, ok := n.(*ast.CreateEventStmt); ok {
- return ev, nil
- }
- }
- return nil, errors.New("no CreateEventStmt in parse result")
-}
-
-// wtInstallPseudoEvent installs a bare name-only event when the real install
-// path fails (Definition was empty, malformed, or rejected by DefineEvent).
-func wtInstallPseudoEvent(cat *catalog.Catalog, em *metadatapb.EventMetadata) error {
- if em == nil || em.Name == "" {
- return errors.New("pseudo event: missing name")
- }
- return cat.DefineEvent(&ast.CreateEventStmt{
- Name: em.Name,
- })
-}
diff --git a/backend/plugin/schema/mysql/walk_through_loader_test.go b/backend/plugin/schema/mysql/walk_through_loader_test.go
deleted file mode 100644
index b80305d9e8..0000000000
--- a/backend/plugin/schema/mysql/walk_through_loader_test.go
+++ /dev/null
@@ -1,1138 +0,0 @@
-package mysql
-
-import (
- "context"
- "strings"
- "testing"
-
- metadatapb "github.com/bytebase/omni/metadata"
- "github.com/bytebase/omni/mysql/catalog"
-)
-
-// These tests exercise loadWalkThroughCatalog against synthetic metadata.
-// Nothing here touches bb_export or any external fixture — every input is a
-// storepb.*Metadata constructed in memory.
-
-const testDBName = "testdb"
-
-// newLoaderTestCatalog returns a catalog with the test database created and
-// selected, matching the setup WalkThroughWithContext performs before the loader runs.
-func newLoaderTestCatalog(t *testing.T) *catalog.Catalog {
- t.Helper()
- c := catalog.New()
- if _, err := c.Exec(
- "SET foreign_key_checks = 0;\nCREATE DATABASE IF NOT EXISTS `"+testDBName+"`;\nUSE `"+testDBName+"`;",
- &catalog.ExecOptions{ContinueOnError: true},
- ); err != nil {
- t.Fatalf("init catalog: %v", err)
- }
- return c
-}
-
-func runLoader(t *testing.T, c *catalog.Catalog, meta *metadatapb.DatabaseSchemaMetadata) {
- t.Helper()
- if err := loadWalkThroughCatalog(context.Background(), c, testDBName, meta); err != nil {
- t.Fatalf("loadWalkThroughCatalog: %v", err)
- }
-}
-
-func mustGetTable(t *testing.T, c *catalog.Catalog, name string) *catalog.Table {
- t.Helper()
- db := c.GetDatabase(testDBName)
- if db == nil {
- t.Fatalf("database %q not in catalog", testDBName)
- }
- tbl := db.Tables[strings.ToLower(name)]
- if tbl == nil {
- t.Fatalf("table %q not in catalog", name)
- }
- return tbl
-}
-
-func mustGetView(t *testing.T, c *catalog.Catalog, name string) *catalog.View {
- t.Helper()
- db := c.GetDatabase(testDBName)
- if db == nil {
- t.Fatalf("database %q not in catalog", testDBName)
- }
- v := db.Views[strings.ToLower(name)]
- if v == nil {
- t.Fatalf("view %q not in catalog", name)
- }
- return v
-}
-
-func mustGetTrigger(t *testing.T, c *catalog.Catalog, name string) *catalog.Trigger {
- t.Helper()
- db := c.GetDatabase(testDBName)
- if db == nil {
- t.Fatalf("database %q not in catalog", testDBName)
- }
- trg := db.Triggers[strings.ToLower(name)]
- if trg == nil {
- t.Fatalf("trigger %q not in catalog", name)
- }
- return trg
-}
-
-func mustGetEvent(t *testing.T, c *catalog.Catalog, name string) *catalog.Event {
- t.Helper()
- db := c.GetDatabase(testDBName)
- if db == nil {
- t.Fatalf("database %q not in catalog", testDBName)
- }
- ev := db.Events[strings.ToLower(name)]
- if ev == nil {
- t.Fatalf("event %q not in catalog", name)
- }
- return ev
-}
-
-// ----------------------------------------------------------------------
-// wtParseTypeName — covers a representative cross-section of MySQL types.
-// ----------------------------------------------------------------------
-
-func TestWtParseTypeName(t *testing.T) {
- cases := []struct {
- in string
- wantName string
- }{
- {"int", "INT"},
- {"int unsigned", "INT"},
- {"bigint(20) unsigned zerofill", "BIGINT"},
- {"varchar(255)", "VARCHAR"},
- {"char(1)", "CHAR"},
- {"text", "TEXT"},
- {"decimal(10,2)", "DECIMAL"},
- {"datetime(6)", "DATETIME"},
- {"timestamp", "TIMESTAMP"},
- {"tinyint(1)", "TINYINT"},
- {"blob", "BLOB"},
- {"json", "JSON"},
- {"enum('a','b','c')", "ENUM"},
- {"set('x','y')", "SET"},
- {"geometry", "GEOMETRY"},
- {"bit(1)", "BIT"},
- }
- for _, tc := range cases {
- t.Run(tc.in, func(t *testing.T) {
- dt, err := wtParseTypeName(tc.in)
- if err != nil {
- t.Fatalf("wtParseTypeName(%q): %v", tc.in, err)
- }
- if !strings.EqualFold(dt.Name, tc.wantName) {
- t.Errorf("wtParseTypeName(%q).Name = %q, want %q", tc.in, dt.Name, tc.wantName)
- }
- })
- }
-}
-
-func TestWtParseTypeName_Failures(t *testing.T) {
- for _, in := range []string{"", "not a real type (((", "int UNSIGNED ZEROFILL NOT_A_THING"} {
- if _, err := wtParseTypeName(in); err == nil {
- t.Errorf("wtParseTypeName(%q): expected error, got nil", in)
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// wtParseExpr — default/ON UPDATE/generated/check expressions.
-// ----------------------------------------------------------------------
-
-func TestWtParseExpr(t *testing.T) {
- for _, in := range []string{
- `'hello'`,
- `CURRENT_TIMESTAMP`,
- `NULL`,
- `0`,
- `a + b`,
- `CONCAT('x', 'y')`,
- `DATE_FORMAT(NOW(), '%Y-%m-%d')`,
- } {
- if _, err := wtParseExpr(in); err != nil {
- t.Errorf("wtParseExpr(%q) unexpectedly failed: %v", in, err)
- }
- }
-}
-
-func TestWtParseExpr_Failures(t *testing.T) {
- // "SELECT 1" is intentionally NOT in this list: wrapped in our SELECT
- // probe it parses as a scalar subquery expression, which is valid.
- for _, in := range []string{"", "(((", "this is not sql at all ###"} {
- if _, err := wtParseExpr(in); err == nil {
- t.Errorf("wtParseExpr(%q): expected error, got nil", in)
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// Real table install — exercises the full column + constraint + option path.
-// ----------------------------------------------------------------------
-
-func TestLoader_TableBasic(t *testing.T) {
- meta := schemaWithTables(&metadatapb.TableMetadata{
- Name: "users",
- Engine: "InnoDB",
- Charset: "utf8mb4",
- Collation: "utf8mb4_0900_ai_ci",
- Comment: "users table",
- Columns: []*metadatapb.ColumnMetadata{
- {
- Name: "id",
- Type: "bigint unsigned",
- Nullable: false,
- Default: autoIncrementSentinel,
- },
- {
- Name: "email",
- Type: "varchar(255)",
- Nullable: false,
- CharacterSet: "utf8mb4",
- Collation: "utf8mb4_0900_ai_ci",
- },
- {
- Name: "created_at",
- Type: "timestamp",
- Nullable: false,
- Default: "CURRENT_TIMESTAMP",
- OnUpdate: "CURRENT_TIMESTAMP",
- },
- {
- Name: "bio",
- Type: "text",
- Nullable: true,
- Comment: "free form text",
- },
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- {Name: "uniq_email", Type: "BTREE", Unique: true, Expressions: []string{"email"}},
- },
- })
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- tbl := mustGetTable(t, c, "users")
- if got := len(tbl.Columns); got != 4 {
- t.Fatalf("Columns: want 4, got %d", got)
- }
- if got := tbl.Engine; got != "InnoDB" {
- t.Errorf("Engine: %q", got)
- }
- if got := tbl.Charset; got != "utf8mb4" {
- t.Errorf("Charset: %q", got)
- }
- if got := tbl.Comment; got != "users table" {
- t.Errorf("Comment: %q", got)
- }
-
- // Column properties.
- idCol := tbl.GetColumn("id")
- if idCol == nil {
- t.Fatal("id column missing")
- }
- if !idCol.AutoIncrement {
- t.Error("id should be AUTO_INCREMENT (sentinel detection)")
- }
- if idCol.Nullable {
- t.Error("id should be NOT NULL")
- }
- if strings.ToLower(idCol.DataType) != "bigint" {
- t.Errorf("id.DataType = %q", idCol.DataType)
- }
-
- emailCol := tbl.GetColumn("email")
- if emailCol == nil || emailCol.Charset != "utf8mb4" {
- t.Errorf("email.Charset: %q", emailCol.Charset)
- }
-
- createdCol := tbl.GetColumn("created_at")
- // Omni's catalog canonicalizes CURRENT_TIMESTAMP to now() when deparsing
- // the parsed expression; both forms are equivalent so assert presence of
- // a default and ON UPDATE rather than a specific string.
- if createdCol == nil || createdCol.Default == nil || *createdCol.Default == "" {
- t.Errorf("created_at default missing: %+v", createdCol)
- }
- if createdCol == nil || createdCol.OnUpdate == "" {
- t.Error("created_at OnUpdate missing")
- }
-
- bioCol := tbl.GetColumn("bio")
- if bioCol == nil || bioCol.Comment != "free form text" {
- t.Errorf("bio.Comment: %q", bioCol.Comment)
- }
-
- // Primary key must be present as a constraint.
- var hasPK, hasUnique bool
- for _, con := range tbl.Constraints {
- switch con.Type {
- case catalog.ConPrimaryKey:
- hasPK = true
- case catalog.ConUniqueKey:
- if con.Name == "uniq_email" {
- hasUnique = true
- }
- default:
- }
- }
- if !hasPK {
- t.Error("missing PRIMARY KEY constraint")
- }
- if !hasUnique {
- t.Error("missing uniq_email unique constraint")
- }
-}
-
-// ----------------------------------------------------------------------
-// Fulltext / Spatial index fix — the regression we're covering in this PR.
-// ----------------------------------------------------------------------
-
-func TestLoader_FulltextAndSpatialIndex(t *testing.T) {
- meta := schemaWithTables(&metadatapb.TableMetadata{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},
- {Name: "body", Type: "text", Nullable: true},
- {Name: "location", Type: "geometry", Nullable: false},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- {Name: "ft_body", Type: "FULLTEXT", Expressions: []string{"body"}},
- {Name: "sp_location", Type: "SPATIAL", Expressions: []string{"location"}},
- },
- })
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- tbl := mustGetTable(t, c, "t")
- var gotFulltext, gotSpatial bool
- for _, idx := range tbl.Indexes {
- switch idx.Name {
- case "ft_body":
- gotFulltext = idx.Fulltext
- if !idx.Fulltext {
- t.Errorf("ft_body installed but not marked as Fulltext; IndexType=%q", idx.IndexType)
- }
- case "sp_location":
- gotSpatial = idx.Spatial
- if !idx.Spatial {
- t.Errorf("sp_location installed but not marked as Spatial; IndexType=%q", idx.IndexType)
- }
- default:
- // Other indexes (e.g. PRIMARY) are irrelevant here.
- }
- }
- if !gotFulltext {
- t.Error("fulltext index not found on table")
- }
- if !gotSpatial {
- t.Error("spatial index not found on table")
- }
-}
-
-// ----------------------------------------------------------------------
-// FK install — foreign_key_checks off, the constraint is recorded unchecked.
-// ----------------------------------------------------------------------
-
-func TestLoader_ForeignKeyAcrossTables(t *testing.T) {
- meta := schemaWithTables(
- &metadatapb.TableMetadata{
- Name: "parent",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- },
- },
- &metadatapb.TableMetadata{
- Name: "child",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},
- {Name: "parent_id", Type: "bigint unsigned", Nullable: false},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- {Name: "idx_parent_id", Type: "BTREE", Expressions: []string{"parent_id"}},
- },
- ForeignKeys: []*metadatapb.ForeignKeyMetadata{
- {
- Name: "fk_child_parent",
- Columns: []string{"parent_id"},
- ReferencedTable: "parent",
- ReferencedColumns: []string{"id"},
- OnDelete: "CASCADE",
- OnUpdate: "RESTRICT",
- },
- },
- },
- )
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- child := mustGetTable(t, c, "child")
- var fk *catalog.Constraint
- for _, con := range child.Constraints {
- if con.Type == catalog.ConForeignKey && con.Name == "fk_child_parent" {
- fk = con
- break
- }
- }
- if fk == nil {
- t.Fatal("fk_child_parent constraint missing")
- }
- if strings.ToUpper(fk.OnDelete) != "CASCADE" {
- t.Errorf("OnDelete = %q, want CASCADE", fk.OnDelete)
- }
- if strings.ToUpper(fk.OnUpdate) != "RESTRICT" {
- t.Errorf("OnUpdate = %q, want RESTRICT", fk.OnUpdate)
- }
-}
-
-// ----------------------------------------------------------------------
-// Generated columns.
-// ----------------------------------------------------------------------
-
-func TestLoader_GeneratedColumn(t *testing.T) {
- meta := schemaWithTables(&metadatapb.TableMetadata{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "a", Type: "int", Nullable: true},
- {Name: "b", Type: "int", Nullable: true},
- {
- Name: "c",
- Type: "int",
- Nullable: true,
- Generation: &metadatapb.GenerationMetadata{
- Type: metadatapb.GenerationMetadata_TYPE_STORED,
- Expression: "a + b",
- },
- },
- },
- })
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- tbl := mustGetTable(t, c, "t")
- col := tbl.GetColumn("c")
- if col == nil || col.Generated == nil {
- t.Fatalf("generated column not recorded: %+v", col)
- }
- if !col.Generated.Stored {
- t.Error("expected STORED, got VIRTUAL")
- }
- if !strings.Contains(col.Generated.Expr, "a") || !strings.Contains(col.Generated.Expr, "b") {
- t.Errorf("Generated.Expr = %q", col.Generated.Expr)
- }
-}
-
-// ----------------------------------------------------------------------
-// View real install.
-// ----------------------------------------------------------------------
-
-func TestLoader_ViewReal(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: testDBName,
- Schemas: []*metadatapb.SchemaMetadata{
- {
- Name: "",
- Tables: []*metadatapb.TableMetadata{
- {
- Name: "users",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},
- {Name: "name", Type: "varchar(255)", Nullable: true},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- },
- },
- },
- Views: []*metadatapb.ViewMetadata{
- {
- Name: "v_users",
- Definition: "SELECT id, name FROM users",
- },
- },
- },
- },
- }
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- v := mustGetView(t, c, "v_users")
- if !strings.Contains(strings.ToUpper(v.Definition), "SELECT") {
- t.Errorf("view Definition not preserved: %q", v.Definition)
- }
-}
-
-// ----------------------------------------------------------------------
-// Pseudo fallbacks. We exercise them directly by calling the helpers because
-// the AST-direct real path is permissive enough that natural failures are
-// rare; going through the helper means we really test the Define* + ast
-// construction that the fallback relies on.
-// ----------------------------------------------------------------------
-
-func TestLoader_PseudoTable(t *testing.T) {
- c := newLoaderTestCatalog(t)
- // Minimal metadata — just column names.
- tblMeta := &metadatapb.TableMetadata{
- Name: "pt",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "a"},
- {Name: "b"},
- {Name: "c"},
- },
- }
- if err := wtInstallPseudoTable(c, testDBName, tblMeta); err != nil {
- t.Fatalf("wtInstallPseudoTable: %v", err)
- }
- tbl := mustGetTable(t, c, "pt")
- if len(tbl.Columns) != 3 {
- t.Fatalf("pseudo columns: got %d want 3", len(tbl.Columns))
- }
- for _, col := range tbl.Columns {
- if !strings.EqualFold(col.DataType, "text") {
- t.Errorf("pseudo column %q: DataType=%q, want text", col.Name, col.DataType)
- }
- }
-}
-
-func TestLoader_PseudoTable_EmptyColumns(t *testing.T) {
- c := newLoaderTestCatalog(t)
- if err := wtInstallPseudoTable(c, testDBName, &metadatapb.TableMetadata{Name: "empty"}); err != nil {
- t.Fatalf("wtInstallPseudoTable(no columns): %v", err)
- }
- tbl := mustGetTable(t, c, "empty")
- if len(tbl.Columns) != 1 {
- t.Fatalf("placeholder columns: got %d want 1", len(tbl.Columns))
- }
- if tbl.Columns[0].Name != "__bb_placeholder" {
- t.Errorf("placeholder column name: %q", tbl.Columns[0].Name)
- }
-}
-
-func TestLoader_PseudoView(t *testing.T) {
- c := newLoaderTestCatalog(t)
- if err := wtInstallPseudoView(c, testDBName, &metadatapb.ViewMetadata{
- Name: "pv",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "x"},
- {Name: "y"},
- },
- }); err != nil {
- t.Fatalf("wtInstallPseudoView: %v", err)
- }
- v := mustGetView(t, c, "pv")
- if len(v.Columns) != 2 {
- t.Fatalf("pseudo view columns: got %d want 2", len(v.Columns))
- }
-}
-
-// ----------------------------------------------------------------------
-// Fallback chain — real install fails, pseudo kicks in.
-// Simulate by giving a column an unparseable type string.
-// ----------------------------------------------------------------------
-
-func TestLoader_RealFailsFallsBackToPseudo(t *testing.T) {
- meta := schemaWithTables(&metadatapb.TableMetadata{
- Name: "broken",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "good", Type: "int"},
- // Type with unbalanced parens — wtParseTypeName returns an error
- // for this, so wtBuildCreateTableStmt propagates and install_real
- // fails, triggering pseudo install.
- {Name: "bad", Type: "varchar((("},
- },
- })
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- tbl := mustGetTable(t, c, "broken")
- if len(tbl.Columns) != 2 {
- t.Fatalf("pseudo columns: got %d want 2", len(tbl.Columns))
- }
- // Pseudo collapses everything to TEXT.
- for _, col := range tbl.Columns {
- if !strings.EqualFold(col.DataType, "text") {
- t.Errorf("pseudo fallback should collapse columns to TEXT; %q.DataType=%q", col.Name, col.DataType)
- }
- }
-}
-
-// ----------------------------------------------------------------------
-// Topological ordering — FK forward references work with FK checks off.
-// ----------------------------------------------------------------------
-
-func TestLoader_ForwardFKInstallsViaFKChecksOff(t *testing.T) {
- // Child declared first in the metadata, referencing parent that appears
- // after. Because the loader flips foreign_key_checks off during bulk
- // install, this must succeed.
- meta := schemaWithTables(
- &metadatapb.TableMetadata{
- Name: "child",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},
- {Name: "parent_id", Type: "bigint unsigned", Nullable: false},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- {Name: "idx_pid", Type: "BTREE", Expressions: []string{"parent_id"}},
- },
- ForeignKeys: []*metadatapb.ForeignKeyMetadata{
- {
- Name: "fk_fwd",
- Columns: []string{"parent_id"},
- ReferencedTable: "parent",
- ReferencedColumns: []string{"id"},
- },
- },
- },
- &metadatapb.TableMetadata{
- Name: "parent",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "bigint unsigned", Nullable: false, Default: autoIncrementSentinel},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- },
- },
- )
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- // Both tables present, child's FK is recorded.
- mustGetTable(t, c, "parent")
- child := mustGetTable(t, c, "child")
- var found bool
- for _, con := range child.Constraints {
- if con.Type == catalog.ConForeignKey && con.Name == "fk_fwd" {
- found = true
- }
- }
- if !found {
- t.Error("fk_fwd not recorded on child; forward-FK install failed")
- }
-}
-
-// schemaWithTables wraps tables into a DatabaseSchemaMetadata with the
-// expected single empty-named schema that MySQL uses.
-func schemaWithTables(tables ...*metadatapb.TableMetadata) *metadatapb.DatabaseSchemaMetadata {
- return &metadatapb.DatabaseSchemaMetadata{
- Name: testDBName,
- Schemas: []*metadatapb.SchemaMetadata{
- {
- Name: "",
- Tables: tables,
- },
- },
- }
-}
-
-// schemaWithEvents wraps events into a DatabaseSchemaMetadata.
-func schemaWithEvents(events ...*metadatapb.EventMetadata) *metadatapb.DatabaseSchemaMetadata {
- return &metadatapb.DatabaseSchemaMetadata{
- Name: testDBName,
- Schemas: []*metadatapb.SchemaMetadata{
- {Name: "", Events: events},
- },
- }
-}
-
-// ----------------------------------------------------------------------
-// Trigger — happy paths.
-//
-// Examples match MySQL 8.0 documentation §25.3.1 (Trigger Syntax and
-// Examples). Each trigger is attached to a real table so the loader exercises
-// the actual kindWTTable → kindWTTrigger ordering.
-// ----------------------------------------------------------------------
-
-// accountTableForTriggers builds the `account` table used by the canonical
-// trigger examples in the MySQL docs.
-func accountTableForTriggers() *metadatapb.TableMetadata {
- return &metadatapb.TableMetadata{
- Name: "account",
- Engine: "InnoDB",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "acct_num", Type: "int", Nullable: true},
- {Name: "amount", Type: "decimal(10,2)", Nullable: true},
- },
- }
-}
-
-func TestLoader_Trigger_SimpleSetSum(t *testing.T) {
- // MySQL docs §25.3.1 Example 1:
- //
- // CREATE TRIGGER ins_sum BEFORE INSERT ON account
- // FOR EACH ROW SET @sum = @sum + NEW.amount;
- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{
- Name: "ins_sum",
- Timing: "BEFORE",
- Event: "INSERT",
- Body: "SET @sum = @sum + NEW.amount",
- }))
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- trg := mustGetTrigger(t, c, "ins_sum")
- if !strings.EqualFold(trg.Timing, "BEFORE") {
- t.Errorf("Timing=%q, want BEFORE", trg.Timing)
- }
- if !strings.EqualFold(trg.Event, "INSERT") {
- t.Errorf("Event=%q, want INSERT", trg.Event)
- }
- if !strings.EqualFold(trg.Table, "account") {
- t.Errorf("Table=%q, want account", trg.Table)
- }
- if !strings.Contains(trg.Body, "@sum") {
- t.Errorf("Body lost raw text: %q", trg.Body)
- }
-}
-
-func TestLoader_Trigger_BeginEndWithIfElse(t *testing.T) {
- // MySQL docs §25.3.1 Example 2 (BEGIN … END with IF/ELSEIF/END IF):
- //
- // CREATE TRIGGER upd_check BEFORE UPDATE ON account
- // FOR EACH ROW
- // BEGIN
- // IF NEW.amount < 0 THEN
- // SET NEW.amount = 0;
- // ELSEIF NEW.amount > 100 THEN
- // SET NEW.amount = 100;
- // END IF;
- // END;
- body := `BEGIN
- IF NEW.amount < 0 THEN
- SET NEW.amount = 0;
- ELSEIF NEW.amount > 100 THEN
- SET NEW.amount = 100;
- END IF;
- END`
- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{
- Name: "upd_check",
- Timing: "BEFORE",
- Event: "UPDATE",
- Body: body,
- }))
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- trg := mustGetTrigger(t, c, "upd_check")
- if !strings.Contains(strings.ToUpper(trg.Body), "IF NEW.AMOUNT") {
- t.Errorf("Body lost IF/THEN content: %q", trg.Body)
- }
-}
-
-func TestLoader_Trigger_AfterUpdateWithSignal(t *testing.T) {
- // MySQL docs §15.6.7.1 (SIGNAL inside a trigger):
- //
- // CREATE TRIGGER validate_amount AFTER UPDATE ON account
- // FOR EACH ROW
- // BEGIN
- // IF NEW.amount < 0 THEN
- // SIGNAL SQLSTATE '45000'
- // SET MESSAGE_TEXT = 'Negative amount not allowed';
- // END IF;
- // END;
- body := `BEGIN
- IF NEW.amount < 0 THEN
- SIGNAL SQLSTATE '45000'
- SET MESSAGE_TEXT = 'Negative amount not allowed';
- END IF;
- END`
- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{
- Name: "validate_amount",
- Timing: "AFTER",
- Event: "UPDATE",
- Body: body,
- }))
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- trg := mustGetTrigger(t, c, "validate_amount")
- if !strings.EqualFold(trg.Timing, "AFTER") {
- t.Errorf("Timing=%q", trg.Timing)
- }
- if !strings.Contains(strings.ToUpper(trg.Body), "SIGNAL") {
- t.Errorf("Body lost SIGNAL: %q", trg.Body)
- }
-}
-
-// ----------------------------------------------------------------------
-// Trigger — pseudo fallback.
-// ----------------------------------------------------------------------
-
-func TestLoader_Trigger_PseudoOnMissingFields(t *testing.T) {
- // metadata has only Name (+ Body) — Timing and Event empty. Real install
- // should fail (DefineTrigger rejects empty Timing/Event / empty Body
- // couldn't parse). Pseudo fills defaults.
- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{
- Name: "trg_needs_defaults",
- // Timing / Event / Body intentionally empty.
- }))
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- trg := mustGetTrigger(t, c, "trg_needs_defaults")
- if trg.Timing == "" {
- t.Error("pseudo trigger Timing should be filled")
- }
- if trg.Event == "" {
- t.Error("pseudo trigger Event should be filled")
- }
- if !strings.EqualFold(trg.Table, "account") {
- t.Errorf("pseudo trigger Table=%q, want account", trg.Table)
- }
-}
-
-func TestLoader_Trigger_PseudoOnUnparseableBody(t *testing.T) {
- // A body the parser can't chew up — real install still succeeds with
- // Body=nil + BodyText populated (DefineTrigger tolerates nil Body), or
- // falls back to pseudo with body "BEGIN END". Either way the trigger
- // must exist in the catalog.
- meta := schemaWithTables(withTriggers(accountTableForTriggers(), &metadatapb.TriggerMetadata{
- Name: "trg_bad_body",
- Timing: "BEFORE",
- Event: "INSERT",
- Body: "this is not sql $$$",
- }))
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- trg := mustGetTrigger(t, c, "trg_bad_body")
- if !strings.EqualFold(trg.Timing, "BEFORE") {
- t.Errorf("Timing=%q", trg.Timing)
- }
-}
-
-func TestLoader_Trigger_OnPseudoParentTable(t *testing.T) {
- // Parent table has a bad column type → real install fails, pseudo
- // installs TEXT-column table. The trigger must still attach.
- parent := &metadatapb.TableMetadata{
- Name: "account",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int"},
- {Name: "bad", Type: "varchar((("}, // forces real install to fail
- },
- }
- meta := schemaWithTables(withTriggers(parent, &metadatapb.TriggerMetadata{
- Name: "ins_stub",
- Timing: "BEFORE",
- Event: "INSERT",
- Body: "SET @x = NEW.id",
- }))
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- // Parent was pseudo-installed.
- tbl := mustGetTable(t, c, "account")
- if !strings.EqualFold(tbl.Columns[0].DataType, "text") {
- t.Fatalf("expected pseudo parent table; columns=%+v", tbl.Columns)
- }
- // Trigger attached anyway.
- mustGetTrigger(t, c, "ins_stub")
-}
-
-// ----------------------------------------------------------------------
-// Event — happy paths.
-//
-// Examples match MySQL 8.0 documentation §15.1.12 (CREATE EVENT). The
-// Definition string is exactly what SHOW CREATE EVENT would produce — our
-// loader parses it to AST via wtParseCreateEventStmt.
-// ----------------------------------------------------------------------
-
-func TestLoader_Event_OneShotAT(t *testing.T) {
- // MySQL docs §15.1.12 Example 1 (one-shot):
- //
- // CREATE EVENT myevent
- // ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR
- // DO UPDATE mytable SET mycol = mycol + 1;
- def := "CREATE EVENT `myevent` " +
- "ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 1 HOUR " +
- "DO UPDATE mytable SET mycol = mycol + 1"
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{
- Name: "myevent",
- Definition: def,
- }))
-
- ev := mustGetEvent(t, c, "myevent")
- if !strings.Contains(strings.ToUpper(ev.Schedule), "CURRENT_TIMESTAMP") &&
- !strings.Contains(strings.ToUpper(ev.Schedule), "INTERVAL") {
- t.Errorf("schedule lost: %q", ev.Schedule)
- }
-}
-
-func TestLoader_Event_RecurringEveryWithComment(t *testing.T) {
- // MySQL docs §15.1.12 Example 2 (recurring + comment):
- //
- // CREATE EVENT e_hourly
- // ON SCHEDULE EVERY 1 HOUR
- // COMMENT 'Clears out sessions table each hour.'
- // DO DELETE FROM site_activity.sessions;
- def := "CREATE EVENT `e_hourly` " +
- "ON SCHEDULE EVERY 1 HOUR " +
- "COMMENT 'Clears out sessions table each hour.' " +
- "DO DELETE FROM site_activity.sessions"
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{
- Name: "e_hourly",
- Definition: def,
- }))
-
- ev := mustGetEvent(t, c, "e_hourly")
- if !strings.Contains(strings.ToUpper(ev.Schedule), "EVERY") ||
- !strings.Contains(strings.ToUpper(ev.Schedule), "HOUR") {
- t.Errorf("schedule lost: %q", ev.Schedule)
- }
- if !strings.Contains(ev.Comment, "sessions") {
- t.Errorf("comment lost: %q", ev.Comment)
- }
-}
-
-func TestLoader_Event_DailyBeginEndBody(t *testing.T) {
- // MySQL docs §15.1.12 Example 3 (EVERY + STARTS + compound body):
- //
- // CREATE EVENT e_daily
- // ON SCHEDULE
- // EVERY 1 DAY
- // STARTS CURRENT_TIMESTAMP + INTERVAL 5 HOUR
- // COMMENT '...'
- // DO
- // BEGIN
- // INSERT INTO totals (time, total) SELECT NOW(), COUNT(*) FROM sessions;
- // DELETE FROM sessions;
- // END;
- def := "CREATE EVENT `e_daily` " +
- "ON SCHEDULE EVERY 1 DAY STARTS CURRENT_TIMESTAMP + INTERVAL 5 HOUR " +
- "COMMENT 'Saves total then clears each day' " +
- "DO BEGIN " +
- " INSERT INTO totals (time, total) SELECT NOW(), COUNT(*) FROM sessions; " +
- " DELETE FROM sessions; " +
- "END"
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{
- Name: "e_daily",
- Definition: def,
- }))
-
- ev := mustGetEvent(t, c, "e_daily")
- if !strings.Contains(strings.ToUpper(ev.Schedule), "STARTS") {
- t.Errorf("schedule lost STARTS: %q", ev.Schedule)
- }
-}
-
-// ----------------------------------------------------------------------
-// Event — pseudo fallback.
-// ----------------------------------------------------------------------
-
-func TestLoader_Event_PseudoOnEmptyDefinition(t *testing.T) {
- c := newLoaderTestCatalog(t)
- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{
- Name: "e_empty",
- // Definition intentionally empty — real install fails, pseudo kicks in.
- }))
- mustGetEvent(t, c, "e_empty")
-}
-
-func TestLoader_Event_PseudoOnGarbageDefinition(t *testing.T) {
- c := newLoaderTestCatalog(t)
- runLoader(t, c, schemaWithEvents(&metadatapb.EventMetadata{
- Name: "e_broken",
- Definition: "this is definitely not a CREATE EVENT statement $$",
- }))
- mustGetEvent(t, c, "e_broken")
-}
-
-// withTriggers is a tiny builder that attaches triggers to a given table.
-func withTriggers(tbl *metadatapb.TableMetadata, triggers ...*metadatapb.TriggerMetadata) *metadatapb.TableMetadata {
- tbl.Triggers = append(tbl.Triggers, triggers...)
- return tbl
-}
-
-// ----------------------------------------------------------------------
-// Regression: MySQL sync populates `Default="NULL"` for every nullable
-// column — including BLOB / JSON / GEOMETRY types whose grammar rejects a
-// DEFAULT clause. The loader must skip DefaultValue for those types so
-// DefineTable doesn't reject the whole table and force pseudo fallback.
-// ----------------------------------------------------------------------
-
-func TestLoader_DefaultNull_OnBlobJsonGeometry_StaysReal(t *testing.T) {
- meta := schemaWithTables(&metadatapb.TableMetadata{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int", Nullable: false, Default: autoIncrementSentinel},
- // All three of these are nullable — sync fills Default="NULL"
- // for them. MySQL grammar does NOT accept DEFAULT on these
- // types, so the loader must drop the default silently.
- {Name: "payload_json", Type: "json", Nullable: true, Default: "NULL"},
- {Name: "blob_body", Type: "blob", Nullable: true, Default: "NULL"},
- {Name: "location", Type: "geometry", Nullable: true, Default: "NULL"},
- // For a type that DOES support DEFAULT, Default="NULL" should
- // still be applied.
- {Name: "name", Type: "varchar(255)", Nullable: true, Default: "NULL"},
- },
- })
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- tbl := mustGetTable(t, c, "t")
- // Real install path keeps real column types — verify by checking a
- // non-TEXT DataType on the JSON / BLOB / GEOMETRY columns.
- for _, want := range []struct{ name, dt string }{
- {"payload_json", "json"},
- {"blob_body", "blob"},
- {"location", "geometry"},
- } {
- got := tbl.GetColumn(want.name)
- if got == nil {
- t.Errorf("column %q missing", want.name)
- continue
- }
- if !strings.EqualFold(got.DataType, want.dt) {
- t.Errorf("column %q: DataType=%q, want %q (pseudo fallback leaked?)", want.name, got.DataType, want.dt)
- }
- // The DEFAULT must have been suppressed — BLOB/JSON/GEOMETRY must
- // NOT carry a stored default.
- if got.Default != nil {
- t.Errorf("column %q: Default=%v, want nil for unsupported-default type", want.name, *got.Default)
- }
- }
- // Regular varchar still carries DEFAULT NULL.
- if name := tbl.GetColumn("name"); name == nil || name.Default == nil {
- t.Error("varchar column lost its DEFAULT NULL")
- }
-}
-
-// ----------------------------------------------------------------------
-// Regression: MySQL 8.0 functional indexes store key parts as expressions
-// like `(lower(name))` or `lower(name)`. Dumping them into Constraint.Columns
-// (bare identifier slice) makes DefineTable reject the table. Expression
-// keys must land in IndexColumns[*].Expr instead.
-// ----------------------------------------------------------------------
-
-func TestLoader_FunctionalIndex_ParenthesizedExpression(t *testing.T) {
- // MySQL docs §10.3.10 Functional Key Parts:
- //
- // CREATE TABLE t1 (
- // col1 INT, col2 INT,
- // INDEX func_index ((ABS(col1)))
- // );
- meta := schemaWithTables(&metadatapb.TableMetadata{
- Name: "t1",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "col1", Type: "int", Nullable: true},
- {Name: "col2", Type: "int", Nullable: true},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {
- Name: "func_index",
- Type: "BTREE",
- Expressions: []string{"(abs(col1))"},
- },
- },
- })
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- tbl := mustGetTable(t, c, "t1")
- if strings.EqualFold(tbl.Columns[0].DataType, "text") {
- t.Fatal("table degraded to pseudo (all columns TEXT) — functional index broke real install")
- }
- // Index must exist.
- var found bool
- for _, idx := range tbl.Indexes {
- if idx.Name == "func_index" {
- found = true
- // Catalog stores the key-part expression in IndexColumn.Expr.
- if len(idx.Columns) != 1 {
- t.Fatalf("func_index: expected 1 key part, got %d", len(idx.Columns))
- }
- }
- }
- if !found {
- t.Error("func_index not installed on table")
- }
-}
-
-func TestLoader_FunctionalIndex_FunctionCallForm(t *testing.T) {
- // Same thing but key part is stored as bare function-call text
- // (lower(name) without outer parens) — another shape the proto comment
- // explicitly calls out.
- meta := schemaWithTables(&metadatapb.TableMetadata{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int", Nullable: false, Default: autoIncrementSentinel},
- {Name: "name", Type: "varchar(255)", Nullable: true},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- {Name: "idx_lower_name", Type: "BTREE", Expressions: []string{"lower(`name`)"}},
- },
- })
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- tbl := mustGetTable(t, c, "t")
- if strings.EqualFold(tbl.Columns[0].DataType, "text") {
- t.Fatal("table degraded to pseudo — function-call index broke real install")
- }
- var found bool
- for _, idx := range tbl.Indexes {
- if idx.Name == "idx_lower_name" {
- found = true
- }
- }
- if !found {
- t.Error("idx_lower_name not installed")
- }
-}
-
-// TestLoader_FunctionalIndex_MixedWithBareColumns is a negative-regression
-// test: when one key part is an expression and another is a bare column, the
-// whole index must still install — all parts route through IndexColumns.
-func TestLoader_FunctionalIndex_MixedWithBareColumns(t *testing.T) {
- meta := schemaWithTables(&metadatapb.TableMetadata{
- Name: "t",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "int", Nullable: false, Default: autoIncrementSentinel},
- {Name: "tag", Type: "varchar(64)", Nullable: true},
- {Name: "name", Type: "varchar(255)", Nullable: true},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "PRIMARY", Type: "BTREE", Primary: true, Unique: true, Expressions: []string{"id"}},
- {
- Name: "idx_mixed",
- Type: "BTREE",
- Expressions: []string{"tag", "(lower(`name`))"},
- },
- },
- })
-
- c := newLoaderTestCatalog(t)
- runLoader(t, c, meta)
-
- tbl := mustGetTable(t, c, "t")
- if strings.EqualFold(tbl.Columns[0].DataType, "text") {
- t.Fatal("table degraded to pseudo — mixed index broke real install")
- }
-}
diff --git a/backend/plugin/schema/mysql/walk_through_test.go b/backend/plugin/schema/mysql/walk_through_test.go
index c2d2849f30..30aeb95f9c 100644
--- a/backend/plugin/schema/mysql/walk_through_test.go
+++ b/backend/plugin/schema/mysql/walk_through_test.go
@@ -1,6 +1,7 @@
package mysql
import (
+ "context"
"io"
"os"
"path/filepath"
@@ -58,7 +59,7 @@ func TestWalkThrough(t *testing.T) {
stmts, _ := sm.GetStatementsForChecks(storepb.Engine_MYSQL, test.Statement)
asts := base.ExtractASTs(stmts)
- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)
+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)
if advice != nil {
// Compare the advice fields
require.NotNil(t, test.Advice, "unexpected advice for statement %q: %+v", test.Statement, advice)
@@ -119,7 +120,7 @@ func TestWalkThroughCreateTableIfNotExistsCTASExistingTable(t *testing.T) {
require.True(t, ok)
require.True(t, createTable.IfNotExists)
- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: statement}, state, asts)
+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: statement}, state, asts)
require.Nil(t, advice)
require.NotNil(t, state.GetSchemaMetadata("").GetTable("t1"))
}
@@ -141,7 +142,7 @@ func TestWalkThroughSRIDInvisible(t *testing.T) {
stmts, _ := sm.GetStatementsForChecks(storepb.Engine_MYSQL, statement)
asts := base.ExtractASTs(stmts)
- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: statement}, state, asts)
+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: statement}, state, asts)
require.Nil(t, advice)
cols := map[string]*metadatapb.ColumnMetadata{}
@@ -164,7 +165,7 @@ func TestWalkThroughSRIDInvisible(t *testing.T) {
// TestWalkThroughSRIDInvisibleSeeded verifies the reverse leg of the WalkThroughWithContext
// round-trip: when the walk-through starts from already-synced metadata that carries SRID
-// / INVISIBLE columns, the proto->catalog seeding (wtBuildColumnDef) must install those
+// / INVISIBLE columns, LoadMetadata must install those
// attributes into the omni catalog so that running an UNRELATED DDL does not strip them
// from the resulting proto. Without seeding, the pre-existing columns load as
// plain/no-SRID and — because columnsEqual now compares these fields — surface as phantom
@@ -196,7 +197,7 @@ func TestWalkThroughSRIDInvisibleSeeded(t *testing.T) {
sm := sheet.NewManager()
stmts, _ := sm.GetStatementsForChecks(storepb.Engine_MYSQL, statement)
asts := base.ExtractASTs(stmts)
- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: statement}, state, asts)
+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: statement}, state, asts)
require.Nil(t, advice)
geo := state.GetSchemaMetadata("").GetTable("geo")
@@ -211,3 +212,45 @@ func TestWalkThroughSRIDInvisibleSeeded(t *testing.T) {
require.Equal(t, uint32(4326), *cols["pt"].Srid)
require.True(t, cols["secret"].IsInvisible, "seeded INVISIBLE must survive")
}
+
+func TestWalkThroughKeepsSeededIndexTypes(t *testing.T) {
+ origin := &metadatapb.DatabaseSchemaMetadata{
+ Name: "test",
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Tables: []*metadatapb.TableMetadata{
+ {
+ Name: "disk",
+ Engine: "InnoDB",
+ Columns: []*metadatapb.ColumnMetadata{{Name: "a", Position: 1, Type: "int"}},
+ Indexes: []*metadatapb.IndexMetadata{{Name: "k", Expressions: []string{"a"}, Type: "BTREE", Visible: true}},
+ },
+ {
+ Name: "mem",
+ Engine: "MEMORY",
+ Columns: []*metadatapb.ColumnMetadata{{Name: "a", Position: 1, Type: "int"}},
+ Indexes: []*metadatapb.IndexMetadata{
+ {Name: "k_hash", Expressions: []string{"a"}, Type: "HASH", Visible: true},
+ {Name: "k_btree", Expressions: []string{"a"}, Type: "BTREE", Visible: true},
+ },
+ },
+ },
+ }},
+ }
+ state := model.NewDatabaseMetadata(origin, nil, nil, storepb.Engine_MYSQL, true)
+
+ statement := "CREATE TABLE unrelated (x INT PRIMARY KEY);"
+ sm := sheet.NewManager()
+ stmts, _ := sm.GetStatementsForChecks(storepb.Engine_MYSQL, statement)
+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: statement}, state, base.ExtractASTs(stmts))
+ require.Nil(t, advice)
+
+ // Each index reports its effective access method, the engine's default one
+ // included.
+ types := map[string]string{}
+ for _, name := range []string{"disk", "mem"} {
+ for _, idx := range state.GetSchemaMetadata("").GetTable(name).GetProto().GetIndexes() {
+ types[name+"."+idx.GetName()] = idx.GetType()
+ }
+ }
+ require.Equal(t, map[string]string{"disk.k": "BTREE", "mem.k_hash": "HASH", "mem.k_btree": "BTREE"}, types)
+}
diff --git a/backend/plugin/schema/pg/walk_through.go b/backend/plugin/schema/pg/walk_through.go
index 6ab8e9a42f..d6f4400f69 100644
--- a/backend/plugin/schema/pg/walk_through.go
+++ b/backend/plugin/schema/pg/walk_through.go
@@ -23,30 +23,28 @@ func init() {
// WalkThrough walks through the PostgreSQL DDL and builds catalog metadata.
func WalkThrough(d *model.DatabaseMetadata, ast []base.AST) *storepb.Advice {
- return WalkThroughWithContext(schema.WalkThroughContext{}, d, ast)
+ return WalkThroughWithContext(context.Background(), schema.WalkThroughContext{}, d, ast)
}
// WalkThroughWithContext performs DDL simulation using the omni catalog.
-// 1. loadWalkThroughCatalog(metadata) → load existing schema into catalog
+// 1. catalog.LoadMetadata(metadata) → load existing schema into catalog
// 2. catalog.Exec(userSQL) → execute user DDL, collect per-statement Changes
// 3. Map exec errors → *storepb.Advice
// 4. Merge Changes into original metadata → FinalMetadata for downstream rules
-func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMetadata, _ []base.AST) *storepb.Advice {
- if ctx.RawSQL == "" {
+func WalkThroughWithContext(ctx context.Context, wtCtx schema.WalkThroughContext, d *model.DatabaseMetadata, _ []base.AST) *storepb.Advice {
+ if wtCtx.RawSQL == "" {
return nil
}
// Step 1: Load existing schema into two catalogs — one as snapshot, one for exec.
- // TODO(BYT-9215): Thread a real context.Context through the call chain instead of using context.Background().
- // WalkThroughContext is not a context.Context and the caller does not pass one.
catBefore := catalog.New()
- if ctx.SessionUser != "" {
- catBefore.SetSessionUser(ctx.SessionUser)
+ if wtCtx.SessionUser != "" {
+ catBefore.SetSessionUser(wtCtx.SessionUser)
}
if searchPath := getConfiguredSearchPath(d); len(searchPath) > 0 {
catBefore.SetSearchPath(searchPath)
}
- if err := loadWalkThroughCatalog(context.Background(), catBefore, d.GetProto()); err != nil {
+ if _, err := catBefore.LoadMetadata(ctx, d.GetProto(), catalog.LoadMetadataOptions{Full: true}); err != nil {
return &storepb.Advice{
Status: storepb.Advice_ERROR,
Code: code.DDLSimulationFailed.Int32(),
@@ -59,7 +57,7 @@ func WalkThroughWithContext(ctx schema.WalkThroughContext, d *model.DatabaseMeta
catAfter := catBefore.Clone()
// Step 2: Execute user SQL on catAfter.
- results, execErr := catAfter.Exec(ctx.RawSQL, &catalog.ExecOptions{ContinueOnError: true})
+ results, execErr := catAfter.Exec(wtCtx.RawSQL, &catalog.ExecOptions{ContinueOnError: true})
if execErr != nil {
return &storepb.Advice{
Status: storepb.Advice_ERROR,
diff --git a/backend/plugin/schema/pg/walk_through_apply.go b/backend/plugin/schema/pg/walk_through_apply.go
index 7277c5fe54..6e07544975 100644
--- a/backend/plugin/schema/pg/walk_through_apply.go
+++ b/backend/plugin/schema/pg/walk_through_apply.go
@@ -2,6 +2,7 @@ package pg
import (
"fmt"
+ "slices"
"strings"
"google.golang.org/protobuf/proto"
@@ -34,7 +35,11 @@ func applyDiffToMetadata(original *metadatapb.DatabaseSchemaMetadata, catBefore,
}
}
+ renamed := renamePartitions(result, catAfter, diff.Relations)
for _, rel := range diff.Relations {
+ if (rel.From != nil && renamed[rel.From.OID]) || (rel.To != nil && renamed[rel.To.OID]) {
+ continue
+ }
if rel.Action == catalog.DiffDrop {
if sm := findSchema(result, rel.SchemaName); sm != nil {
dropRelation(sm, rel)
@@ -220,6 +225,9 @@ func dropRelation(sm *metadatapb.SchemaMetadata, rel catalog.RelationDiffEntry)
sm.Tables = removeTableByName(sm.Tables, name)
sm.Views = removeViewByName(sm.Views, name)
sm.MaterializedViews = removeMatViewByName(sm.MaterializedViews, name)
+ for _, t := range sm.Tables {
+ t.Partitions = removePartitionByName(t.Partitions, name)
+ }
}
func modifyRelation(sm *metadatapb.SchemaMetadata, catBefore, cat *catalog.Catalog, rel catalog.RelationDiffEntry) {
@@ -230,6 +238,10 @@ func modifyRelation(sm *metadatapb.SchemaMetadata, catBefore, cat *catalog.Catal
case 'r', 'p', 'f':
tbl := findTable(sm, rel.Name)
if tbl == nil {
+ if p := findPartition(sm, rel.Name); p != nil {
+ applyPartitionDiffs(p, catBefore, cat, rel)
+ return
+ }
sm.Tables = append(sm.Tables, relationToTableProto(cat, rel.To))
return
}
@@ -246,6 +258,46 @@ func modifyRelation(sm *metadatapb.SchemaMetadata, catBefore, cat *catalog.Catal
}
}
+// renamePartitions renames each partition the diff renames where it stands,
+// with its indexes and constraints as the catalog now has them. The diff shows
+// a rename as a drop and an add of the same relation, which would otherwise
+// turn the partition into a table of its own. It returns the OIDs it handled.
+func renamePartitions(result *metadatapb.DatabaseSchemaMetadata, catAfter *catalog.Catalog, relations []catalog.RelationDiffEntry) map[uint32]bool {
+ added := make(map[uint32]catalog.RelationDiffEntry)
+ for _, rel := range relations {
+ if rel.Action == catalog.DiffAdd && rel.To != nil {
+ added[rel.To.OID] = rel
+ }
+ }
+ renamed := make(map[uint32]bool)
+ for _, rel := range relations {
+ if rel.Action != catalog.DiffDrop || rel.From == nil {
+ continue
+ }
+ add, ok := added[rel.From.OID]
+ sm := findSchema(result, rel.SchemaName)
+ if !ok || add.SchemaName != rel.SchemaName || sm == nil {
+ continue
+ }
+ if p := findPartition(sm, rel.Name); p != nil {
+ tbl := relationToTableProto(catAfter, add.To)
+ p.Name = add.Name
+ p.Indexes, p.CheckConstraints, p.ExcludeConstraints = tbl.Indexes, tbl.CheckConstraints, tbl.ExcludeConstraints
+ renamed[rel.From.OID] = true
+ }
+ }
+ return renamed
+}
+
+// applyPartitionDiffs applies a partition's index and constraint changes. A
+// partition's metadata holds only those; its columns are its parent table's.
+func applyPartitionDiffs(p *metadatapb.TablePartitionMetadata, catBefore, cat *catalog.Catalog, rel catalog.RelationDiffEntry) {
+ tbl := &metadatapb.TableMetadata{Indexes: p.Indexes, CheckConstraints: p.CheckConstraints, ExcludeConstraints: p.ExcludeConstraints}
+ applyIndexDiffs(tbl, cat, rel)
+ applyConstraintDiffs(tbl, catBefore, cat, rel)
+ p.Indexes, p.CheckConstraints, p.ExcludeConstraints = tbl.Indexes, tbl.CheckConstraints, tbl.ExcludeConstraints
+}
+
func applyColumnDiffs(tbl *metadatapb.TableMetadata, cat *catalog.Catalog, rel catalog.RelationDiffEntry) {
for _, cd := range rel.Columns {
switch cd.Action {
@@ -529,6 +581,37 @@ func findTable(sm *metadatapb.SchemaMetadata, name string) *metadatapb.TableMeta
return nil
}
+// findPartition returns the partition named name, at any depth, of a table in sm.
+func findPartition(sm *metadatapb.SchemaMetadata, name string) *metadatapb.TablePartitionMetadata {
+ var find func(partitions []*metadatapb.TablePartitionMetadata) *metadatapb.TablePartitionMetadata
+ find = func(partitions []*metadatapb.TablePartitionMetadata) *metadatapb.TablePartitionMetadata {
+ for _, p := range partitions {
+ if p.Name == name {
+ return p
+ }
+ if sub := find(p.Subpartitions); sub != nil {
+ return sub
+ }
+ }
+ return nil
+ }
+ for _, t := range sm.Tables {
+ if p := find(t.Partitions); p != nil {
+ return p
+ }
+ }
+ return nil
+}
+
+// removePartitionByName removes the partition named name, at any depth.
+func removePartitionByName(partitions []*metadatapb.TablePartitionMetadata, name string) []*metadatapb.TablePartitionMetadata {
+ partitions = slices.DeleteFunc(partitions, func(p *metadatapb.TablePartitionMetadata) bool { return p.Name == name })
+ for _, p := range partitions {
+ p.Subpartitions = removePartitionByName(p.Subpartitions, name)
+ }
+ return partitions
+}
+
func removeTableByName(tables []*metadatapb.TableMetadata, name string) []*metadatapb.TableMetadata {
out := make([]*metadatapb.TableMetadata, 0, len(tables))
for _, t := range tables {
@@ -719,7 +802,7 @@ func removeFunctionByIdentity(funcs []*metadatapb.FunctionMetadata, identity str
// metadata (the catalog does not track comments, and it only keeps the bare
// collation name) for attributes that survive by name. Attributes unchanged
// between from and to keep their previous metadata verbatim, so a composite
-// degraded to the text-backed pseudo fallback does not rewrite untouched
+// degraded to a text-backed stand-in does not rewrite untouched
// attributes to text — mirroring the per-column granularity of table diffs.
//
// Within a degraded composite this is a deliberate trade-off: an attribute
@@ -727,7 +810,7 @@ func removeFunctionByIdentity(funcs []*metadatapb.FunctionMetadata, identity str
// read text in from and to), so that rare change is missed in favor of not
// corrupting every untouched attribute. Walk-through metadata is advisory;
// post-execution sync restores ground truth. The root fix is modeling the
-// types the loader cannot install today (e.g. domains).
+// types LoadMetadata cannot install today (e.g. domains).
func compositeTypeToProto(cat *catalog.Catalog, name string, from, to *catalog.Relation, previous *metadatapb.CompositeTypeMetadata) *metadatapb.CompositeTypeMetadata {
previousAttributes := make(map[string]*metadatapb.CompositeTypeAttribute)
composite := &metadatapb.CompositeTypeMetadata{Name: name}
@@ -877,3 +960,11 @@ func quoteWalkThroughIdent(name string) string {
}
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
}
+
+func wtIsSystemSchema(s string) bool {
+ switch s {
+ case "pg_catalog", "pg_toast", "information_schema":
+ return true
+ }
+ return strings.HasPrefix(s, "pg_")
+}
diff --git a/backend/plugin/schema/pg/walk_through_loader_test.go b/backend/plugin/schema/pg/walk_through_apply_test.go
similarity index 68%
rename from backend/plugin/schema/pg/walk_through_loader_test.go
rename to backend/plugin/schema/pg/walk_through_apply_test.go
index f11c673063..7e9afb0b28 100644
--- a/backend/plugin/schema/pg/walk_through_loader_test.go
+++ b/backend/plugin/schema/pg/walk_through_apply_test.go
@@ -9,59 +9,9 @@ import (
"github.com/stretchr/testify/require"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/schema"
"github.com/bytebase/bytebase/backend/store/model"
)
-// TestClone_BasicIsolation verifies that Clone produces an independent catalog:
-// mutations on the clone do not affect the original.
-func TestClone_BasicIsolation(t *testing.T) {
- original := catalog.New()
- _, err := original.Exec(`
- CREATE SCHEMA test_schema;
- CREATE TABLE test_schema.t1 (id serial PRIMARY KEY, name text NOT NULL);
- CREATE INDEX t1_name_idx ON test_schema.t1 (name);
- `, nil)
- require.NoError(t, err)
-
- clone := original.Clone()
-
- // Mutate clone
- _, err = clone.Exec(`
- ALTER TABLE test_schema.t1 ADD COLUMN email text;
- CREATE TABLE test_schema.t2 (id int PRIMARY KEY);
- DROP INDEX test_schema.t1_name_idx;
- `, nil)
- require.NoError(t, err)
-
- // Verify original is untouched
- origRel := original.GetRelation("test_schema", "t1")
- require.NotNil(t, origRel, "original t1 should still exist")
- require.Equal(t, 2, len(origRel.Columns), "original t1 should have 2 columns (id, name), not 3")
-
- require.Nil(t, original.GetRelation("test_schema", "t2"), "original should NOT have t2")
-
- origIndexes := original.IndexesOf(origRel.OID)
- found := false
- for _, idx := range origIndexes {
- if idx.Name == "t1_name_idx" {
- found = true
- }
- }
- require.True(t, found, "original should still have t1_name_idx")
-
- // Verify clone has the changes
- cloneRel := clone.GetRelation("test_schema", "t1")
- require.NotNil(t, cloneRel)
- require.Equal(t, 3, len(cloneRel.Columns), "clone t1 should have 3 columns")
- require.NotNil(t, clone.GetRelation("test_schema", "t2"), "clone should have t2")
-
- // Diff should show changes
- diff := catalog.Diff(original, clone)
- require.False(t, diff.IsEmpty(), "diff should not be empty")
- require.Greater(t, len(diff.Relations), 0, "should have relation diffs")
-}
-
// TestClone_WalkThroughIntegration tests the full walk-through flow using Clone:
// load catalog → clone → exec user DDL on clone → diff → apply.
func TestClone_WalkThroughIntegration(t *testing.T) {
@@ -87,7 +37,7 @@ func TestClone_WalkThroughIntegration(t *testing.T) {
}
catBefore := catalog.New()
- err := loadWalkThroughCatalog(context.Background(), catBefore, meta)
+ _, err := catBefore.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})
require.NoError(t, err)
catAfter := catBefore.Clone()
@@ -132,87 +82,49 @@ func TestClone_WalkThroughIntegration(t *testing.T) {
require.Nil(t, origMeta.GetSchemaMetadata("public").GetTable("posts"), "original should not have posts")
}
-// TestClone_SearchPath tests Clone preserves search path and session user.
-func TestClone_SearchPath(t *testing.T) {
+func TestWalkThroughKeepsPartitionsUnderTheirTable(t *testing.T) {
meta := &metadatapb.DatabaseSchemaMetadata{
Name: "postgres",
- Schemas: []*metadatapb.SchemaMetadata{
- {Name: "public"},
- {Name: "alice"},
- },
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{{
+ Name: "orders",
+ Columns: []*metadatapb.ColumnMetadata{{Name: "id", Type: "integer"}},
+ Partitions: []*metadatapb.TablePartitionMetadata{
+ {Name: "orders_2023"},
+ {Name: "orders_2024", Subpartitions: []*metadatapb.TablePartitionMetadata{{Name: "orders_2024_q1"}, {Name: "orders_2024_q2"}}},
+ },
+ }},
+ }},
}
-
catBefore := catalog.New()
- catBefore.SetSessionUser("alice")
- catBefore.SetSearchPath([]string{"$user", "public"})
- err := loadWalkThroughCatalog(context.Background(), catBefore, meta)
+ _, err := catBefore.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})
require.NoError(t, err)
-
catAfter := catBefore.Clone()
-
- // CREATE TABLE without schema should use search path ($user → alice)
- _, err = catAfter.Exec(`CREATE TABLE my_table (id int);`, nil)
+ results, err := catAfter.Exec(`
+ DROP TABLE public.orders_2023;
+ DROP TABLE public.orders_2024_q2;
+ CREATE INDEX orders_2024_q1_id_idx ON public.orders_2024_q1 (id);
+ ALTER TABLE public.orders_2024 RENAME TO orders_2024_archived;
+ CREATE INDEX orders_2024_archived_id_idx ON public.orders_2024_archived (id);
+ `, &catalog.ExecOptions{ContinueOnError: true})
require.NoError(t, err)
-
- // Table should be in alice schema on clone
- require.NotNil(t, catAfter.GetRelation("alice", "my_table"))
-
- // Original should NOT have it
- require.Nil(t, catBefore.GetRelation("alice", "my_table"))
-
- // Diff should show the new table
- diff := catalog.Diff(catBefore, catAfter)
- require.False(t, diff.IsEmpty())
-
- foundAliceTable := false
- for _, rel := range diff.Relations {
- if rel.SchemaName == "alice" && rel.Name == "my_table" && rel.Action == catalog.DiffAdd {
- foundAliceTable = true
- }
- }
- require.True(t, foundAliceTable, "diff should show alice.my_table as added")
-}
-
-// TestClone_WalkThroughFunction tests the actual WalkThroughWithContext function
-// using Clone (if enabled) produces correct FinalMetadata.
-func TestClone_WalkThroughFunction(t *testing.T) {
- meta := &metadatapb.DatabaseSchemaMetadata{
- Name: "postgres",
- Schemas: []*metadatapb.SchemaMetadata{
- {
- Name: "public",
- Tables: []*metadatapb.TableMetadata{
- {
- Name: "test",
- Columns: []*metadatapb.ColumnMetadata{
- {Name: "id", Type: "integer", Position: 1},
- {Name: "name", Type: "text", Position: 2, Nullable: true},
- },
- Indexes: []*metadatapb.IndexMetadata{
- {Name: "test_pkey", Expressions: []string{"id"}, Unique: true, Primary: true},
- },
- },
- },
- },
- },
- }
-
- state := model.NewDatabaseMetadata(meta, nil, nil, storepb.Engine_POSTGRES, true)
- ctx := schema.WalkThroughContext{
- RawSQL: `CREATE TABLE public.new_table (id int PRIMARY KEY, val text);`,
+ for _, r := range results {
+ require.NoError(t, r.Error, r.SQL)
}
- advice := WalkThroughWithContext(ctx, state, nil)
- require.Nil(t, advice, "walk-through should succeed")
-
- // Check FinalMetadata has the new table
- newTbl := state.GetSchemaMetadata("public").GetTable("new_table")
- require.NotNil(t, newTbl, "new_table should exist in FinalMetadata")
-
- // Check original table is preserved
- origTbl := state.GetSchemaMetadata("public").GetTable("test")
- require.NotNil(t, origTbl, "test table should still exist")
- require.Equal(t, 2, len(origTbl.GetProto().Columns))
+ schema := applyDiffToMetadata(meta, catBefore, catAfter, catalog.Diff(catBefore, catAfter)).Schemas[0]
+ require.Len(t, schema.Tables, 1, "a changed partition must stay under its table")
+ partitions := schema.Tables[0].Partitions
+ require.Len(t, partitions, 1)
+ require.Equal(t, "orders_2024_archived", partitions[0].Name)
+ require.Len(t, partitions[0].Indexes, 1)
+ require.Equal(t, "orders_2024_archived_id_idx", partitions[0].Indexes[0].Name)
+ require.Len(t, partitions[0].Subpartitions, 1)
+ q1 := partitions[0].Subpartitions[0]
+ require.Equal(t, "orders_2024_q1", q1.Name)
+ require.Len(t, q1.Indexes, 1)
+ require.Equal(t, "orders_2024_q1_id_idx", q1.Indexes[0].Name)
}
// compositeWalkThroughMetadata has an adversarial name pair (aa_nested sorts
@@ -271,25 +183,10 @@ func compositeWalkThroughMetadata() *metadatapb.DatabaseSchemaMetadata {
}
}
-func TestWalkThroughLoadsCompositeTypes(t *testing.T) {
- meta := compositeWalkThroughMetadata()
-
- cat := catalog.New()
- err := loadWalkThroughCatalog(context.Background(), cat, meta)
- require.NoError(t, err)
-
- require.NotNil(t, cat.GetRelation("public", "zz_base"), "composite type must be installed")
- require.NotNil(t, cat.GetRelation("public", "aa_nested"), "nested composite type must be installed")
- arrayOnly := cat.GetRelation("public", "aa_array_only")
- require.NotNil(t, arrayOnly, "array-only referencing composite must be installed")
- require.Len(t, arrayOnly.Columns, 1, "array-only composite must install with its real attribute")
- require.NotNil(t, cat.GetRelation("public", "users"), "table using a composite column must install")
-}
-
func TestWalkThroughCompositeFallbackPreservesAttributeNames(t *testing.T) {
- // A domain-typed attribute cannot install (domains are not loader
- // objects), forcing the pseudo fallback — which must keep attribute
- // names so later DDL targeting them still resolves.
+ // LoadMetadata does not install domains, so a domain-typed attribute makes
+ // the composite stand in — which must keep attribute names so later DDL
+ // targeting them still resolves.
meta := &metadatapb.DatabaseSchemaMetadata{
Schemas: []*metadatapb.SchemaMetadata{
{
@@ -308,7 +205,7 @@ func TestWalkThroughCompositeFallbackPreservesAttributeNames(t *testing.T) {
}
cat := catalog.New()
- err := loadWalkThroughCatalog(context.Background(), cat, meta)
+ _, err := cat.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})
require.NoError(t, err)
rel := cat.GetRelation("public", "with_domain")
@@ -326,7 +223,7 @@ func TestWalkThroughCompositeFallbackPreservesAttributeNames(t *testing.T) {
}
// A rename keeps the attribute number, so the renamed attribute of a
- // degraded composite must also keep its real previous type.
+ // stood-in composite must also keep its real previous type.
renameResults, err := catAfter.Exec(`ALTER TYPE public.with_domain RENAME ATTRIBUTE p TO q;`, &catalog.ExecOptions{ContinueOnError: true})
require.NoError(t, err)
for _, r := range renameResults {
@@ -373,7 +270,7 @@ func TestWalkThroughDropReaddAttributeReadsCatalogType(t *testing.T) {
}
cat := catalog.New()
- err := loadWalkThroughCatalog(context.Background(), cat, meta)
+ _, err := cat.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})
require.NoError(t, err)
catAfter := cat.Clone()
@@ -412,7 +309,7 @@ func TestWalkThroughAppliesCompositeTypeChanges(t *testing.T) {
meta := compositeWalkThroughMetadata()
catBefore := catalog.New()
- err := loadWalkThroughCatalog(context.Background(), catBefore, meta)
+ _, err := catBefore.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})
require.NoError(t, err)
catAfter := catBefore.Clone()
diff --git a/backend/plugin/schema/pg/walk_through_loader.go b/backend/plugin/schema/pg/walk_through_loader.go
deleted file mode 100644
index aa506e661a..0000000000
--- a/backend/plugin/schema/pg/walk_through_loader.go
+++ /dev/null
@@ -1,1497 +0,0 @@
-//nolint:unused
-package pg
-
-import (
- "cmp"
- "context"
- "fmt"
- "regexp"
- "slices"
- "strings"
-
- "github.com/pkg/errors"
-
- metadatapb "github.com/bytebase/omni/metadata"
- "github.com/bytebase/omni/pg/ast"
- "github.com/bytebase/omni/pg/catalog"
- omniparser "github.com/bytebase/omni/pg/parser"
-)
-
-// loadWalkThroughCatalog installs every schema object from DatabaseSchemaMetadata
-// into the omni catalog in dependency-topological order with pseudo fallback.
-// Indexes and constraints are installed after their parent tables/views.
-func loadWalkThroughCatalog(ctx context.Context, cat *catalog.Catalog, meta *metadatapb.DatabaseSchemaMetadata) error {
- if cat == nil {
- return errors.New("loadWalkThroughCatalog: nil catalog")
- }
- if meta == nil {
- return nil
- }
-
- objects := wtCollectObjects(meta)
- sorted := wtTopoSort(objects)
- for _, obj := range sorted {
- if err := ctx.Err(); err != nil {
- return err
- }
- wtInstallOne(cat, obj)
- }
- return nil
-}
-
-// wtObjectKind identifies the kind of a walk-through loader object.
-type wtObjectKind int
-
-const (
- kindWTSchema wtObjectKind = iota
- kindWTEnum
- kindWTComposite
- kindWTSequence
- kindWTTable
- kindWTView
- kindWTMatView
- kindWTFunction
- kindWTIndex
- kindWTConstraint
-)
-
-// wtObjectEntry is one unit of work for the walk-through loader.
-type wtObjectEntry struct {
- kind wtObjectKind
- schema string
- name string
-
- // Exactly one of the following is set based on kind.
- enumMeta *metadatapb.EnumTypeMetadata
- compositeMeta *metadatapb.CompositeTypeMetadata
- seqMeta *metadatapb.SequenceMetadata
- tableMeta *metadatapb.TableMetadata
- viewMeta *metadatapb.ViewMetadata
- matViewMeta *metadatapb.MaterializedViewMetadata
- funcMeta *metadatapb.FunctionMetadata
- idxMeta *metadatapb.IndexMetadata
- // For kindWTConstraint: the parent table metadata.
- constraintTable *metadatapb.TableMetadata
- // For kindWTConstraint, kindWTIndex: the parent relation name.
- parentName string
-}
-
-func (e *wtObjectEntry) key() string {
- switch e.kind {
- case kindWTSchema:
- return "schema:" + e.schema
- case kindWTEnum, kindWTComposite:
- return "type:" + e.schema + "." + e.name
- case kindWTSequence:
- return "seq:" + e.schema + "." + e.name
- case kindWTTable, kindWTView, kindWTMatView:
- return "rel:" + e.schema + "." + e.name
- case kindWTFunction:
- return "func:" + e.schema + "." + e.name + "|" + wtFuncSigKey(e.funcMeta)
- case kindWTIndex:
- return "idx:" + e.schema + "." + e.parentName + "." + e.name
- case kindWTConstraint:
- return "con:" + e.schema + "." + e.parentName + "." + e.name
- }
- return "unknown:" + e.schema + "." + e.name
-}
-
-func (e *wtObjectEntry) sortKey() string {
- base := e.schema + "\x00" + e.name + "\x00" + wtKindLabel(e.kind)
- if e.kind == kindWTFunction {
- base += "\x00" + wtFuncSigKey(e.funcMeta)
- }
- if e.kind == kindWTIndex || e.kind == kindWTConstraint {
- base += "\x00" + e.parentName
- }
- return base
-}
-
-func wtFuncSigKey(fn *metadatapb.FunctionMetadata) string {
- if fn == nil {
- return ""
- }
- if fn.Signature != "" {
- return fn.Signature
- }
- return fn.Definition
-}
-
-func wtKindLabel(k wtObjectKind) string {
- switch k {
- case kindWTSchema:
- return "0schema"
- case kindWTEnum:
- return "1enum"
- case kindWTComposite:
- return "1composite"
- case kindWTSequence:
- return "2seq"
- case kindWTTable:
- return "3table"
- case kindWTView:
- return "4view"
- case kindWTMatView:
- return "5matview"
- case kindWTFunction:
- return "6function"
- case kindWTIndex:
- return "7index"
- case kindWTConstraint:
- return "8constraint"
- }
- return "9unknown"
-}
-
-// wtCollectObjects flattens DatabaseSchemaMetadata into wtObjectEntry values.
-func wtCollectObjects(meta *metadatapb.DatabaseSchemaMetadata) []*wtObjectEntry {
- var out []*wtObjectEntry
- for _, sm := range meta.Schemas {
- if sm.Name == "" {
- continue
- }
- out = append(out, &wtObjectEntry{
- kind: kindWTSchema,
- schema: sm.Name,
- name: sm.Name,
- })
- for _, enum := range sm.EnumTypes {
- out = append(out, &wtObjectEntry{
- kind: kindWTEnum,
- schema: sm.Name,
- name: enum.Name,
- enumMeta: enum,
- })
- }
- for _, composite := range sm.CompositeTypes {
- out = append(out, &wtObjectEntry{
- kind: kindWTComposite,
- schema: sm.Name,
- name: composite.Name,
- compositeMeta: composite,
- })
- }
- for _, seq := range sm.Sequences {
- if wtIsIdentitySequence(seq, sm.Tables) {
- continue
- }
- out = append(out, &wtObjectEntry{
- kind: kindWTSequence,
- schema: sm.Name,
- name: seq.Name,
- seqMeta: seq,
- })
- }
- for _, tbl := range sm.Tables {
- out = append(out, &wtObjectEntry{
- kind: kindWTTable,
- schema: sm.Name,
- name: tbl.Name,
- tableMeta: tbl,
- })
- for _, idx := range tbl.Indexes {
- out = append(out, &wtObjectEntry{
- kind: kindWTIndex,
- schema: sm.Name,
- name: idx.Name,
- parentName: tbl.Name,
- idxMeta: idx,
- })
- }
- for _, fk := range tbl.ForeignKeys {
- out = append(out, &wtObjectEntry{
- kind: kindWTConstraint,
- schema: sm.Name,
- name: fk.Name,
- parentName: tbl.Name,
- constraintTable: tbl,
- })
- }
- for _, chk := range tbl.CheckConstraints {
- out = append(out, &wtObjectEntry{
- kind: kindWTConstraint,
- schema: sm.Name,
- name: chk.Name,
- parentName: tbl.Name,
- constraintTable: tbl,
- })
- }
- for _, exc := range tbl.ExcludeConstraints {
- out = append(out, &wtObjectEntry{
- kind: kindWTConstraint,
- schema: sm.Name,
- name: exc.Name,
- parentName: tbl.Name,
- constraintTable: tbl,
- })
- }
- }
- for _, view := range sm.Views {
- out = append(out, &wtObjectEntry{
- kind: kindWTView,
- schema: sm.Name,
- name: view.Name,
- viewMeta: view,
- })
- }
- for _, mv := range sm.MaterializedViews {
- out = append(out, &wtObjectEntry{
- kind: kindWTMatView,
- schema: sm.Name,
- name: mv.Name,
- matViewMeta: mv,
- })
- for _, idx := range mv.Indexes {
- out = append(out, &wtObjectEntry{
- kind: kindWTIndex,
- schema: sm.Name,
- name: idx.Name,
- parentName: mv.Name,
- idxMeta: idx,
- })
- }
- }
- for _, fn := range sm.Functions {
- out = append(out, &wtObjectEntry{
- kind: kindWTFunction,
- schema: sm.Name,
- name: fn.Name,
- funcMeta: fn,
- })
- }
- for _, proc := range sm.Procedures {
- out = append(out, &wtObjectEntry{
- kind: kindWTFunction,
- schema: sm.Name,
- name: proc.Name,
- funcMeta: &metadatapb.FunctionMetadata{
- Name: proc.Name,
- Definition: proc.Definition,
- Signature: proc.Signature,
- },
- })
- }
- }
- return out
-}
-
-// wtTopoSort orders objects by dependency using Tarjan SCC.
-func wtTopoSort(objects []*wtObjectEntry) []*wtObjectEntry {
- if len(objects) == 0 {
- return nil
- }
- index := make(map[string]*wtObjectEntry, len(objects))
- for _, obj := range objects {
- index[obj.key()] = obj
- }
-
- edges := wtBuildEdges(objects, index)
- sccs := wtTarjanSCC(objects, edges)
-
- sccOf := make(map[string]int, len(objects))
- for i, scc := range sccs {
- for _, obj := range scc {
- sccOf[obj.key()] = i
- }
- }
-
- condensedEdges := make([][]int, len(sccs))
- inDegree := make([]int, len(sccs))
- seenEdge := make(map[[2]int]bool)
- for src, dsts := range edges {
- srcSCC, ok := sccOf[src]
- if !ok {
- continue
- }
- for _, dst := range dsts {
- dstSCC, ok2 := sccOf[dst]
- if !ok2 || dstSCC == srcSCC {
- continue
- }
- edge := [2]int{dstSCC, srcSCC}
- if seenEdge[edge] {
- continue
- }
- seenEdge[edge] = true
- condensedEdges[dstSCC] = append(condensedEdges[dstSCC], srcSCC)
- inDegree[srcSCC]++
- }
- }
-
- ready := make([]int, 0)
- for i := range sccs {
- if inDegree[i] == 0 {
- ready = append(ready, i)
- }
- }
- wtSortSCCsByMin(ready, sccs)
-
- var flat []*wtObjectEntry
- for len(ready) > 0 {
- next := ready[0]
- ready = ready[1:]
- flat = append(flat, wtSortedSCCMembers(sccs[next])...)
- for _, nb := range condensedEdges[next] {
- inDegree[nb]--
- if inDegree[nb] == 0 {
- ready = append(ready, nb)
- }
- }
- wtSortSCCsByMin(ready, sccs)
- }
-
- if len(flat) != len(objects) {
- emitted := make(map[string]bool, len(flat))
- for _, o := range flat {
- emitted[o.key()] = true
- }
- var missed []*wtObjectEntry
- for _, o := range objects {
- if !emitted[o.key()] {
- missed = append(missed, o)
- }
- }
- slices.SortStableFunc(missed, func(a, b *wtObjectEntry) int {
- return cmp.Compare(a.sortKey(), b.sortKey())
- })
- flat = append(flat, missed...)
- }
- return flat
-}
-
-// wtBuildEdges computes dependency edges for walk-through objects.
-func wtBuildEdges(objects []*wtObjectEntry, index map[string]*wtObjectEntry) map[string][]string {
- edges := make(map[string][]string, len(objects))
- for _, obj := range objects {
- var deps []string
-
- // Every non-schema object depends on its schema.
- if obj.kind != kindWTSchema {
- if key := "schema:" + obj.schema; index[key] != nil {
- deps = append(deps, key)
- }
- }
-
- switch obj.kind {
- case kindWTTable:
- for _, col := range obj.tableMeta.Columns {
- for _, ref := range wtExtractUserTypeRefs(col.Type) {
- if k := "type:" + ref.Schema + "." + ref.Name; index[k] != nil {
- deps = append(deps, k)
- }
- }
- // Sequence dependency from nextval('schema.seq'::regclass).
- if col.Default != "" {
- if seqKey := wtExtractSeqRef(col.Default, obj.schema, index); seqKey != "" {
- deps = append(deps, seqKey)
- }
- }
- }
- case kindWTView:
- for _, dep := range obj.viewMeta.DependencyColumns {
- if k := "rel:" + dep.Schema + "." + dep.Table; index[k] != nil {
- deps = append(deps, k)
- }
- }
- case kindWTMatView:
- for _, dep := range obj.matViewMeta.DependencyColumns {
- if k := "rel:" + dep.Schema + "." + dep.Table; index[k] != nil {
- deps = append(deps, k)
- }
- }
- case kindWTFunction:
- argTypes, _ := wtParseFuncArgTypes(obj.funcMeta.Signature)
- for _, at := range argTypes {
- for _, ref := range wtExtractUserTypeRefs(at) {
- if k := "type:" + ref.Schema + "." + ref.Name; index[k] != nil {
- deps = append(deps, k)
- }
- }
- }
- case kindWTIndex:
- if k := "rel:" + obj.schema + "." + obj.parentName; index[k] != nil {
- deps = append(deps, k)
- }
- case kindWTConstraint:
- if k := "rel:" + obj.schema + "." + obj.parentName; index[k] != nil {
- deps = append(deps, k)
- }
- if obj.constraintTable != nil {
- for _, fk := range obj.constraintTable.ForeignKeys {
- if fk.Name == obj.name {
- refSchema := fk.ReferencedSchema
- if refSchema == "" {
- refSchema = obj.schema
- }
- if k := "rel:" + refSchema + "." + fk.ReferencedTable; index[k] != nil {
- deps = append(deps, k)
- }
- // FK depends on referenced table's PK/Unique indexes.
- for _, o := range objects {
- if o.kind == kindWTIndex && o.schema == refSchema &&
- o.parentName == fk.ReferencedTable &&
- (o.idxMeta.Primary || (o.idxMeta.IsConstraint && o.idxMeta.Unique)) {
- if index[o.key()] != nil {
- deps = append(deps, o.key())
- }
- }
- }
- break
- }
- }
- }
- case kindWTComposite:
- for _, attribute := range obj.compositeMeta.Attributes {
- for _, ref := range wtExtractUserTypeRefs(attribute.Type) {
- if k := "type:" + ref.Schema + "." + ref.Name; index[k] != nil {
- deps = append(deps, k)
- }
- // An attribute may be typed with a table/view row type.
- if k := "rel:" + ref.Schema + "." + ref.Name; index[k] != nil {
- deps = append(deps, k)
- }
- }
- }
- default:
- // kindWTSchema, kindWTEnum, kindWTSequence have no additional deps.
- }
-
- if len(deps) > 0 {
- edges[obj.key()] = wtDedupStrings(deps)
- }
- }
- return edges
-}
-
-// wtTarjanSCC runs Tarjan's SCC algorithm over walk-through objects.
-func wtTarjanSCC(objects []*wtObjectEntry, edges map[string][]string) [][]*wtObjectEntry {
- type state struct {
- index, low int
- onStack bool
- }
- st := make(map[string]*state, len(objects))
- byKey := make(map[string]*wtObjectEntry, len(objects))
- for _, obj := range objects {
- byKey[obj.key()] = obj
- }
-
- keys := make([]string, 0, len(objects))
- for _, obj := range objects {
- keys = append(keys, obj.key())
- }
- slices.Sort(keys)
-
- var (
- idx int
- stack []string
- result [][]*wtObjectEntry
- )
-
- var strongconnect func(v string)
- strongconnect = func(v string) {
- st[v] = &state{index: idx, low: idx, onStack: true}
- idx++
- stack = append(stack, v)
- for _, w := range edges[v] {
- if _, ok := byKey[w]; !ok {
- continue
- }
- if _, seen := st[w]; !seen {
- strongconnect(w)
- st[v].low = min(st[v].low, st[w].low)
- } else if st[w].onStack {
- st[v].low = min(st[v].low, st[w].index)
- }
- }
- if st[v].low == st[v].index {
- var scc []*wtObjectEntry
- for {
- w := stack[len(stack)-1]
- stack = stack[:len(stack)-1]
- st[w].onStack = false
- scc = append(scc, byKey[w])
- if w == v {
- break
- }
- }
- result = append(result, scc)
- }
- }
-
- for _, k := range keys {
- if _, seen := st[k]; !seen {
- strongconnect(k)
- }
- }
- return result
-}
-
-func wtSortedSCCMembers(scc []*wtObjectEntry) []*wtObjectEntry {
- out := make([]*wtObjectEntry, len(scc))
- copy(out, scc)
- slices.SortStableFunc(out, func(a, b *wtObjectEntry) int {
- return cmp.Compare(a.sortKey(), b.sortKey())
- })
- return out
-}
-
-func wtSortSCCsByMin(indices []int, sccs [][]*wtObjectEntry) {
- slices.SortStableFunc(indices, func(a, b int) int {
- return cmp.Compare(wtMinSortKey(sccs[a]), wtMinSortKey(sccs[b]))
- })
-}
-
-func wtMinSortKey(scc []*wtObjectEntry) string {
- if len(scc) == 0 {
- return ""
- }
- best := scc[0].sortKey()
- for _, o := range scc[1:] {
- if k := o.sortKey(); k < best {
- best = k
- }
- }
- return best
-}
-
-func wtDedupStrings(in []string) []string {
- if len(in) < 2 {
- return in
- }
- seen := make(map[string]bool, len(in))
- out := in[:0]
- for _, s := range in {
- if !seen[s] {
- seen[s] = true
- out = append(out, s)
- }
- }
- return out
-}
-
-// wtInstallOne attempts real install, falls back to pseudo for tables/views/matviews.
-func wtInstallOne(cat *catalog.Catalog, obj *wtObjectEntry) {
- if err := wtInstallReal(cat, obj); err == nil {
- return
- }
- // Only pseudo-fallback for relations; other kinds are left uninstalled on failure.
- switch obj.kind {
- case kindWTTable:
- cols := wtColNames(obj.tableMeta)
- _ = cat.DefineRelation(wtPseudoTableStmt(obj.schema, obj.name, cols), 'r')
- case kindWTView:
- cols := wtViewColNames(obj.viewMeta)
- if stmt, err := wtPseudoViewStmt(obj.schema, obj.name, cols); err == nil {
- _ = cat.DefineView(stmt)
- }
- case kindWTMatView:
- cols := wtMatViewColNames(obj.matViewMeta)
- if stmt, err := wtPseudoMatViewStmt(obj.schema, obj.name, cols); err == nil {
- _ = cat.ExecCreateTableAs(stmt)
- }
- case kindWTComposite:
- // Name-preserving, text-backed fallback (mirroring the table pseudo)
- // so later DDL targeting an existing attribute still resolves.
- items := make([]ast.Node, 0, len(obj.compositeMeta.Attributes))
- for _, attribute := range obj.compositeMeta.Attributes {
- if attribute.Name == "" {
- continue
- }
- items = append(items, &ast.ColumnDef{
- Colname: attribute.Name,
- TypeName: wtPseudoTextTypeName(),
- })
- }
- _ = cat.DefineCompositeType(&ast.CompositeTypeStmt{
- Typevar: &ast.RangeVar{Schemaname: obj.schema, Relname: obj.name},
- Coldeflist: &ast.List{Items: items},
- })
- default:
- // No pseudo form for schemas, enums, sequences, functions, indexes, constraints.
- }
-}
-
-// wtInstallReal performs the real catalog install for one object.
-func wtInstallReal(cat *catalog.Catalog, obj *wtObjectEntry) error {
- switch obj.kind {
- case kindWTSchema:
- err := cat.CreateSchemaCommand(&ast.CreateSchemaStmt{Schemaname: obj.schema})
- var cErr *catalog.Error
- if errors.As(err, &cErr) && cErr.Code == catalog.CodeDuplicateSchema {
- return nil
- }
- return err
-
- case kindWTEnum:
- vals := make([]ast.Node, 0, len(obj.enumMeta.Values))
- for _, v := range obj.enumMeta.Values {
- vals = append(vals, &ast.String{Str: v})
- }
- return cat.DefineEnum(&ast.CreateEnumStmt{
- TypeName: wtQualifiedList(obj.schema, obj.name),
- Vals: &ast.List{Items: vals},
- })
-
- case kindWTComposite:
- cols := make([]ast.Node, 0, len(obj.compositeMeta.Attributes))
- for _, attribute := range obj.compositeMeta.Attributes {
- if attribute.Name == "" {
- continue
- }
- tn, err := wtTypeNameFromString(attribute.Type)
- if err != nil {
- return errors.Wrapf(err, "attribute %q", attribute.Name)
- }
- cols = append(cols, &ast.ColumnDef{
- Colname: attribute.Name,
- TypeName: tn,
- CollClause: collateClauseFromReference(attribute.Collation),
- })
- }
- return cat.DefineCompositeType(&ast.CompositeTypeStmt{
- Typevar: &ast.RangeVar{Schemaname: obj.schema, Relname: obj.name},
- Coldeflist: &ast.List{Items: cols},
- })
-
- case kindWTSequence:
- return wtInstallSequence(cat, obj)
-
- case kindWTTable:
- return wtInstallTable(cat, obj)
-
- case kindWTView:
- return wtInstallView(cat, obj)
-
- case kindWTMatView:
- return wtInstallMatView(cat, obj)
-
- case kindWTFunction:
- return wtInstallFunction(cat, obj)
-
- case kindWTIndex:
- if obj.idxMeta.Primary {
- return cat.AddConstraint(obj.schema, obj.parentName, catalog.ConstraintDef{
- Name: obj.idxMeta.Name,
- Type: catalog.ConstraintPK,
- Columns: wtUnquoteColumns(obj.idxMeta.Expressions),
- })
- }
- if obj.idxMeta.IsConstraint && obj.idxMeta.Unique {
- return cat.AddConstraint(obj.schema, obj.parentName, catalog.ConstraintDef{
- Name: obj.idxMeta.Name,
- Type: catalog.ConstraintUnique,
- Columns: wtUnquoteColumns(obj.idxMeta.Expressions),
- })
- }
- return wtInstallIndex(cat, obj)
-
- case kindWTConstraint:
- return wtInstallConstraint(cat, obj)
- }
- return errors.Errorf("unknown object kind %d", obj.kind)
-}
-
-func wtInstallSequence(cat *catalog.Catalog, obj *wtObjectEntry) error {
- seq := obj.seqMeta
- if seq == nil {
- return errors.New("sequence has no metadata")
- }
-
- stmt := &ast.CreateSeqStmt{
- Sequence: &ast.RangeVar{
- Schemaname: obj.schema,
- Relname: seq.Name,
- Relpersistence: 'p',
- },
- }
-
- var opts []ast.Node
- if seq.Increment != "" && seq.Increment != "0" {
- opts = append(opts, &ast.DefElem{Defname: "increment", Arg: &ast.Integer{Ival: wtParseInt64(seq.Increment)}})
- }
- if seq.MinValue != "" && seq.MinValue != "0" {
- opts = append(opts, &ast.DefElem{Defname: "minvalue", Arg: &ast.Integer{Ival: wtParseInt64(seq.MinValue)}})
- }
- if seq.MaxValue != "" && seq.MaxValue != "0" {
- opts = append(opts, &ast.DefElem{Defname: "maxvalue", Arg: &ast.Integer{Ival: wtParseInt64(seq.MaxValue)}})
- }
- if seq.Start != "" && seq.Start != "0" {
- opts = append(opts, &ast.DefElem{Defname: "start", Arg: &ast.Integer{Ival: wtParseInt64(seq.Start)}})
- }
- if seq.CacheSize != "" && seq.CacheSize != "0" {
- opts = append(opts, &ast.DefElem{Defname: "cache", Arg: &ast.Integer{Ival: wtParseInt64(seq.CacheSize)}})
- }
- if seq.Cycle {
- opts = append(opts, &ast.DefElem{Defname: "cycle", Arg: &ast.Boolean{Boolval: true}})
- }
- if len(opts) > 0 {
- stmt.Options = &ast.List{Items: opts}
- }
-
- return cat.DefineSequence(stmt)
-}
-
-// wtIsIdentitySequence checks if a sequence is owned by an identity column.
-// Identity columns auto-create their sequence during DefineRelation, so
-// pre-creating it would cause a duplicate error.
-func wtIsIdentitySequence(seq *metadatapb.SequenceMetadata, tables []*metadatapb.TableMetadata) bool {
- if seq.OwnerTable == "" || seq.OwnerColumn == "" {
- return false
- }
- for _, tbl := range tables {
- if tbl.Name != seq.OwnerTable {
- continue
- }
- for _, col := range tbl.Columns {
- if col.Name == seq.OwnerColumn && col.IsIdentity {
- return true
- }
- }
- }
- return false
-}
-
-func wtParseInt64(s string) int64 {
- var v int64
- _, _ = fmt.Sscanf(s, "%d", &v)
- return v
-}
-
-func wtInstallTable(cat *catalog.Catalog, obj *wtObjectEntry) error {
- items := make([]ast.Node, 0, len(obj.tableMeta.Columns))
- for _, col := range obj.tableMeta.Columns {
- if col.Name == "" {
- continue
- }
- if col.Type == "" {
- return errors.Errorf("column %q: empty type", col.Name)
- }
- tn, err := wtTypeNameFromString(col.Type)
- if err != nil {
- return errors.Wrapf(err, "column %q", col.Name)
- }
- colDef := &ast.ColumnDef{
- Colname: col.Name,
- TypeName: tn,
- IsNotNull: !col.Nullable,
- }
- if col.Default != "" && col.Generation == nil {
- if rawDefault, err := wtParseDefaultExpr(col.Default); err == nil {
- colDef.RawDefault = rawDefault
- }
- }
- if col.Generation != nil && col.Generation.Type == metadatapb.GenerationMetadata_TYPE_STORED && col.Generation.Expression != "" {
- colDef.Generated = 's'
- if genExpr := wtParseExpr(col.Generation.Expression); genExpr != nil {
- colDef.Constraints = &ast.List{Items: []ast.Node{
- &ast.Constraint{Contype: ast.CONSTR_GENERATED, RawExpr: genExpr},
- }}
- }
- }
- if col.IsIdentity {
- switch col.IdentityGeneration {
- case metadatapb.ColumnMetadata_ALWAYS:
- colDef.Identity = 'a'
- case metadatapb.ColumnMetadata_BY_DEFAULT:
- colDef.Identity = 'd'
- default:
- colDef.Identity = 'd'
- }
- }
- items = append(items, colDef)
- }
- return cat.DefineRelation(&ast.CreateStmt{
- Relation: &ast.RangeVar{
- Schemaname: obj.schema,
- Relname: obj.tableMeta.Name,
- Relpersistence: 'p',
- },
- TableElts: &ast.List{Items: items},
- }, 'r')
-}
-
-func wtInstallView(cat *catalog.Catalog, obj *wtObjectEntry) error {
- if obj.viewMeta.Definition == "" {
- return errors.New("empty view definition")
- }
- sel, err := wtParseSelectBody(obj.viewMeta.Definition)
- if err != nil {
- return errors.Wrapf(err, "view %q", obj.name)
- }
- return cat.DefineView(&ast.ViewStmt{
- View: &ast.RangeVar{
- Schemaname: obj.schema,
- Relname: obj.name,
- Relpersistence: 'p',
- },
- Query: sel,
- })
-}
-
-func wtInstallMatView(cat *catalog.Catalog, obj *wtObjectEntry) error {
- if obj.matViewMeta.Definition == "" {
- return errors.New("empty matview definition")
- }
- sel, err := wtParseSelectBody(obj.matViewMeta.Definition)
- if err != nil {
- return errors.Wrapf(err, "matview %q", obj.name)
- }
- return cat.ExecCreateTableAs(&ast.CreateTableAsStmt{
- Query: sel,
- Objtype: ast.OBJECT_MATVIEW,
- Into: &ast.IntoClause{
- Rel: &ast.RangeVar{
- Schemaname: obj.schema,
- Relname: obj.name,
- Relpersistence: 'p',
- },
- },
- })
-}
-
-func wtInstallFunction(cat *catalog.Catalog, obj *wtObjectEntry) error {
- fn := obj.funcMeta
- if fn.Definition != "" {
- nodes, err := omniparser.Parse(fn.Definition)
- if err == nil && nodes != nil && len(nodes.Items) == 1 {
- node := nodes.Items[0]
- if raw, ok := node.(*ast.RawStmt); ok {
- node = raw.Stmt
- }
- if parsed, ok := node.(*ast.CreateFunctionStmt); ok {
- return cat.CreateFunctionStmt(parsed)
- }
- }
- }
- // Fallback: build from signature.
- argTypes, err := wtParseFuncArgTypes(fn.Signature)
- if err != nil {
- return errors.Wrapf(err, "function %q signature %q", fn.Name, fn.Signature)
- }
- params := make([]ast.Node, 0, len(argTypes))
- for i, at := range argTypes {
- tn, err := wtTypeNameFromString(at)
- if err != nil {
- return errors.Wrapf(err, "function %q arg %d", fn.Name, i)
- }
- params = append(params, &ast.FunctionParameter{
- ArgType: tn,
- Mode: ast.FUNC_PARAM_IN,
- })
- }
- stmt := &ast.CreateFunctionStmt{
- Funcname: wtQualifiedList(obj.schema, fn.Name),
- ReturnType: wtPseudoTextTypeName(),
- }
- if len(params) > 0 {
- stmt.Parameters = &ast.List{Items: params}
- }
- stmt.Options = &ast.List{Items: []ast.Node{
- &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},
- &ast.DefElem{
- Defname: "as",
- Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT NULL::text"}}},
- },
- }}
- return cat.CreateFunctionStmt(stmt)
-}
-
-func wtInstallIndex(cat *catalog.Catalog, obj *wtObjectEntry) error {
- idx := obj.idxMeta
- if idx.Name == "" {
- return errors.New("index has no name")
- }
- params := make([]ast.Node, 0, len(idx.Expressions))
- for i, expr := range idx.Expressions {
- if expr == "" {
- continue
- }
- elem := &ast.IndexElem{}
- if i < len(idx.Descending) && idx.Descending[i] {
- elem.Ordering = ast.SORTBY_DESC
- }
- bare := wtUnquoteIdent(expr)
- if wtIsExpressionIndex(bare) {
- if parsed := wtParseExpr(expr); parsed != nil {
- elem.Expr = parsed
- } else {
- elem.Name = bare
- }
- } else {
- elem.Name = bare
- }
- params = append(params, elem)
- }
- return cat.DefineIndex(&ast.IndexStmt{
- Idxname: idx.Name,
- Relation: &ast.RangeVar{Schemaname: obj.schema, Relname: obj.parentName},
- AccessMethod: idx.Type,
- IndexParams: &ast.List{Items: params},
- Unique: idx.Unique,
- Primary: idx.Primary,
- })
-}
-
-func wtInstallConstraint(cat *catalog.Catalog, obj *wtObjectEntry) error {
- tbl := obj.constraintTable
- if tbl == nil {
- return errors.New("constraint has no parent table")
- }
-
- // Find which constraint this entry represents.
- for _, fk := range tbl.ForeignKeys {
- if fk.Name != obj.name {
- continue
- }
- refSchema := fk.ReferencedSchema
- if refSchema == "" {
- refSchema = obj.schema
- }
- def := catalog.ConstraintDef{
- Name: fk.Name,
- Type: catalog.ConstraintFK,
- Columns: fk.Columns,
- RefSchema: refSchema,
- RefTable: fk.ReferencedTable,
- RefColumns: fk.ReferencedColumns,
- FKUpdAction: wtFKActionByte(fk.OnUpdate),
- FKDelAction: wtFKActionByte(fk.OnDelete),
- FKMatchType: wtFKMatchByte(fk.MatchType),
- }
- return cat.AddConstraint(obj.schema, obj.parentName, def)
- }
- for _, chk := range tbl.CheckConstraints {
- if chk.Name != obj.name {
- continue
- }
- return cat.AddConstraint(obj.schema, obj.parentName, catalog.ConstraintDef{
- Name: chk.Name,
- Type: catalog.ConstraintCheck,
- CheckExpr: chk.Expression,
- })
- }
- for _, exc := range tbl.ExcludeConstraints {
- if exc.Name != obj.name {
- continue
- }
- return cat.AddConstraint(obj.schema, obj.parentName, catalog.ConstraintDef{
- Name: exc.Name,
- Type: catalog.ConstraintExclude,
- CheckExpr: exc.Expression,
- })
- }
- return errors.Errorf("constraint %q not found in table %q", obj.name, obj.parentName)
-}
-
-// ---- pseudo forms ----
-
-func wtPseudoTableStmt(schema, name string, cols []string) *ast.CreateStmt {
- items := make([]ast.Node, 0, len(cols))
- for _, col := range cols {
- if col == "" {
- continue
- }
- items = append(items, &ast.ColumnDef{
- Colname: col,
- TypeName: wtPseudoTextTypeName(),
- })
- }
- return &ast.CreateStmt{
- Relation: &ast.RangeVar{
- Schemaname: schema,
- Relname: name,
- Relpersistence: 'p',
- },
- TableElts: &ast.List{Items: items},
- }
-}
-
-func wtPseudoViewStmt(schema, name string, cols []string) (*ast.ViewStmt, error) {
- sel, err := wtPseudoConstantSelect(cols)
- if err != nil {
- return nil, err
- }
- return &ast.ViewStmt{
- View: &ast.RangeVar{
- Schemaname: schema,
- Relname: name,
- Relpersistence: 'p',
- },
- Query: sel,
- }, nil
-}
-
-func wtPseudoMatViewStmt(schema, name string, cols []string) (*ast.CreateTableAsStmt, error) {
- sel, err := wtPseudoConstantSelect(cols)
- if err != nil {
- return nil, err
- }
- return &ast.CreateTableAsStmt{
- Query: sel,
- Objtype: ast.OBJECT_MATVIEW,
- Into: &ast.IntoClause{
- Rel: &ast.RangeVar{
- Schemaname: schema,
- Relname: name,
- Relpersistence: 'p',
- },
- },
- }, nil
-}
-
-func wtPseudoConstantSelect(cols []string) (*ast.SelectStmt, error) {
- var targets []string
- for _, col := range cols {
- if col == "" {
- continue
- }
- targets = append(targets, "NULL::text AS "+wtQuoteIdent(col))
- }
- if len(targets) == 0 {
- targets = []string{"NULL::text"}
- }
- sql := "SELECT " + strings.Join(targets, ", ")
- nodes, err := omniparser.Parse(sql)
- if err != nil {
- return nil, errors.Wrap(err, "parse pseudo constant select")
- }
- if nodes == nil || len(nodes.Items) != 1 {
- return nil, errors.New("pseudo constant select: expected 1 statement")
- }
- node := nodes.Items[0]
- if raw, ok := node.(*ast.RawStmt); ok {
- node = raw.Stmt
- }
- sel, ok := node.(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("pseudo constant select: expected SelectStmt, got %T", node)
- }
- return sel, nil
-}
-
-// ---- column name helpers ----
-
-func wtColNames(tbl *metadatapb.TableMetadata) []string {
- if tbl == nil {
- return nil
- }
- seen := make(map[string]bool, len(tbl.Columns))
- out := make([]string, 0, len(tbl.Columns))
- for _, col := range tbl.Columns {
- if col.Name == "" || seen[col.Name] {
- continue
- }
- seen[col.Name] = true
- out = append(out, col.Name)
- }
- return out
-}
-
-func wtViewColNames(v *metadatapb.ViewMetadata) []string {
- if v == nil {
- return nil
- }
- if len(v.Columns) > 0 {
- seen := make(map[string]bool, len(v.Columns))
- out := make([]string, 0, len(v.Columns))
- for _, col := range v.Columns {
- if col.Name == "" || seen[col.Name] {
- continue
- }
- seen[col.Name] = true
- out = append(out, col.Name)
- }
- return out
- }
- seen := make(map[string]bool)
- var out []string
- for _, dc := range v.DependencyColumns {
- if dc.Column == "" || seen[dc.Column] {
- continue
- }
- seen[dc.Column] = true
- out = append(out, dc.Column)
- }
- return out
-}
-
-func wtMatViewColNames(m *metadatapb.MaterializedViewMetadata) []string {
- if m == nil {
- return nil
- }
- seen := make(map[string]bool)
- var out []string
- for _, dc := range m.DependencyColumns {
- if dc.Column == "" || seen[dc.Column] {
- continue
- }
- seen[dc.Column] = true
- out = append(out, dc.Column)
- }
- return out
-}
-
-// ---- type parsing helpers ----
-
-// wtTypeNameFromString parses a PG type string into *ast.TypeName via
-// "SELECT NULL::<type>" to reuse omni's SELECT parse path.
-func wtTypeNameFromString(typeStr string) (*ast.TypeName, error) {
- nodes, err := omniparser.Parse("SELECT NULL::" + typeStr)
- if err != nil {
- return nil, errors.Wrapf(err, "parse type %q", typeStr)
- }
- if nodes == nil || len(nodes.Items) != 1 {
- return nil, errors.Errorf("type %q: expected 1 statement", typeStr)
- }
- node := nodes.Items[0]
- if raw, ok := node.(*ast.RawStmt); ok {
- node = raw.Stmt
- }
- sel, ok := node.(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("type %q: expected SelectStmt, got %T", typeStr, node)
- }
- if sel.TargetList == nil || len(sel.TargetList.Items) != 1 {
- return nil, errors.Errorf("type %q: expected 1 target", typeStr)
- }
- rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)
- if !ok {
- return nil, errors.Errorf("type %q: expected ResTarget, got %T", typeStr, sel.TargetList.Items[0])
- }
- cast, ok := rt.Val.(*ast.TypeCast)
- if !ok {
- return nil, errors.Errorf("type %q: expected TypeCast, got %T", typeStr, rt.Val)
- }
- return cast.TypeName, nil
-}
-
-// wtParseDefaultExpr parses a default expression string and returns the AST node.
-func wtParseDefaultExpr(expr string) (ast.Node, error) {
- nodes, err := omniparser.Parse("SELECT " + expr)
- if err != nil {
- return nil, errors.Wrapf(err, "parse default %q", expr)
- }
- if nodes == nil || len(nodes.Items) != 1 {
- return nil, errors.Errorf("default %q: expected 1 statement", expr)
- }
- node := nodes.Items[0]
- if raw, ok := node.(*ast.RawStmt); ok {
- node = raw.Stmt
- }
- sel, ok := node.(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("default %q: expected SelectStmt", expr)
- }
- if sel.TargetList == nil || len(sel.TargetList.Items) != 1 {
- return nil, errors.Errorf("default %q: expected 1 target", expr)
- }
- rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)
- if !ok {
- return nil, errors.Errorf("default %q: expected ResTarget", expr)
- }
- return rt.Val, nil
-}
-
-// wtParseSelectBody parses a SQL string into *ast.SelectStmt.
-func wtParseSelectBody(sql string) (*ast.SelectStmt, error) {
- nodes, err := omniparser.Parse(sql)
- if err != nil {
- return nil, errors.Wrap(err, "parse")
- }
- if nodes == nil || len(nodes.Items) != 1 {
- return nil, errors.Errorf("expected 1 statement, got %d", len(nodes.Items))
- }
- node := nodes.Items[0]
- if raw, ok := node.(*ast.RawStmt); ok {
- node = raw.Stmt
- }
- sel, ok := node.(*ast.SelectStmt)
- if !ok {
- return nil, errors.Errorf("expected SelectStmt, got %T", node)
- }
- return sel, nil
-}
-
-// ---- type ref extraction (local, no import from parser/pg) ----
-
-type wtUserTypeRef struct {
- Schema string
- Name string
-}
-
-func wtExtractUserTypeRefs(typeStr string) []wtUserTypeRef {
- if typeStr == "" {
- return nil
- }
- base := wtStripTypeModifiers(typeStr)
- if base == "" || wtIsBuiltinType(base) {
- return nil
- }
- if strings.HasPrefix(base, "_") {
- return nil
- }
- schema, name, ok := wtSplitQualifiedName(base)
- if !ok {
- return nil
- }
- if wtIsSystemSchema(schema) {
- return nil
- }
- return []wtUserTypeRef{{Schema: schema, Name: name}}
-}
-
-func wtStripTypeModifiers(s string) string {
- s = strings.TrimSpace(s)
- // Array suffixes wrap the element type: "public.addr[]" refers to
- // public.addr. Quoted identifiers end with a quote, so a genuine name
- // containing "[]" is never clipped.
- for strings.HasSuffix(s, "[]") {
- s = strings.TrimSpace(strings.TrimSuffix(s, "[]"))
- }
- lower := strings.ToLower(s)
- for _, sfx := range []string{" with time zone", " without time zone"} {
- if strings.HasSuffix(lower, sfx) {
- s = s[:len(s)-len(sfx)]
- lower = lower[:len(lower)-len(sfx)]
- }
- }
- if i := strings.Index(s, "("); i >= 0 {
- s = strings.TrimSpace(s[:i])
- }
- return s
-}
-
-func wtSplitQualifiedName(s string) (schema, name string, ok bool) {
- var parts []string
- var cur strings.Builder
- inQuote := false
- for i := 0; i < len(s); i++ {
- c := s[i]
- switch {
- case c == '"':
- inQuote = !inQuote
- case c == '.' && !inQuote:
- parts = append(parts, cur.String())
- cur.Reset()
- default:
- cur.WriteByte(c)
- }
- }
- if inQuote {
- return "", "", false
- }
- parts = append(parts, cur.String())
- if len(parts) != 2 {
- return "", "", false
- }
- return parts[0], parts[1], true
-}
-
-func wtIsSystemSchema(s string) bool {
- switch s {
- case "pg_catalog", "pg_toast", "information_schema":
- return true
- }
- return strings.HasPrefix(s, "pg_")
-}
-
-func wtIsBuiltinType(s string) bool {
- _, ok := wtBuiltinTypes[strings.ToLower(s)]
- return ok
-}
-
-var wtBuiltinTypes = map[string]struct{}{
- "smallint": {}, "integer": {}, "int": {}, "int2": {}, "int4": {}, "int8": {},
- "bigint": {}, "decimal": {}, "numeric": {}, "real": {}, "double precision": {},
- "float4": {}, "float8": {}, "smallserial": {}, "serial": {}, "bigserial": {},
- "money": {}, "character": {}, "character varying": {}, "char": {}, "varchar": {},
- "text": {}, "bytea": {}, "bpchar": {}, "name": {}, "bit": {}, "bit varying": {},
- "varbit": {}, "date": {}, "time": {}, "timetz": {}, "timestamp": {}, "timestamptz": {},
- "interval": {}, "boolean": {}, "bool": {}, "json": {}, "jsonb": {}, "xml": {},
- "uuid": {}, "point": {}, "line": {}, "lseg": {}, "box": {}, "path": {},
- "polygon": {}, "circle": {}, "cidr": {}, "inet": {}, "macaddr": {}, "macaddr8": {},
- "tsvector": {}, "tsquery": {}, "void": {}, "record": {}, "anyelement": {},
- "anyarray": {}, "anynonarray": {}, "anyenum": {}, "anyrange": {}, "any": {},
- "trigger": {}, "event_trigger": {}, "cstring": {}, "internal": {},
- "language_handler": {}, "fdw_handler": {}, "index_am_handler": {}, "tsm_handler": {},
- "pg_lsn": {}, "oid": {}, "regclass": {}, "regproc": {}, "regprocedure": {},
- "regoper": {}, "regoperator": {}, "regtype": {}, "regconfig": {}, "regdictionary": {},
- "regnamespace": {}, "regrole": {},
-}
-
-// wtExtractSeqRef extracts the seq key from a nextval('schema.seq'::regclass) default.
-// Returns the "seq:schema.name" key if found, empty string otherwise.
-var reNextval = regexp.MustCompile(`nextval\('([^']+)'`)
-
-func wtExtractSeqRef(defaultExpr, tableSchema string, index map[string]*wtObjectEntry) string {
- m := reNextval.FindStringSubmatch(defaultExpr)
- if m == nil {
- return ""
- }
- // m[1] is the content: "schema.seq_name" or "seq_name"
- raw := strings.TrimSuffix(m[1], "::regclass")
- raw = strings.Trim(raw, "\"")
- var seqSchema, seqName string
- if dot := strings.Index(raw, "."); dot >= 0 {
- seqSchema = strings.Trim(raw[:dot], "\"")
- seqName = strings.Trim(raw[dot+1:], "\"")
- } else {
- seqSchema = tableSchema
- seqName = raw
- }
- k := "seq:" + seqSchema + "." + seqName
- if index[k] != nil {
- return k
- }
- return ""
-}
-
-// ---- function signature helpers ----
-
-func wtParseFuncArgTypes(signature string) ([]string, error) {
- open := strings.Index(signature, "(")
- if open < 0 {
- return nil, nil
- }
- closeIdx := strings.LastIndex(signature, ")")
- if closeIdx < 0 || closeIdx <= open {
- return nil, errors.Errorf("signature %q: unbalanced parens", signature)
- }
- return wtSplitTopLevelCommas(signature[open+1 : closeIdx]), nil
-}
-
-func wtSplitTopLevelCommas(s string) []string {
- s = strings.TrimSpace(s)
- if s == "" {
- return nil
- }
- var parts []string
- depth := 0
- start := 0
- for i := 0; i < len(s); i++ {
- switch s[i] {
- case '(':
- depth++
- case ')':
- if depth > 0 {
- depth--
- }
- case ',':
- if depth == 0 {
- parts = append(parts, strings.TrimSpace(s[start:i]))
- start = i + 1
- }
- default:
- // other bytes are part of the current token
- }
- }
- parts = append(parts, strings.TrimSpace(s[start:]))
- return parts
-}
-
-// ---- FK action helpers ----
-
-func wtFKActionByte(action string) byte {
- switch strings.ToUpper(action) {
- case "RESTRICT":
- return 'r'
- case "CASCADE":
- return 'c'
- case "SET NULL":
- return 'n'
- case "SET DEFAULT":
- return 'd'
- default:
- return 'a' // NO ACTION
- }
-}
-
-func wtFKMatchByte(match string) byte {
- switch strings.ToUpper(match) {
- case "FULL":
- return 'f'
- case "PARTIAL":
- return 'p'
- default:
- return 's' // SIMPLE
- }
-}
-
-// ---- small AST helpers ----
-
-func wtQualifiedList(schema, name string) *ast.List {
- items := make([]ast.Node, 0, 2)
- if schema != "" {
- items = append(items, &ast.String{Str: schema})
- }
- items = append(items, &ast.String{Str: name})
- return &ast.List{Items: items}
-}
-
-func wtPseudoTextTypeName() *ast.TypeName {
- return &ast.TypeName{
- Names: &ast.List{Items: []ast.Node{&ast.String{Str: "text"}}},
- Typemod: -1,
- }
-}
-
-func wtQuoteIdent(name string) string {
- return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
-}
-
-func wtIsExpressionIndex(expr string) bool {
- return strings.ContainsAny(expr, "( ") || strings.Contains(expr, "::")
-}
-
-func wtUnquoteIdent(s string) string {
- if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' {
- return strings.ReplaceAll(s[1:len(s)-1], `""`, `"`)
- }
- return s
-}
-
-func wtUnquoteColumns(cols []string) []string {
- out := make([]string, len(cols))
- for i, c := range cols {
- out[i] = wtUnquoteIdent(c)
- }
- return out
-}
-
-func wtParseExpr(expr string) ast.Node {
- stmts, err := omniparser.Parse("SELECT (" + expr + ")")
- if err != nil || stmts == nil || len(stmts.Items) == 0 {
- return nil
- }
- raw, ok := stmts.Items[0].(*ast.RawStmt)
- if !ok {
- return nil
- }
- sel, ok := raw.Stmt.(*ast.SelectStmt)
- if !ok || sel.TargetList == nil || len(sel.TargetList.Items) == 0 {
- return nil
- }
- rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)
- if !ok {
- return nil
- }
- return rt.Val
-}
-
-// collateClauseFromReference converts a stored emit-ready collation reference
-// (quoted as needed, schema-qualified outside pg_catalog, e.g. `"C"` or
-// `locale.en_us`) into an AST collate clause. Returns nil for no collation.
-func collateClauseFromReference(reference string) *ast.CollateClause {
- if reference == "" {
- return nil
- }
- var items []ast.Node
- if schemaName, collationName, ok := parseQualifiedTypeIdent(reference); ok {
- items = []ast.Node{&ast.String{Str: schemaName}, &ast.String{Str: collationName}}
- } else {
- name := strings.Trim(reference, `"`)
- name = strings.ReplaceAll(name, `""`, `"`)
- items = []ast.Node{&ast.String{Str: name}}
- }
- return &ast.CollateClause{Collname: &ast.List{Items: items}}
-}
diff --git a/backend/plugin/schema/pg/walk_through_test.go b/backend/plugin/schema/pg/walk_through_test.go
index 58a25df9eb..55c853c079 100644
--- a/backend/plugin/schema/pg/walk_through_test.go
+++ b/backend/plugin/schema/pg/walk_through_test.go
@@ -1,12 +1,14 @@
package pg
import (
+ "context"
"io"
"os"
"path/filepath"
"testing"
metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/bytebase/omni/pg/catalog"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
@@ -103,7 +105,7 @@ func TestWalkThrough(t *testing.T) {
stmts, _ := sm.GetStatementsForChecks(storepb.Engine_POSTGRES, test.Statement)
asts := base.ExtractASTs(stmts)
- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)
+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)
if test.Advice != nil {
require.NotNil(t, advice)
require.Equal(t, test.Advice.Code, advice.Code)
@@ -207,7 +209,7 @@ func TestWalkThroughANTLR(t *testing.T) {
asts := base.ExtractASTs(stmts)
// Call WalkThrough with AST
- advice := WalkThroughWithContext(schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)
+ advice := WalkThroughWithContext(context.Background(), schema.WalkThroughContext{RawSQL: test.Statement}, state, asts)
if advice != nil {
// Compare the advice fields
if test.Advice != nil {
@@ -323,9 +325,9 @@ func TestWalkThroughSearchPathState(t *testing.T) {
state := newSearchPathTestState(test.searchPath)
stmts, err := base.ParseStatements(storepb.Engine_POSTGRES, test.sql)
require.NoError(t, err)
- ctx := test.session
- ctx.RawSQL = test.sql
- advice := WalkThroughWithContext(ctx, state, base.ExtractASTs(stmts))
+ wtCtx := test.session
+ wtCtx.RawSQL = test.sql
+ advice := WalkThroughWithContext(context.Background(), wtCtx, state, base.ExtractASTs(stmts))
require.Nil(t, advice)
test.assert(t, state)
})
@@ -369,3 +371,86 @@ func newSearchPathTestTable(name string) *metadatapb.TableMetadata {
},
}
}
+
+// TestClone_SearchPath tests Clone preserves search path and session user.
+func TestClone_SearchPath(t *testing.T) {
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "postgres",
+ Schemas: []*metadatapb.SchemaMetadata{
+ {Name: "public"},
+ {Name: "alice"},
+ },
+ }
+
+ catBefore := catalog.New()
+ catBefore.SetSessionUser("alice")
+ catBefore.SetSearchPath([]string{"$user", "public"})
+ _, err := catBefore.LoadMetadata(context.Background(), meta, catalog.LoadMetadataOptions{Full: true})
+ require.NoError(t, err)
+
+ catAfter := catBefore.Clone()
+
+ // CREATE TABLE without schema should use search path ($user → alice)
+ _, err = catAfter.Exec(`CREATE TABLE my_table (id int);`, nil)
+ require.NoError(t, err)
+
+ // Table should be in alice schema on clone
+ require.NotNil(t, catAfter.GetRelation("alice", "my_table"))
+
+ // Original should NOT have it
+ require.Nil(t, catBefore.GetRelation("alice", "my_table"))
+
+ // Diff should show the new table
+ diff := catalog.Diff(catBefore, catAfter)
+ require.False(t, diff.IsEmpty())
+
+ foundAliceTable := false
+ for _, rel := range diff.Relations {
+ if rel.SchemaName == "alice" && rel.Name == "my_table" && rel.Action == catalog.DiffAdd {
+ foundAliceTable = true
+ }
+ }
+ require.True(t, foundAliceTable, "diff should show alice.my_table as added")
+}
+
+// TestClone_WalkThroughFunction tests the actual WalkThroughWithContext function
+// using Clone produces correct FinalMetadata.
+func TestClone_WalkThroughFunction(t *testing.T) {
+ meta := &metadatapb.DatabaseSchemaMetadata{
+ Name: "postgres",
+ Schemas: []*metadatapb.SchemaMetadata{
+ {
+ Name: "public",
+ Tables: []*metadatapb.TableMetadata{
+ {
+ Name: "test",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "integer", Position: 1},
+ {Name: "name", Type: "text", Position: 2, Nullable: true},
+ },
+ Indexes: []*metadatapb.IndexMetadata{
+ {Name: "test_pkey", Expressions: []string{"id"}, Unique: true, Primary: true},
+ },
+ },
+ },
+ },
+ },
+ }
+
+ state := model.NewDatabaseMetadata(meta, nil, nil, storepb.Engine_POSTGRES, true)
+ wtCtx := schema.WalkThroughContext{
+ RawSQL: `CREATE TABLE public.new_table (id int PRIMARY KEY, val text);`,
+ }
+
+ advice := WalkThroughWithContext(context.Background(), wtCtx, state, nil)
+ require.Nil(t, advice, "walk-through should succeed")
+
+ // Check FinalMetadata has the new table
+ newTbl := state.GetSchemaMetadata("public").GetTable("new_table")
+ require.NotNil(t, newTbl, "new_table should exist in FinalMetadata")
+
+ // Check original table is preserved
+ origTbl := state.GetSchemaMetadata("public").GetTable("test")
+ require.NotNil(t, origTbl, "test table should still exist")
+ require.Equal(t, 2, len(origTbl.GetProto().Columns))
+}
diff --git a/backend/plugin/schema/schema.go b/backend/plugin/schema/schema.go
index 71c62d5c11..d7128afb7e 100644
--- a/backend/plugin/schema/schema.go
+++ b/backend/plugin/schema/schema.go
@@ -1,6 +1,7 @@
package schema
import (
+ "context"
"fmt"
"strings"
"sync"
@@ -65,7 +66,7 @@ type sdlDropAdvices func(userSDLText string, currentSchema *model.DatabaseMetada
type diffSDLMigration func(sourceSDL, targetSDL, engineVersion string, sessionCtx *SDLSessionContextMap) (string, error)
type diffMetadataMigration func(oldSchema, newSchema *model.DatabaseMetadata) (string, error)
type walkThrough func(*model.DatabaseMetadata, []base.AST) *storepb.Advice
-type walkThroughWithContext func(WalkThroughContext, *model.DatabaseMetadata, []base.AST) *storepb.Advice
+type walkThroughWithContext func(context.Context, WalkThroughContext, *model.DatabaseMetadata, []base.AST) *storepb.Advice
// WalkThroughContext carries optional session state into schema walk-through implementations.
type WalkThroughContext struct {
@@ -533,12 +534,12 @@ func RegisterWalkThroughWithContext(engine storepb.Engine, f walkThroughWithCont
}
func WalkThrough(engine storepb.Engine, d *model.DatabaseMetadata, ast []base.AST) *storepb.Advice {
- return WalkThroughWithContext(engine, WalkThroughContext{}, d, ast)
+ return WalkThroughWithContext(context.Background(), engine, WalkThroughContext{}, d, ast)
}
-func WalkThroughWithContext(engine storepb.Engine, ctx WalkThroughContext, d *model.DatabaseMetadata, ast []base.AST) *storepb.Advice {
+func WalkThroughWithContext(ctx context.Context, engine storepb.Engine, wtCtx WalkThroughContext, d *model.DatabaseMetadata, ast []base.AST) *storepb.Advice {
if f, ok := walkThroughsWithContext[engine]; ok {
- return f(ctx, d, ast)
+ return f(ctx, wtCtx, d, ast)
}
f, ok := walkThroughs[engine]
if !ok {
diff --git a/go.mod b/go.mod
index 79421ce3e3..0bb16e5fe8 100644
--- a/go.mod
+++ b/go.mod
@@ -34,7 +34,7 @@ require (
github.com/bmatcuk/doublestar/v4 v4.10.0
github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0
- github.com/bytebase/omni v0.0.0-20260911085056-71713ba5c090
+ github.com/bytebase/omni v0.0.0-20260912023254-4574e69bb9f1
github.com/caarlos0/env/v11 v11.4.1
github.com/cenkalti/backoff/v7 v7.0.0
github.com/cockroachdb/cockroachdb-parser v0.25.2
diff --git a/go.sum b/go.sum
index 5d42a50795..5e26f4029e 100644
--- a/go.sum
+++ b/go.sum
@@ -252,8 +252,8 @@ github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735 h1:v8y6mXeHmYVsdM
github.com/bytebase/gomongo v0.0.0-20260730033313-67ba6e249735/go.mod h1:vTHjcvfn8Pmf65WjnV1ytTCYyC7xNhFjrniv4YWvtA8=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=
github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=
-github.com/bytebase/omni v0.0.0-20260911085056-71713ba5c090 h1:0EZ0/nFVpkwDBp88Zb6nJAUmHAeP570s5dkOegvElxQ=
-github.com/bytebase/omni v0.0.0-20260911085056-71713ba5c090/go.mod h1:9XZE1hqtvr09Oc+DdLu1/yCb8MDcoy/EPdA4AONDHlE=
+github.com/bytebase/omni v0.0.0-20260912023254-4574e69bb9f1 h1:F0XizxSJZ5z5mdDkEMWwjgIso2mWG6kZB4MBNjwwmE0=
+github.com/bytebase/omni v0.0.0-20260912023254-4574e69bb9f1/go.mod h1:9XZE1hqtvr09Oc+DdLu1/yCb8MDcoy/EPdA4AONDHlE=
github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=
github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
github.com/bytebase/pkcs8 v0.0.0-20240612095628-fcd0a7484c94 h1:6PNsuNmSqCuR8Z616uQLzDIvujWCFsKzeVc3ECmVubk=