Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(pg): eager per-object catalog loader with pseudo fallback [BYT-9215] (#20023)
rebelicea99da93799d2
diff --git a/backend/plugin/parser/pg/query_span_loader.go b/backend/plugin/parser/pg/query_span_loader.gonew file mode 100644index 0000000000..0272365635--- /dev/null+++ b/backend/plugin/parser/pg/query_span_loader.go@@ -0,0 +1,695 @@+package pg++import (+ "cmp"+ "context"+ "slices"++ "github.com/pkg/errors"++ "github.com/bytebase/omni/pg/ast"+ "github.com/bytebase/omni/pg/catalog"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// catalogLoader installs every schema object from a storepb.DatabaseSchemaMetadata+// into an omni catalog, in dependency-topological order, with an inline+// pseudo fallback at each failed slot.+//+// Invariants after Load() returns without error:+// 1. Every object from the metadata is present in the catalog — as either a+// real install or a pseudo install. `loaderObjects` records its identity.+// 2. `degraded[key]` contains the real-install error for objects whose real+// install failed and fell back to pseudo.+// 3. `trulyBroken[key]` contains the pseudo-install error for objects whose+// even-pseudo install failed. These are the few objects callers can+// expect analyzer errors on.+//+// The loader holds no reference to the catalog after Load() completes; the+// catalog object belongs to the caller.+type catalogLoader struct {+ cat *catalog.Catalog+ meta *storepb.DatabaseSchemaMetadata++ // loaderObjects maps canonical object keys (schema.name) to true for+ // every object the loader collected from metadata. Used by the classifier+ // to tell "loader knew about X and something went wrong" from "X was+ // never in metadata."+ loaderObjects map[string]bool++ // degraded records objects whose real install failed. The pseudo version+ // is in the catalog for these entries.+ degraded map[string]error++ // trulyBroken records objects whose pseudo install also failed. The+ // catalog has nothing at this slot.+ trulyBroken map[string]error+}++// objectKind identifies the kind of an Catalog loader object. It drives the+// install switch in installReal/installPseudo and the dependency-edge+// computation in buildDependencyGraph.+type objectKind int++const (+ kindSchema objectKind = iota+ kindEnum+ kindTable+ kindView+ kindMatView+ kindFunction+)++// objectEntry is the flattened, order-free unit of work the loader processes.+// One entry per schema object. Entries for the same (schema, name) across+// different kinds are allowed (e.g. a table and a function with the same+// name) and treated as independent nodes in the dep graph.+type objectEntry struct {+ kind objectKind+ schema string+ name string++ // Exactly one of the following is populated based on kind.+ enumMeta *storepb.EnumTypeMetadata+ tableMeta *storepb.TableMetadata+ viewMeta *storepb.ViewMetadata+ matViewMeta *storepb.MaterializedViewMetadata+ funcMeta *storepb.FunctionMetadata+}++// key returns a canonical kind-prefixed identifier used for dependency+// lookups and set membership. Tables, views, and matviews share a namespace+// (relation namespace in PG), so they use the same prefix "rel:".+//+// Functions are identified by schema, name AND signature so that overloaded+// names (e.g. `public.fn(int4)` and `public.fn(text)`) do not collide and+// collapse into a single entry during topo-sort bookkeeping.+func (e *objectEntry) key() string {+ switch e.kind {+ case kindSchema:+ return "schema:" + e.schema+ case kindEnum:+ return "type:" + e.schema + "." + e.name+ case kindTable, kindView, kindMatView:+ return "rel:" + e.schema + "." + e.name+ case kindFunction:+ return "func:" + e.schema + "." + e.name + "|" + funcSignatureKey(e.funcMeta)+ }+ return "unknown:" + e.schema + "." + e.name+}++// sortKey returns the intra-SCC lexicographic key. Matches hard contract C10.+// For functions the signature is appended so overloads have distinct sort+// keys and the install order is fully deterministic.+func (e *objectEntry) sortKey() string {+ base := e.schema + "\x00" + e.name + "\x00" + kindLabel(e.kind)+ if e.kind == kindFunction {+ base += "\x00" + funcSignatureKey(e.funcMeta)+ }+ return base+}++// funcSignatureKey returns a stable, comparable identifier for a function+// signature. Uses the metadata Signature string when present; falls back to+// the definition hash to keep distinct overloads distinguishable even when+// sync emitted an empty Signature field.+func funcSignatureKey(fn *storepb.FunctionMetadata) string {+ if fn == nil {+ return ""+ }+ if fn.Signature != "" {+ return fn.Signature+ }+ return fn.Definition+}++func kindLabel(k objectKind) string {+ switch k {+ case kindSchema:+ return "0schema"+ case kindEnum:+ return "1enum"+ case kindTable:+ return "2table"+ case kindView:+ return "3view"+ case kindMatView:+ return "4matview"+ case kindFunction:+ return "5function"+ }+ return "9unknown"+}++// newCatalogLoader returns a loader primed for Load(). It does not touch the+// catalog until Load() is called.+func newCatalogLoader(cat *catalog.Catalog, meta *storepb.DatabaseSchemaMetadata) *catalogLoader {+ return &catalogLoader{+ cat: cat,+ meta: meta,+ loaderObjects: make(map[string]bool),+ degraded: make(map[string]error),+ trulyBroken: make(map[string]error),+ }+}++// Load collects every object from metadata, topologically orders them, and+// installs each either real or (on failure) pseudo. The returned error is+// non-nil only for catastrophic conditions (nil catalog, ctx cancellation);+// per-object failures are recorded in degraded/trulyBroken and do not stop+// the loader.+func (l *catalogLoader) Load(ctx context.Context) error {+ if l.cat == nil {+ return errors.New("catalogLoader: nil catalog")+ }+ if l.meta == nil {+ return nil+ }++ objects := l.collectObjects()+ for _, obj := range objects {+ l.loaderObjects[obj.key()] = true+ }++ sorted := topoSortObjects(objects)+ for _, obj := range sorted {+ if err := ctx.Err(); err != nil {+ return err+ }+ l.installOne(obj)+ }+ return nil+}++// collectObjects flattens DatabaseSchemaMetadata into a list of objectEntry+// values, one per schema object the loader will install. Schemas themselves+// become kindSchema entries so CREATE SCHEMA is part of the topo order.+func (l *catalogLoader) collectObjects() []*objectEntry {+ var out []*objectEntry+ for _, sm := range l.meta.Schemas {+ if sm.Name == "" {+ // PG sync sometimes emits a no-schema placeholder; skip.+ continue+ }+ out = append(out, &objectEntry{+ kind: kindSchema,+ schema: sm.Name,+ name: sm.Name,+ })+ for _, enum := range sm.EnumTypes {+ out = append(out, &objectEntry{+ kind: kindEnum,+ schema: sm.Name,+ name: enum.Name,+ enumMeta: enum,+ })+ }+ for _, tbl := range sm.Tables {+ out = append(out, &objectEntry{+ kind: kindTable,+ schema: sm.Name,+ name: tbl.Name,+ tableMeta: tbl,+ })+ }+ for _, view := range sm.Views {+ out = append(out, &objectEntry{+ kind: kindView,+ schema: sm.Name,+ name: view.Name,+ viewMeta: view,+ })+ }+ for _, mv := range sm.MaterializedViews {+ out = append(out, &objectEntry{+ kind: kindMatView,+ schema: sm.Name,+ name: mv.Name,+ matViewMeta: mv,+ })+ }+ for _, fn := range sm.Functions {+ out = append(out, &objectEntry{+ kind: kindFunction,+ schema: sm.Name,+ name: fn.Name,+ funcMeta: fn,+ })+ }+ }+ return out+}++// topoSortObjects orders objects by their dependency edges so each object is+// processed only after its dependencies. Cycles are broken via Tarjan SCC+// with intra-SCC lexicographic ordering (C10): one member of an SCC installs+// first (and typically fails → pseudo), the rest install against the pseudo.+func topoSortObjects(objects []*objectEntry) []*objectEntry {+ if len(objects) == 0 {+ return nil+ }++ index := make(map[string]*objectEntry, len(objects))+ for _, obj := range objects {+ index[obj.key()] = obj+ }++ edges := buildDependencyEdges(objects, index)+ sccs := tarjanSCC(objects, edges)++ // Build a condensed DAG of SCCs, then topo sort and flatten.+ sccOf := make(map[string]int, len(objects))+ for i, scc := range sccs {+ for _, obj := range scc {+ sccOf[obj.key()] = i+ }+ }+ condensedEdges := make([][]int, len(sccs))+ inDegree := make([]int, len(sccs))+ seenEdge := make(map[[2]int]bool)+ for src, dsts := range edges {+ srcSCC, ok := sccOf[src]+ if !ok {+ continue+ }+ for _, dst := range dsts {+ dstSCC, ok := sccOf[dst]+ if !ok || dstSCC == srcSCC {+ continue+ }+ edge := [2]int{dstSCC, srcSCC} // dst must come before src+ if seenEdge[edge] {+ continue+ }+ seenEdge[edge] = true+ condensedEdges[dstSCC] = append(condensedEdges[dstSCC], srcSCC)+ inDegree[srcSCC]+++ }+ }++ // Kahn's algorithm on the condensed graph with a deterministic tiebreak:+ // ready SCCs are sorted by the lex-smallest sort key of their members.+ readyHeap := make([]int, 0)+ for i := range sccs {+ if inDegree[i] == 0 {+ readyHeap = append(readyHeap, i)+ }+ }+ sortSCCsByMin(readyHeap, sccs)++ var flat []*objectEntry+ for len(readyHeap) > 0 {+ next := readyHeap[0]+ readyHeap = readyHeap[1:]+ flat = append(flat, sortedSCCMembers(sccs[next])...)+ for _, nb := range condensedEdges[next] {+ inDegree[nb]--+ if inDegree[nb] == 0 {+ readyHeap = append(readyHeap, nb)+ }+ }+ sortSCCsByMin(readyHeap, sccs)+ }++ // Safety net: if any objects were not emitted (e.g. invariant violation+ // in Tarjan / edge building), emit them in deterministic order so the+ // loader still covers them.+ if len(flat) != len(objects) {+ emitted := make(map[string]bool, len(flat))+ for _, o := range flat {+ emitted[o.key()] = true+ }+ var missed []*objectEntry+ for _, o := range objects {+ if !emitted[o.key()] {+ missed = append(missed, o)+ }+ }+ slices.SortStableFunc(missed, func(a, b *objectEntry) int {+ return cmp.Compare(a.sortKey(), b.sortKey())+ })+ flat = append(flat, missed...)+ }+ return flat+}++// buildDependencyEdges returns a map from each object's key to the keys of+// objects it depends on. Only edges whose target actually exists in the+// collected set are recorded; references to unknown objects become no-ops+// (the install will fail and trigger pseudo).+func buildDependencyEdges(objects []*objectEntry, index map[string]*objectEntry) map[string][]string {+ edges := make(map[string][]string, len(objects))+ for _, obj := range objects {+ var deps []string+ // Every non-schema object depends on its schema.+ if obj.kind != kindSchema {+ if key := "schema:" + obj.schema; index[key] != nil {+ deps = append(deps, key)+ }+ }+ switch obj.kind {+ case kindTable:+ for _, col := range obj.tableMeta.Columns {+ for _, ref := range extractUserTypeRefs(col.Type) {+ if k := typeRefKey(ref); index[k] != nil {+ deps = append(deps, k)+ }+ }+ }+ case kindView:+ for _, dep := range obj.viewMeta.DependencyColumns {+ if k := relationKey(dep.Schema, dep.Table); index[k] != nil {+ deps = append(deps, k)+ }+ }+ case kindMatView:+ for _, dep := range obj.matViewMeta.DependencyColumns {+ if k := relationKey(dep.Schema, dep.Table); index[k] != nil {+ deps = append(deps, k)+ }+ }+ case kindFunction:+ argTypes, _ := parseFunctionSignatureArgTypes(obj.funcMeta.Signature)+ for _, at := range argTypes {+ for _, ref := range extractUserTypeRefs(at) {+ if k := typeRefKey(ref); index[k] != nil {+ deps = append(deps, k)+ }+ }+ }+ case kindSchema, kindEnum:+ // Schemas have no outgoing edges (they are the top of the graph).+ // Enums depend only on their schema, already captured above.+ default:+ // Unknown kinds contribute no edges; install will surface the+ // problem at real-install time.+ }+ if len(deps) > 0 {+ edges[obj.key()] = dedupStrings(deps)+ }+ }+ return edges+}++func typeRefKey(r UserTypeRef) string {+ return "type:" + r.Schema + "." + r.Name+}++func relationKey(schema, name string) string {+ return "rel:" + schema + "." + name+}++func dedupStrings(in []string) []string {+ if len(in) < 2 {+ return in+ }+ seen := make(map[string]bool, len(in))+ out := in[:0]+ for _, s := range in {+ if !seen[s] {+ seen[s] = true+ out = append(out, s)+ }+ }+ return out+}++// tarjanSCC runs Tarjan's strongly-connected-components algorithm on the+// dependency graph and returns SCCs. Each SCC's members are returned in the+// order the algorithm discovered them; callers sort for determinism.+func tarjanSCC(objects []*objectEntry, edges map[string][]string) [][]*objectEntry {+ type state struct {+ index, low int+ onStack bool+ }+ st := make(map[string]*state, len(objects))+ byKey := make(map[string]*objectEntry, len(objects))+ for _, obj := range objects {+ byKey[obj.key()] = obj+ }++ // Walk nodes in deterministic order so tests and shadow diffs reproduce.+ keys := make([]string, 0, len(objects))+ for _, obj := range objects {+ keys = append(keys, obj.key())+ }+ slices.Sort(keys)++ var (+ index int+ stack []string+ result [][]*objectEntry+ )++ var strongconnect func(v string)+ strongconnect = func(v string) {+ st[v] = &state{index: index, low: index, onStack: true}+ index+++ stack = append(stack, v)++ for _, w := range edges[v] {+ if _, ok := byKey[w]; !ok {+ continue+ }+ if _, seen := st[w]; !seen {+ strongconnect(w)+ st[v].low = min(st[v].low, st[w].low)+ } else if st[w].onStack {+ st[v].low = min(st[v].low, st[w].index)+ }+ }++ if st[v].low == st[v].index {+ var scc []*objectEntry+ for {+ w := stack[len(stack)-1]+ stack = stack[:len(stack)-1]+ st[w].onStack = false+ scc = append(scc, byKey[w])+ if w == v {+ break+ }+ }+ result = append(result, scc)+ }+ }++ for _, k := range keys {+ if _, seen := st[k]; !seen {+ strongconnect(k)+ }+ }+ return result+}++// sortedSCCMembers returns a copy of scc sorted by sortKey for deterministic+// install order.+func sortedSCCMembers(scc []*objectEntry) []*objectEntry {+ out := make([]*objectEntry, len(scc))+ copy(out, scc)+ slices.SortStableFunc(out, func(a, b *objectEntry) int {+ return cmp.Compare(a.sortKey(), b.sortKey())+ })+ return out+}++// sortSCCsByMin sorts a slice of SCC indices in place by the lex-smallest+// sortKey among each SCC's members. Stable ordering.+func sortSCCsByMin(indices []int, sccs [][]*objectEntry) {+ slices.SortStableFunc(indices, func(a, b int) int {+ return cmp.Compare(minSortKey(sccs[a]), minSortKey(sccs[b]))+ })+}++func minSortKey(scc []*objectEntry) string {+ if len(scc) == 0 {+ return ""+ }+ best := scc[0].sortKey()+ for _, o := range scc[1:] {+ if k := o.sortKey(); k < best {+ best = k+ }+ }+ return best+}++// installOne attempts a real install; on failure installs a pseudo at the+// same slot. All outcomes are recorded in l.degraded / l.trulyBroken; this+// function never returns an error to the caller because per-object failures+// are not fatal to the loader.+//+// Functions are a special case: when a function's real install fails, we+// deliberately leave the slot empty. Query span has a dedicated fallback for+// user functions (`tryMetadataFuncLookup` in query_span_omni.go) that+// reconstructs a synthetic proc from metadata at query time. Installing a+// text-backed pseudo function would crowd out that path, producing wrong+// overload resolution and losing body-level lineage. Tables/views/types are+// fine to pseudo-install because their lookups only need the object to exist.+func (l *catalogLoader) installOne(obj *objectEntry) {+ realErr := l.installReal(obj)+ if realErr == nil {+ return+ }+ l.degraded[obj.key()] = realErr+ if obj.kind == kindFunction {+ // No pseudo — rely on tryMetadataFuncLookup at query time.+ return+ }+ if pErr := l.installPseudo(obj); pErr != nil {+ l.trulyBroken[obj.key()] = pErr+ }+}++// installReal translates the object's metadata into an AST and calls the+// matching omni DefineX / ExecCreateTableAs / CreateFunctionStmt. Returns+// the first error encountered.+func (l *catalogLoader) installReal(obj *objectEntry) error {+ switch obj.kind {+ case kindSchema:+ err := l.cat.CreateSchemaCommand(&ast.CreateSchemaStmt{+ Schemaname: obj.schema,+ })+ // Built-in schemas (public, pg_catalog, pg_toast) are preloaded by+ // catalog.New(). Treating duplicate-schema as success keeps the+ // loader's "every metadata object is present post-Load" invariant+ // without needing a hardcoded exclusion list.+ var cErr *catalog.Error+ if errors.As(err, &cErr) && cErr.Code == catalog.CodeDuplicateSchema {+ return nil+ }+ return err+ case kindEnum:+ return l.cat.DefineEnum(buildCreateEnumStmt(obj.schema, obj.enumMeta))+ case kindTable:+ stmt, err := buildCreateStmt(obj.schema, obj.tableMeta)+ if err != nil {+ return err+ }+ return l.cat.DefineRelation(stmt, 'r')+ case kindView:+ stmt, err := buildViewStmt(obj.schema, obj.viewMeta)+ if err != nil {+ return err+ }+ return l.cat.DefineView(stmt)+ case kindMatView:+ stmt, err := buildCreateTableAsStmt(obj.schema, obj.matViewMeta)+ if err != nil {+ return err+ }+ return l.cat.ExecCreateTableAs(stmt)+ case kindFunction:+ stmt, err := buildCreateFunctionStmt(obj.schema, obj.funcMeta)+ if err != nil {+ return err+ }+ return l.cat.CreateFunctionStmt(stmt)+ }+ return errors.Errorf("unknown object kind %d", obj.kind)+}++// installPseudo builds and installs the pseudo variant of an object. Every+// pseudo form is text-backed and free of user-object dependencies, so this+// function essentially never fails; the few error paths exist for defense in+// depth.+func (l *catalogLoader) installPseudo(obj *objectEntry) error {+ switch obj.kind {+ case kindSchema:+ // Schemas have no pseudo form — they either exist or they don't.+ // If CreateSchemaCommand failed, treat as trulyBroken.+ return errors.New("schema has no pseudo form")+ case kindEnum:+ return l.cat.DefineEnum(pseudoCreateEnumStmt(obj.schema, obj.name))+ case kindTable:+ cols := columnNamesFromTableMetadata(obj.tableMeta)+ return l.cat.DefineRelation(pseudoCreateTableStmt(obj.schema, obj.name, cols), 'r')+ case kindView:+ cols := viewColumnNames(obj.viewMeta)+ stmt, err := pseudoViewStmt(obj.schema, obj.name, cols)+ if err != nil {+ return err+ }+ return l.cat.DefineView(stmt)+ case kindMatView:+ cols := matviewColumnNames(obj.matViewMeta)+ stmt, err := pseudoCreateTableAsStmt(obj.schema, obj.name, cols)+ if err != nil {+ return err+ }+ return l.cat.ExecCreateTableAs(stmt)+ case kindFunction:+ argCount := functionArgCountFromSignature(obj.funcMeta.Signature)+ return l.cat.CreateFunctionStmt(pseudoCreateFunctionStmt(obj.schema, obj.name, argCount))+ }+ return errors.Errorf("unknown object kind %d", obj.kind)+}++// columnNamesFromTableMetadata returns the deduplicated column names of a+// table's metadata, in order, for use as pseudo column names.+func columnNamesFromTableMetadata(tbl *storepb.TableMetadata) []string {+ if tbl == nil {+ return nil+ }+ seen := make(map[string]bool, len(tbl.Columns))+ out := make([]string, 0, len(tbl.Columns))+ for _, col := range tbl.Columns {+ if col.Name == "" || seen[col.Name] {+ continue+ }+ seen[col.Name] = true+ out = append(out, col.Name)+ }+ return out+}++// viewColumnNames returns the pseudo column-name list for a view. Prefers+// explicit Columns (when sync populated them); falls back to the set of+// dependency_columns.column values.+func viewColumnNames(v *storepb.ViewMetadata) []string {+ if v == nil {+ return nil+ }+ if len(v.Columns) > 0 {+ seen := make(map[string]bool, len(v.Columns))+ out := make([]string, 0, len(v.Columns))+ for _, col := range v.Columns {+ if col.Name == "" || seen[col.Name] {+ continue+ }+ seen[col.Name] = true+ out = append(out, col.Name)+ }+ return out+ }+ seen := make(map[string]bool, len(v.DependencyColumns))+ out := make([]string, 0, len(v.DependencyColumns))+ for _, dc := range v.DependencyColumns {+ if dc.Column == "" || seen[dc.Column] {+ continue+ }+ seen[dc.Column] = true+ out = append(out, dc.Column)+ }+ return out+}++// matviewColumnNames returns pseudo column names for a materialized view.+// MaterializedViewMetadata does not carry a Columns field, so we fall back+// to dependency_columns.+func matviewColumnNames(m *storepb.MaterializedViewMetadata) []string {+ if m == nil {+ return nil+ }+ seen := make(map[string]bool, len(m.DependencyColumns))+ out := make([]string, 0, len(m.DependencyColumns))+ for _, dc := range m.DependencyColumns {+ if dc.Column == "" || seen[dc.Column] {+ continue+ }+ seen[dc.Column] = true+ out = append(out, dc.Column)+ }+ return out+}diff --git a/backend/plugin/parser/pg/query_span_loader_builders.go b/backend/plugin/parser/pg/query_span_loader_builders.gonew file mode 100644index 0000000000..dc71d55dd0--- /dev/null+++ b/backend/plugin/parser/pg/query_span_loader_builders.go@@ -0,0 +1,245 @@+package pg++import (+ "strings"++ "github.com/pkg/errors"++ "github.com/bytebase/omni/pg/ast"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// This file translates storepb metadata into omni AST nodes. Each build*+// function is deterministic, allocates fresh nodes, and does not touch the+// catalog. Builders may return errors for inputs the cheat parser rejects;+// the loader catches those errors and installs a pseudo at the same slot.++// buildCreateEnumStmt translates EnumTypeMetadata + schema into a+// CreateEnumStmt. Enum values are preserved as plain string literals.+func buildCreateEnumStmt(schema string, enum *storepb.EnumTypeMetadata) *ast.CreateEnumStmt {+ vals := make([]ast.Node, 0, len(enum.Values))+ for _, v := range enum.Values {+ vals = append(vals, &ast.String{Str: v})+ }+ return &ast.CreateEnumStmt{+ TypeName: qualifiedNameList(schema, enum.Name),+ Vals: &ast.List{Items: vals},+ }+}++// buildCreateStmt translates TableMetadata into a CreateStmt with real+// column types. Column types are parsed via typeNameFromString; an error on+// any single column propagates back so the loader can pseudo the whole+// relation rather than install a partially-typed catalog entry.+//+// Columns with empty names are skipped.+func buildCreateStmt(schema string, table *storepb.TableMetadata) (*ast.CreateStmt, error) {+ items := make([]ast.Node, 0, len(table.Columns))+ for _, col := range table.Columns {+ if col.Name == "" {+ continue+ }+ if col.Type == "" {+ return nil, errors.Errorf("column %q: empty type", col.Name)+ }+ tn, err := typeNameFromString(col.Type)+ if err != nil {+ return nil, errors.Wrapf(err, "column %q", col.Name)+ }+ items = append(items, &ast.ColumnDef{+ Colname: col.Name,+ TypeName: tn,+ IsNotNull: !col.Nullable,+ })+ }+ return &ast.CreateStmt{+ Relation: &ast.RangeVar{+ Schemaname: schema,+ Relname: table.Name,+ Relpersistence: 'p',+ },+ TableElts: &ast.List{Items: items},+ }, nil+}++// buildViewStmt translates ViewMetadata into a ViewStmt. The view body is+// parsed via ParsePg; the result must be a *ast.SelectStmt (not a RawStmt+// wrapper) because omni's DefineView type-asserts directly.+//+// If the body is empty or unparseable, returns an error; the loader will+// pseudo the view using dependency_columns metadata.+func buildViewStmt(schema string, view *storepb.ViewMetadata) (*ast.ViewStmt, error) {+ if view.Definition == "" {+ return nil, errors.New("empty view definition")+ }+ sel, err := parseSelectBody(view.Definition)+ if err != nil {+ return nil, errors.Wrapf(err, "view %q body", view.Name)+ }+ return &ast.ViewStmt{+ View: &ast.RangeVar{+ Schemaname: schema,+ Relname: view.Name,+ Relpersistence: 'p',+ },+ Query: sel,+ }, nil+}++// buildCreateTableAsStmt translates MaterializedViewMetadata into a+// CreateTableAsStmt with Objtype OBJECT_MATVIEW. Same SelectStmt contract+// as buildViewStmt: body must parse to *ast.SelectStmt.+func buildCreateTableAsStmt(schema string, mv *storepb.MaterializedViewMetadata) (*ast.CreateTableAsStmt, error) {+ if mv.Definition == "" {+ return nil, errors.New("empty matview definition")+ }+ sel, err := parseSelectBody(mv.Definition)+ if err != nil {+ return nil, errors.Wrapf(err, "matview %q body", mv.Name)+ }+ return &ast.CreateTableAsStmt{+ Query: sel,+ Objtype: ast.OBJECT_MATVIEW,+ Into: &ast.IntoClause{+ Rel: &ast.RangeVar{+ Schemaname: schema,+ Relname: mv.Name,+ Relpersistence: 'p',+ },+ },+ }, nil+}++// buildCreateFunctionStmt translates FunctionMetadata into a+// *ast.CreateFunctionStmt. The primary path parses the function's full+// Definition (an `omni`-compatible CREATE FUNCTION/PROCEDURE statement),+// which preserves parameter types, return-type signatures (including+// RETURNS TABLE / RETURNS SETOF), and the dollar-quoted body verbatim.+//+// When the definition string is missing or cannot be parsed as a+// CreateFunctionStmt, we fall back to a minimal text-typed shape+// reconstructed from the signature string. The fallback loses return-type+// fidelity but keeps analyzer name resolution working for queries that+// reference the function by name.+func buildCreateFunctionStmt(schema string, fn *storepb.FunctionMetadata) (*ast.CreateFunctionStmt, error) {+ if fn.Definition != "" {+ stmts, err := ParsePg(fn.Definition)+ if err == nil && len(stmts) == 1 {+ node := stmts[0].AST+ if raw, ok := node.(*ast.RawStmt); ok {+ node = raw.Stmt+ }+ if parsed, ok := node.(*ast.CreateFunctionStmt); ok {+ return parsed, nil+ }+ }+ }+ return buildCreateFunctionStmtFromSignature(schema, fn)+}++// buildCreateFunctionStmtFromSignature is the minimal-shape fallback used+// when the full definition is missing or unparseable. Parameters and return+// type collapse to text; the body is a trivial `SELECT NULL::text`.+func buildCreateFunctionStmtFromSignature(schema string, fn *storepb.FunctionMetadata) (*ast.CreateFunctionStmt, error) {+ argTypes, err := parseFunctionSignatureArgTypes(fn.Signature)+ if err != nil {+ return nil, errors.Wrapf(err, "function %q signature %q", fn.Name, fn.Signature)+ }+ params := make([]ast.Node, 0, len(argTypes))+ for i, argType := range argTypes {+ tn, err := typeNameFromString(argType)+ if err != nil {+ return nil, errors.Wrapf(err, "function %q arg %d", fn.Name, i)+ }+ params = append(params, &ast.FunctionParameter{+ ArgType: tn,+ Mode: ast.FUNC_PARAM_IN,+ })+ }+ stmt := &ast.CreateFunctionStmt{+ Funcname: qualifiedNameList(schema, fn.Name),+ ReturnType: pseudoTextTypeName(),+ }+ if len(params) > 0 {+ stmt.Parameters = &ast.List{Items: params}+ }+ stmt.Options = &ast.List{Items: []ast.Node{+ &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},+ &ast.DefElem{+ Defname: "as",+ Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT NULL::text"}}},+ },+ }}+ return stmt, nil+}++// parseSelectBody parses a SQL string expected to be a single SELECT+// statement and returns the *ast.SelectStmt. Rejects empty input, multiple+// statements, and non-SELECT top-level nodes. Unwraps a RawStmt wrapper if+// ParsePg returns one.+func parseSelectBody(sql string) (*ast.SelectStmt, error) {+ stmts, err := ParsePg(sql)+ if err != nil {+ return nil, errors.Wrap(err, "parse")+ }+ if len(stmts) != 1 {+ return nil, errors.Errorf("expected 1 statement, got %d", len(stmts))+ }+ node := stmts[0].AST+ if raw, ok := node.(*ast.RawStmt); ok {+ node = raw.Stmt+ }+ sel, ok := node.(*ast.SelectStmt)+ if !ok {+ return nil, errors.Errorf("expected SelectStmt, got %T", node)+ }+ return sel, nil+}++// parseFunctionSignatureArgTypes extracts the comma-separated argument types+// from a signature like "fn(integer, text)" or "fn(numeric(10,2), boolean)".+// An empty parameter list returns an empty slice.+func parseFunctionSignatureArgTypes(signature string) ([]string, error) {+ open := strings.Index(signature, "(")+ if open < 0 {+ return nil, nil+ }+ closeIdx := strings.LastIndex(signature, ")")+ if closeIdx < 0 || closeIdx <= open {+ return nil, errors.Errorf("signature %q: unbalanced parentheses", signature)+ }+ return splitTopLevelCommas(signature[open+1 : closeIdx]), nil+}++// splitTopLevelCommas splits a string on commas that are not nested inside+// parentheses. Used for function signature argument lists so that types like+// `numeric(10,2)` are not split.+func splitTopLevelCommas(s string) []string {+ s = strings.TrimSpace(s)+ if s == "" {+ return nil+ }+ var parts []string+ depth := 0+ start := 0+ for i := 0; i < len(s); i++ {+ switch s[i] {+ case '(':+ depth+++ case ')':+ if depth > 0 {+ depth--+ }+ case ',':+ if depth == 0 {+ parts = append(parts, strings.TrimSpace(s[start:i]))+ start = i + 1+ }+ default:+ // other bytes are part of the current segment+ }+ }+ parts = append(parts, strings.TrimSpace(s[start:]))+ return parts+}diff --git a/backend/plugin/parser/pg/query_span_loader_classify.go b/backend/plugin/parser/pg/query_span_loader_classify.gonew file mode 100644index 0000000000..e10d1d6c71--- /dev/null+++ b/backend/plugin/parser/pg/query_span_loader_classify.go@@ -0,0 +1,83 @@+package pg++import (+ "github.com/pkg/errors"++ "github.com/bytebase/omni/pg/catalog"+)++// fallbackReason classifies why analyzer analysis fell through to+// extractFallbackColumns / fallbackExtractLineage. It is a pure classification+// of the error from omni's AnalyzeSelectStmt, used by tests to assert which+// branch a given query takes.+//+// This is NOT a runtime gate, a metric, or a telemetry signal. It exists so+// that unit tests can distinguish "pseudo-induced degradation" from+// "undefined reference" from "analyzer does not support this feature", and so+// that debugging from logs carries a meaningful category.+type fallbackReason int++const (+ reasonNone fallbackReason = iota+ // reasonExpectedPseudoSemantic: analyzer failed because a pseudo-installed+ // object looks like text in place of its original user type. Common forms:+ // UndefinedFunction (overload signature no longer matches), DatatypeMismatch+ // (operator resolution against text-backed pseudo), AmbiguousFunction /+ // AmbiguousColumn (candidate pile from pseudo-widening).+ reasonExpectedPseudoSemantic+ // reasonUndefinedReference: analyzer could not find a relation, column,+ // schema, or object. This can be a loader bug (we should have installed+ // it) OR a legitimate user error (query typos, stale metadata). Without a+ // structured ErrorIdent on *catalog.Error we cannot programmatically+ // distinguish the two; humans triage from logs.+ reasonUndefinedReference+ // reasonAnalyzerUnsupported: non-*catalog.Error internal analyzer failure+ // (returned as fmt.Errorf from analyze.go code paths that hit an+ // unsupported PG feature or a defensive check).+ reasonAnalyzerUnsupported+)++// classifyAnalyzeError buckets an omni AnalyzeSelectStmt error into one of+// the fallbackReason categories. Non-nil errors always return a non-None+// reason; a nil error returns reasonNone.+func classifyAnalyzeError(err error) fallbackReason {+ if err == nil {+ return reasonNone+ }+ var cErr *catalog.Error+ if !errors.As(err, &cErr) {+ return reasonAnalyzerUnsupported+ }+ switch cErr.Code {+ case catalog.CodeUndefinedFunction,+ catalog.CodeAmbiguousFunction,+ catalog.CodeDatatypeMismatch,+ catalog.CodeFeatureNotSupported,+ catalog.CodeAmbiguousColumn:+ return reasonExpectedPseudoSemantic+ case catalog.CodeUndefinedTable,+ catalog.CodeUndefinedColumn,+ catalog.CodeUndefinedObject,+ catalog.CodeUndefinedSchema:+ return reasonUndefinedReference+ default:+ return reasonAnalyzerUnsupported+ }+}++// String returns a short lowercase label suitable for test assertions and+// log lines.+func (r fallbackReason) String() string {+ switch r {+ case reasonNone:+ return "none"+ case reasonExpectedPseudoSemantic:+ return "expected_pseudo_semantic"+ case reasonUndefinedReference:+ return "undefined_reference"+ case reasonAnalyzerUnsupported:+ return "analyzer_unsupported"+ default:+ return "unknown"+ }+}diff --git a/backend/plugin/parser/pg/query_span_loader_integration_test.go b/backend/plugin/parser/pg/query_span_loader_integration_test.gonew file mode 100644index 0000000000..43d39ffec6--- /dev/null+++ b/backend/plugin/parser/pg/query_span_loader_integration_test.go@@ -0,0 +1,283 @@+package pg++import (+ "context"+ "testing"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// ---------------------------------------------------------------------------+// Catalog loader integration tests+// ---------------------------------------------------------------------------+//+// These tests drive the full GetQuerySpan pipeline (parse → initCatalog via+// catalogLoader → AnalyzeSelectStmt → extractLineage) against hand-built+// DatabaseSchemaMetadata fixtures that reproduce the bugs the loader is designed to+// prevent, plus the cascade-prevention story.+//+// Each test asserts one of:+// - A query against an unrelated object succeeds even when a bad object is+// present in the schema (BYT-9215 / BYT-9261 class, blast-radius).+// - A query against a broken-but-pseudo'd object still returns lineage.+// - A cascade chain (bad root → dependent table → dependent view) installs+// downstream objects as real, with query lineage intact.++func TestLoaderIntegration_BYT9215_BadQuotedIdentifier(t *testing.T) {+ // BYT-9215 class: a table name contains a character sequence the DDL+ // deparse-and-reparse loop chokes on. Under the old path this would kill+ // query span for the entire database. Under the loader, the bad table is+ // pseudo-installed and unrelated queries succeed.+ meta := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{+ // A table with a quoted identifier containing an apostrophe —+ // the kind of input BYT-9215 reported failing under Exec(ddl).+ // Real install should succeed (no DDL roundtrip), but even if+ // it didn't, pseudo would catch it.+ {+ Name: "'weird'table",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ },+ },+ // An unrelated healthy table.+ {+ Name: "accounts",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "email", Type: "text"},+ },+ },+ },+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, email FROM accounts`)+ if len(span.Results) != 2 {+ t.Fatalf("accounts query: got %d results, want 2", len(span.Results))+ }+ for _, r := range span.Results {+ if len(r.SourceColumns) == 0 {+ t.Errorf("result %q has empty sources (lineage lost)", r.Name)+ }+ }+}++func TestLoaderIntegration_BrokenEnumCascade(t *testing.T) {+ // Real failure chain: table references a user-defined type that does+ // not exist in metadata. Real install of the table fails (omni cannot+ // resolve the type). Pseudo install of the table succeeds with text+ // columns. Query against the table returns its metadata column names.+ meta := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{{+ Name: "tasks",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ // References an enum that is NOT declared in metadata —+ // buildCreateStmt will succeed (typeNameFromString works)+ // but DefineRelation fails (enum not in catalog).+ {Name: "status", Type: "public.nonexistent_enum"},+ {Name: "title", Type: "text"},+ },+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, status, title FROM tasks`)+ if len(span.Results) != 3 {+ t.Fatalf("tasks query: got %d results, want 3", len(span.Results))+ }+ wantNames := []string{"id", "status", "title"}+ for i, want := range wantNames {+ if i >= len(span.Results) {+ break+ }+ if span.Results[i].Name != want {+ t.Errorf("result[%d]: got %q, want %q", i, span.Results[i].Name, want)+ }+ if len(span.Results[i].SourceColumns) == 0 {+ t.Errorf("result[%d] (%s): empty sources", i, span.Results[i].Name)+ }+ }+}++func TestLoaderIntegration_BrokenRootTableAndHealthyNeighbor(t *testing.T) {+ // A query against a healthy table must succeed even when an unrelated+ // table in the same schema references a broken type. This is the core+ // blast-radius claim: one bad object does not poison all queries.+ meta := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {+ Name: "broken",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "bad", Type: "public.nonexistent_type"},+ },+ },+ {+ Name: "healthy",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "label", Type: "text"},+ },+ },+ },+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, label FROM healthy`)+ if len(span.Results) != 2 {+ t.Fatalf("healthy query: got %d results, want 2", len(span.Results))+ }+ for _, r := range span.Results {+ if len(r.SourceColumns) == 0 {+ t.Errorf("result %q lost lineage because of unrelated broken table", r.Name)+ }+ }+}++func TestLoaderIntegration_ViewOverBrokenTableStillResolves(t *testing.T) {+ // Chain: enum missing → table T references it (degrades to pseudo) →+ // view V on T installs real (against pseudo T) → query on V resolves.+ meta := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{{+ Name: "records",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "status", Type: "public.nonexistent_enum"},+ {Name: "title", Type: "text"},+ },+ }},+ Views: []*storepb.ViewMetadata{{+ Name: "records_view",+ Definition: "SELECT id, title, status FROM records",+ DependencyColumns: []*storepb.DependencyColumn{+ {Schema: "public", Table: "records", Column: "id"},+ {Schema: "public", Table: "records", Column: "title"},+ {Schema: "public", Table: "records", Column: "status"},+ },+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, title, status FROM records_view`)+ if len(span.Results) != 3 {+ t.Fatalf("view query: got %d results, want 3", len(span.Results))+ }+ for i, r := range span.Results {+ if len(r.SourceColumns) == 0 {+ t.Errorf("result[%d] (%s) lost lineage", i, r.Name)+ }+ }+}++func TestLoaderIntegration_SimpleHealthyPath(t *testing.T) {+ // Baseline sanity: a clean schema must produce exact lineage down to+ // (schema, table, column) — no degraded flags.+ meta := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{{+ Name: "users",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "email", Type: "text"},+ {Name: "created_at", Type: "timestamp with time zone"},+ },+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, email FROM users WHERE id > 0`)+ if len(span.Results) != 2 {+ t.Fatalf("got %d results, want 2", len(span.Results))+ }+ mustHaveExactSource(t, span.Results[0], "users", "id")+ mustHaveExactSource(t, span.Results[1], "users", "email")+}++func TestLoaderIntegration_EnumWorksWhenDeclared(t *testing.T) {+ // When an enum IS declared in metadata, the table installs as real and+ // enum-typed columns resolve through their real type. This is the+ // positive control for TestLoaderIntegration_BrokenEnumCascade.+ meta := &storepb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ EnumTypes: []*storepb.EnumTypeMetadata{{+ Name: "task_status",+ Values: []string{"pending", "running", "done"},+ }},+ Tables: []*storepb.TableMetadata{{+ Name: "tasks",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "status", Type: "public.task_status"},+ },+ }},+ }},+ }++ span := mustGetQuerySpan(t, meta, `SELECT id, status FROM tasks WHERE status = 'pending'`)+ if len(span.Results) != 2 {+ t.Fatalf("got %d results, want 2", len(span.Results))+ }+ mustHaveExactSource(t, span.Results[0], "tasks", "id")+ mustHaveExactSource(t, span.Results[1], "tasks", "status")+}++// ---------- helpers ----------++// loaderTestDB / loaderTestSchema are the fixed database/schema names+// every catalog-loader integration test uses; keeping them as constants keeps the call+// sites terse and satisfies the unparam linter.+const (+ loaderTestDB = "db"+ loaderTestSchema = "public"+)++func mustGetQuerySpan(t *testing.T, meta *storepb.DatabaseSchemaMetadata, sql string) *base.QuerySpan {+ t.Helper()+ getter, lister := buildMockDatabaseMetadataGetter([]*storepb.DatabaseSchemaMetadata{meta})+ span, err := GetQuerySpan(context.TODO(), base.GetQuerySpanContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ }, base.Statement{Text: sql}, loaderTestDB, "", false)+ if err != nil {+ t.Fatalf("GetQuerySpan(%q): %v", sql, err)+ }+ if span == nil {+ t.Fatalf("GetQuerySpan(%q): nil span", sql)+ }+ return span+}++func mustHaveExactSource(t *testing.T, result base.QuerySpanResult, table, column string) {+ t.Helper()+ want := base.ColumnResource{+ Database: loaderTestDB,+ Schema: loaderTestSchema,+ Table: table,+ Column: column,+ }+ for src := range result.SourceColumns {+ if src == want {+ return+ }+ }+ t.Errorf("result %q missing source %+v; have %+v", result.Name, want, result.SourceColumns)+}diff --git a/backend/plugin/parser/pg/query_span_loader_poc_test.go b/backend/plugin/parser/pg/query_span_loader_poc_test.gonew file mode 100644index 0000000000..389dc9246f--- /dev/null+++ b/backend/plugin/parser/pg/query_span_loader_poc_test.go@@ -0,0 +1,2527 @@+package pg++// v5 PoC — AST-only catalog population.+//+// This PoC demonstrates that we can:+// 1. Construct a *ast.CreateStmt directly in Go code (no SQL text, no parser).+// 2. Call catalog.DefineRelation on it to install a table into an omni catalog.+// 3. Parse a SELECT query against that table.+// 4. Call AnalyzeSelectStmt and get correct column lineage.+//+// The entire "table install" half of the pipeline never touches pgparser.Parse,+// which is the source of BYT-9215 / BYT-9261 class failures. Only the SELECT+// statement itself is parsed.+//+// If this PoC passes, v5 ("lazy per-object AST install") is the recommended+// architecture for PG query span and supersedes v2/v3/v4 plans.++import (+ "fmt"+ "strings"+ "testing"++ "github.com/pkg/errors"++ "github.com/bytebase/omni/pg/ast"+ "github.com/bytebase/omni/pg/catalog"+)++// mustTypeName is a test-only shim around the production typeNameFromString.+// It fails the test on any parse error; production code uses typeNameFromString+// directly and handles the error.+func mustTypeName(t *testing.T, typeStr string) *ast.TypeName {+ t.Helper()+ tn, err := typeNameFromString(typeStr)+ if err != nil {+ t.Fatalf("typeNameFromString(%q): %v", typeStr, err)+ }+ return tn+}++// typeName builds an unqualified PG type name AST node.+// For built-in types, use the PG internal name: int -> int4, text -> text,+// bigint -> int8, varchar -> varchar, timestamp -> timestamp.+func typeName(name string) *ast.TypeName {+ return &ast.TypeName{+ Names: &ast.List{Items: []ast.Node{&ast.String{Str: name}}},+ Typemod: -1,+ }+}++// columnDef builds a minimal ColumnDef AST node.+func columnDef(name string, typ *ast.TypeName, notNull bool) *ast.ColumnDef {+ return &ast.ColumnDef{+ Colname: name,+ TypeName: typ,+ IsNotNull: notNull,+ }+}++// buildCreateTableStmt constructs a *ast.CreateStmt for a simple table,+// purely from Go code with no SQL text. This is the shape that a v5+// catalog loader would produce from bytebase metadata.+//+// parameter signals intent and keeps the helper ready for cross-schema tests.+//+//nolint:unparam // schema is always "public" in these PoCs; keeping the+func buildCreateTableStmt(schema, name string, cols []*ast.ColumnDef) *ast.CreateStmt {+ nodes := make([]ast.Node, len(cols))+ for i, c := range cols {+ nodes[i] = c+ }+ return &ast.CreateStmt{+ Relation: &ast.RangeVar{+ Schemaname: schema,+ Relname: name,+ Relpersistence: 'p',+ },+ TableElts: &ast.List{Items: nodes},+ }+}++// TestLoaderPoC_AstOnlyTableInstallAndAnalyze is the main PoC.+//+// Run with:+//+// go test -v -count=1 -run TestLoaderPoC github.com/bytebase/bytebase/backend/plugin/parser/pg+func TestLoaderPoC_AstOnlyTableInstallAndAnalyze(t *testing.T) {+ // Step 1: fresh catalog. public / pg_catalog / pg_toast are preloaded.+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Step 2: hand-construct CREATE TABLE public.plan (+ // id int NOT NULL,+ // title text,+ // status text,+ // creator_id int+ // ) — no SQL string involved.+ planStmt := buildCreateTableStmt("public", "plan", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("title", typeName("text"), false),+ columnDef("status", typeName("text"), false),+ columnDef("creator_id", typeName("int4"), false),+ })++ // Step 3: install via DefineRelation. Second arg 'r' = regular table.+ if err := cat.DefineRelation(planStmt, 'r'); err != nil {+ t.Fatalf("DefineRelation(plan) failed: %v", err)+ }++ // Step 4: install a second table to test JOINs too.+ principalStmt := buildCreateTableStmt("public", "principal", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("name", typeName("text"), false),+ columnDef("email", typeName("text"), false),+ })+ if err := cat.DefineRelation(principalStmt, 'r'); err != nil {+ t.Fatalf("DefineRelation(principal) failed: %v", err)+ }++ // Step 5: verify catalog state — both tables resolvable by name.+ if rel := cat.GetRelation("public", "plan"); rel == nil {+ t.Fatal("GetRelation(public, plan) returned nil after DefineRelation")+ } else if len(rel.Columns) != 4 {+ t.Errorf("plan column count: got %d, want 4", len(rel.Columns))+ }+ if rel := cat.GetRelation("public", "principal"); rel == nil {+ t.Fatal("GetRelation(public, principal) returned nil after DefineRelation")+ }++ // Step 6: parse a SELECT query that touches both tables.+ // The SELECT parser is used here — that's expected and safe (it's not+ // the DDL parser that breaks in BYT-9215/9261).+ stmts, err := ParsePg(`+ SELECT p.id, p.title, principal.name+ FROM plan p+ JOIN principal ON principal.id = p.creator_id+ WHERE p.status = 'open'+ `)+ if err != nil {+ t.Fatalf("ParsePg failed: %v", err)+ }+ if len(stmts) != 1 {+ t.Fatalf("expected 1 statement, got %d", len(stmts))+ }+ selStmt, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ t.Fatalf("expected *ast.SelectStmt, got %T", stmts[0].AST)+ }++ // Step 7: THE KEY CALL — analyze the SELECT against the hand-installed+ // catalog. If this returns a non-nil Query with a TargetList matching+ // the three SELECT columns, v5 architecture is viable.+ query, err := cat.AnalyzeSelectStmt(selStmt)+ if err != nil {+ t.Fatalf("AnalyzeSelectStmt failed: %v", err)+ }+ if query == nil {+ t.Fatal("AnalyzeSelectStmt returned nil query")+ }++ // Step 8: assert lineage is sane — 3 target columns.+ if got := len(query.TargetList); got != 3 {+ t.Errorf("TargetList length: got %d, want 3", got)+ for i, te := range query.TargetList {+ t.Logf(" [%d] %s", i, te.ResName)+ }+ }++ // Step 9: assert each target is a VarExpr (plain column reference)+ // pointing at the expected column names.+ wantNames := []string{"id", "title", "name"}+ for i, want := range wantNames {+ if i >= len(query.TargetList) {+ break+ }+ te := query.TargetList[i]+ if te.ResName != want {+ t.Errorf("TargetList[%d].ResName: got %q, want %q", i, te.ResName, want)+ }+ if _, isVar := te.Expr.(*catalog.VarExpr); !isVar {+ t.Errorf("TargetList[%d].Expr: got %T, want *VarExpr", i, te.Expr)+ }+ }++ // Step 10: verify JoinTree.Quals is analyzed (WHERE was walked).+ // We don't assert exact shape here — just that analysis didn't bail.+ if query.JoinTree == nil {+ t.Error("JoinTree is nil — analyzer did not build a join tree")+ } else if query.JoinTree.Quals == nil {+ t.Error("JoinTree.Quals is nil — WHERE clause was not analyzed")+ }++ t.Logf("v5 PoC passed: hand-constructed CreateStmt → DefineRelation → " ++ "AnalyzeSelectStmt pipeline works end-to-end with no DDL text.")+}++// TestLoaderPoC_BrokenAstIsIsolated demonstrates blast-radius containment:+// installing a table with an unresolvable type fails locally, and queries+// against OTHER tables continue to work. This is the core fidelity claim+// v5 makes over the current whole-schema DDL init path.+func TestLoaderPoC_BrokenAstIsIsolated(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Install a good table.+ goodStmt := buildCreateTableStmt("public", "good", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("label", typeName("text"), false),+ })+ if err := cat.DefineRelation(goodStmt, 'r'); err != nil {+ t.Fatalf("DefineRelation(good) failed: %v", err)+ }++ // Try to install a broken table — column type does not exist.+ brokenStmt := buildCreateTableStmt("public", "broken", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("payload", typeName("nonexistent_type_xyz"), false),+ })+ err := cat.DefineRelation(brokenStmt, 'r')+ if err == nil {+ t.Log("note: DefineRelation accepted unresolvable type without error")+ t.Log(" this is informational — the analyzer may still reject it at query time")+ } else {+ t.Logf("DefineRelation(broken) rejected as expected: %v", err)+ }++ // Key assertion: the good table is still queryable, regardless of+ // whether the broken table was accepted.+ if rel := cat.GetRelation("public", "good"); rel == nil {+ t.Fatal("good table was lost after broken install attempt")+ }++ stmts, err := ParsePg(`SELECT id, label FROM good WHERE id = 1`)+ if err != nil {+ t.Fatalf("ParsePg failed: %v", err)+ }+ selStmt, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ t.Fatalf("expected *ast.SelectStmt, got %T", stmts[0].AST)+ }++ query, err := cat.AnalyzeSelectStmt(selStmt)+ if err != nil {+ t.Fatalf("AnalyzeSelectStmt on good table failed after broken install: %v", err)+ }+ if query == nil || len(query.TargetList) != 2 {+ t.Errorf("expected 2 target columns on good table query, got %v", query)+ }++ t.Log("v5 PoC blast-radius isolation: broken table did not break queries " ++ "against unrelated good table.")+}++// ---------- Risk scenario 1: complex typmods (numeric(10,2)) ----------++func TestLoaderPoC_ComplexTypmod(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Build a column with numeric(10,2) via the cheat parser.+ stmt := buildCreateTableStmt("public", "orders", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),+ columnDef("ts", mustTypeName(t, "timestamp(3) with time zone"), false),+ })+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation with complex typmods failed: %v", err)+ }++ rel := cat.GetRelation("public", "orders")+ if rel == nil || len(rel.Columns) != 3 {+ t.Fatalf("orders install incomplete: %+v", rel)+ }++ // Query the table — analyzer must resolve each column.+ q := mustAnalyze(t, cat, `SELECT id, amount, ts FROM orders WHERE amount > 0`)+ if len(q.TargetList) != 3 {+ t.Errorf("TargetList: got %d, want 3", len(q.TargetList))+ }+ t.Log("complex typmod PoC: numeric(10,2) and timestamp(3) with time zone resolve via cheat parser")+}++// ---------- Risk scenario 2: array type ----------++func TestLoaderPoC_ArrayType(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ stmt := buildCreateTableStmt("public", "tagged", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("tags", mustTypeName(t, "text[]"), false),+ columnDef("scores", mustTypeName(t, "int4[]"), false),+ })+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation with array columns failed: %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT id, tags FROM tagged`)+ if len(q.TargetList) != 2 {+ t.Errorf("TargetList: got %d, want 2", len(q.TargetList))+ }+ t.Log("array type PoC: text[] and int4[] resolve via cheat parser")+}++// ---------- Risk scenario 3: SELECT * expansion ----------++func TestLoaderPoC_StarExpansion(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ if err := cat.DefineRelation(buildCreateTableStmt("public", "plan", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("title", typeName("text"), false),+ columnDef("status", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("install: %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT * FROM plan`)+ // With a 3-column table, SELECT * should expand to 3 target entries.+ if len(q.TargetList) != 3 {+ t.Errorf("SELECT * expansion: got %d targets, want 3", len(q.TargetList))+ }+ wantCols := []string{"id", "title", "status"}+ for i, want := range wantCols {+ if i >= len(q.TargetList) {+ break+ }+ if q.TargetList[i].ResName != want {+ t.Errorf("target[%d]: got %q, want %q", i, q.TargetList[i].ResName, want)+ }+ }+ t.Log("SELECT * expansion PoC: analyzer expands star against hand-installed columns")+}++// ---------- Risk scenario 4: CTE (WITH clause) ----------++func TestLoaderPoC_CTE(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ if err := cat.DefineRelation(buildCreateTableStmt("public", "users", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("name", typeName("text"), false),+ columnDef("dept", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("install: %v", err)+ }++ q := mustAnalyze(t, cat, `+ WITH active AS (+ SELECT id, name FROM users WHERE dept = 'eng'+ )+ SELECT id, name FROM active+ `)+ if len(q.TargetList) != 2 {+ t.Errorf("CTE analyze: got %d targets, want 2", len(q.TargetList))+ }+ t.Log("CTE PoC: WITH clause analyzed against hand-installed table")+}++// ---------- Risk scenario 5: correlated subquery ----------++func TestLoaderPoC_CorrelatedSubquery(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ if err := cat.DefineRelation(buildCreateTableStmt("public", "dept", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("name", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("install dept: %v", err)+ }+ if err := cat.DefineRelation(buildCreateTableStmt("public", "emp", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("dept_id", typeName("int4"), false),+ columnDef("salary", typeName("int4"), false),+ }), 'r'); err != nil {+ t.Fatalf("install emp: %v", err)+ }++ q := mustAnalyze(t, cat, `+ SELECT d.name,+ (SELECT MAX(e.salary) FROM emp e WHERE e.dept_id = d.id) AS max_sal+ FROM dept d+ `)+ if len(q.TargetList) != 2 {+ t.Errorf("correlated: got %d targets, want 2", len(q.TargetList))+ }+ t.Log("correlated subquery PoC: outer scope d.id resolved from inner subquery")+}++// ---------- Risk scenario 6: user enum type in column ----------++func TestLoaderPoC_UserEnumType(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Step 1: install the enum via DefineEnum with hand-constructed stmt.+ enumStmt := &ast.CreateEnumStmt{+ TypeName: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "task_status"},+ }},+ Vals: &ast.List{Items: []ast.Node{+ &ast.String{Str: "pending"},+ &ast.String{Str: "running"},+ &ast.String{Str: "done"},+ }},+ }+ if err := cat.DefineEnum(enumStmt); err != nil {+ t.Fatalf("DefineEnum failed: %v", err)+ }++ // Step 2: install a table whose column uses the enum.+ // We go through the cheat parser so the TypeName carries the qualified+ // name format omni expects for user types.+ stmt := buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("status", mustTypeName(t, "public.task_status"), false),+ })+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation with enum column failed: %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT id, status FROM tasks WHERE status = 'pending'`)+ if len(q.TargetList) != 2 {+ t.Errorf("enum-col query: got %d targets, want 2", len(q.TargetList))+ }+ t.Log("user enum PoC: DefineEnum + DefineRelation with enum-typed column + WHERE filter works")+}++// ---------- Risk scenario 7: view via DefineView with parsed body ----------++func TestLoaderPoC_DefineViewWithParsedBody(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Install an underlying table.+ if err := cat.DefineRelation(buildCreateTableStmt("public", "plan", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("title", typeName("text"), false),+ columnDef("creator_id", typeName("int4"), false),+ }), 'r'); err != nil {+ t.Fatalf("install plan: %v", err)+ }++ // View body is stored as text in bytebase metadata. Parse it once.+ viewBody := `SELECT id, title FROM plan WHERE creator_id = 1`+ stmts, err := ParsePg(viewBody)+ if err != nil {+ t.Fatalf("parse view body: %v", err)+ }+ selStmt, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ t.Fatalf("view body: expected SelectStmt, got %T", stmts[0].AST)+ }++ // Construct ViewStmt and install via DefineView.+ viewStmt := &ast.ViewStmt{+ View: &ast.RangeVar{+ Schemaname: "public",+ Relname: "plan_mine",+ Relpersistence: 'p',+ },+ Query: selStmt,+ }+ if err := cat.DefineView(viewStmt); err != nil {+ t.Fatalf("DefineView failed: %v", err)+ }++ rel := cat.GetRelation("public", "plan_mine")+ if rel == nil {+ t.Fatal("view not installed")+ }+ if rel.RelKind != 'v' {+ t.Errorf("RelKind: got %c, want 'v'", rel.RelKind)+ }++ // Query the view. analyzer should be able to expand the view.+ q := mustAnalyze(t, cat, `SELECT id, title FROM plan_mine`)+ if len(q.TargetList) != 2 {+ t.Errorf("query on view: got %d targets, want 2", len(q.TargetList))+ }+ t.Log("DefineView PoC: parsed-body ViewStmt + DefineView + query on view works")+}++// ---------- Risk scenario 8: nested view (view on view) ----------++func TestLoaderPoC_NestedView(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("user_id", typeName("int4"), false),+ columnDef("amount", typeName("int4"), false),+ }), 'r'); err != nil {+ t.Fatalf("install orders: %v", err)+ }++ // View 1: orders_by_user+ body1 := `SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id`+ sel1 := mustParseSelect(t, body1)+ if err := cat.DefineView(&ast.ViewStmt{+ View: &ast.RangeVar{Schemaname: "public", Relname: "orders_by_user", Relpersistence: 'p'},+ Query: sel1,+ }); err != nil {+ t.Fatalf("DefineView v1: %v", err)+ }++ // View 2: top spenders — references view 1.+ body2 := `SELECT user_id, total FROM orders_by_user WHERE total > 100`+ sel2 := mustParseSelect(t, body2)+ if err := cat.DefineView(&ast.ViewStmt{+ View: &ast.RangeVar{Schemaname: "public", Relname: "top_spenders", Relpersistence: 'p'},+ Query: sel2,+ }); err != nil {+ t.Fatalf("DefineView v2 (nested): %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT user_id, total FROM top_spenders`)+ if len(q.TargetList) != 2 {+ t.Errorf("nested view query: got %d targets, want 2", len(q.TargetList))+ }+ t.Log("nested view PoC: view-on-view resolves correctly via DefineView chain")+}++// ---------- helpers ----------++func mustAnalyze(t *testing.T, cat *catalog.Catalog, sql string) *catalog.Query {+ t.Helper()+ stmts, err := ParsePg(sql)+ if err != nil {+ t.Fatalf("parse %q: %v", sql, err)+ }+ selStmt, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ t.Fatalf("expected SelectStmt, got %T", stmts[0].AST)+ }+ q, err := cat.AnalyzeSelectStmt(selStmt)+ if err != nil {+ t.Fatalf("AnalyzeSelectStmt(%q): %v", sql, err)+ }+ return q+}++func mustParseSelect(t *testing.T, sql string) *ast.SelectStmt {+ t.Helper()+ stmts, err := ParsePg(sql)+ if err != nil {+ t.Fatalf("parse %q: %v", sql, err)+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ t.Fatalf("expected SelectStmt, got %T", stmts[0].AST)+ }+ return sel+}++// ---------- Risk scenario 9: typeNameFromString with quoted/reserved identifier ----------++func TestLoaderPoC_QuotedIdentifierType(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Create an enum with a reserved-ish name.+ enumStmt := &ast.CreateEnumStmt{+ TypeName: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "user"}, // "user" is a reserved word in PG+ }},+ Vals: &ast.List{Items: []ast.Node{+ &ast.String{Str: "admin"},+ &ast.String{Str: "member"},+ }},+ }+ if err := cat.DefineEnum(enumStmt); err != nil {+ t.Fatalf("DefineEnum for reserved-word enum failed: %v", err)+ }++ // The cheat parser should be able to resolve this through SELECT NULL::"public"."user".+ // Note: caller is responsible for quoting — this is documented in typeNameFromString.+ tn := mustTypeName(t, `"public"."user"`)+ if tn == nil {+ t.Fatal("typeNameFromString returned nil for quoted reserved-word type")+ }++ // And we can use it in a CREATE TABLE.+ stmt := buildCreateTableStmt("public", "accts", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("role", tn, false),+ })+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation with quoted-reserved-word enum column failed: %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT id, role FROM accts`)+ if len(q.TargetList) != 2 {+ t.Errorf("quoted-identifier enum query: got %d, want 2", len(q.TargetList))+ }+ t.Log("quoted identifier PoC: cheat parser handles reserved-word-adjacent type names")+}++// ---------- Risk scenario 10: partitioned table ----------++func TestLoaderPoC_PartitionedTable(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Construct CREATE TABLE sales (region text, amount int4) PARTITION BY LIST (region).+ stmt := &ast.CreateStmt{+ Relation: &ast.RangeVar{+ Schemaname: "public",+ Relname: "sales",+ Relpersistence: 'p',+ },+ TableElts: &ast.List{Items: []ast.Node{+ columnDef("region", typeName("text"), false),+ columnDef("amount", typeName("int4"), false),+ }},+ Partspec: &ast.PartitionSpec{+ Strategy: "l", // list+ PartParams: &ast.List{Items: []ast.Node{+ &ast.PartitionElem{Name: "region"},+ }},+ },+ }+ // Note: omni test convention passes 'r' even for partitioned tables;+ // omni flips relkind internally based on Partspec presence.+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation partitioned failed: %v", err)+ }++ rel := cat.GetRelation("public", "sales")+ if rel == nil {+ t.Fatal("partitioned table not installed")+ }++ q := mustAnalyze(t, cat, `SELECT region, amount FROM sales WHERE region = 'us'`)+ if len(q.TargetList) != 2 {+ t.Errorf("partitioned table query: got %d, want 2", len(q.TargetList))+ }+ t.Logf("partitioned table PoC: DefineRelation with Partspec works; relation RelKind=%c", rel.RelKind)+}++// ---------- Risk scenario 11: inherited table ----------++func TestLoaderPoC_InheritedTable(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Parent with 2 columns.+ parentStmt := buildCreateTableStmt("public", "parent", []*ast.ColumnDef{+ columnDef("a", typeName("int4"), false),+ columnDef("b", typeName("text"), false),+ })+ if err := cat.DefineRelation(parentStmt, 'r'); err != nil {+ t.Fatalf("install parent failed: %v", err)+ }++ // Child with 1 locally-defined column, inheriting from parent.+ childStmt := &ast.CreateStmt{+ Relation: &ast.RangeVar{+ Schemaname: "public",+ Relname: "child",+ Relpersistence: 'p',+ },+ InhRelations: &ast.List{Items: []ast.Node{+ &ast.RangeVar{Schemaname: "public", Relname: "parent"},+ }},+ TableElts: &ast.List{Items: []ast.Node{+ columnDef("c", typeName("int4"), false),+ }},+ }+ if err := cat.DefineRelation(childStmt, 'r'); err != nil {+ t.Fatalf("install child with InhRelations failed: %v", err)+ }++ child := cat.GetRelation("public", "child")+ if child == nil {+ t.Fatal("child relation not installed")+ }+ // Child should have inherited parent's columns plus its own local column.+ if len(child.Columns) < 3 {+ t.Errorf("child column count: got %d, want >= 3 (a, b inherited + c local)", len(child.Columns))+ for _, col := range child.Columns {+ t.Logf(" col: %s", col.Name)+ }+ }++ // Query should resolve all inherited + local columns.+ q := mustAnalyze(t, cat, `SELECT a, b, c FROM child`)+ if len(q.TargetList) != 3 {+ t.Errorf("inherited query: got %d targets, want 3", len(q.TargetList))+ }+ t.Log("inherited table PoC: InhRelations merges parent columns into child")+}++// ---------- Risk scenario 12: materialized view via ExecCreateTableAs ----------++func TestLoaderPoC_MaterializedView(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("val", typeName("int4"), false),+ }), 'r'); err != nil {+ t.Fatalf("install t: %v", err)+ }++ // Parse the matview body — same pattern as DefineView.+ body := `SELECT id, val * 2 AS doubled FROM t WHERE val > 0`+ selStmt := mustParseSelect(t, body)++ stmt := &ast.CreateTableAsStmt{+ Query: selStmt,+ Objtype: ast.OBJECT_MATVIEW,+ Into: &ast.IntoClause{+ Rel: &ast.RangeVar{+ Schemaname: "public",+ Relname: "t_doubled",+ Relpersistence: 'p',+ },+ },+ }+ if err := cat.ExecCreateTableAs(stmt); err != nil {+ t.Fatalf("ExecCreateTableAs failed: %v", err)+ }++ rel := cat.GetRelation("public", "t_doubled")+ if rel == nil {+ t.Fatal("matview not installed")+ }+ if rel.RelKind != 'm' {+ t.Errorf("matview RelKind: got %c, want 'm'", rel.RelKind)+ }++ q := mustAnalyze(t, cat, `SELECT id, doubled FROM t_doubled`)+ if len(q.TargetList) != 2 {+ t.Errorf("matview query: got %d, want 2", len(q.TargetList))+ }+ t.Log("matview PoC: ExecCreateTableAs with OBJECT_MATVIEW works; query on matview analyzes")+}++// ---------- Risk scenario 13: function + overload via CreateFunctionStmt ----------++func TestLoaderPoC_FunctionOverload(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Helper to build a CreateFunctionStmt — mirrors omni's makeCreateFuncStmt+ // but uses explicit AST nodes.+ makeFuncStmt := func(schema, name string, argTypeNames []string, retTypeName, language, body string) *ast.CreateFunctionStmt {+ var nameItems []ast.Node+ if schema != "" {+ nameItems = append(nameItems, &ast.String{Str: schema})+ }+ nameItems = append(nameItems, &ast.String{Str: name})++ var params []ast.Node+ for _, argTypeStr := range argTypeNames {+ params = append(params, &ast.FunctionParameter{+ ArgType: typeName(argTypeStr),+ Mode: ast.FUNC_PARAM_IN,+ })+ }++ stmt := &ast.CreateFunctionStmt{+ Funcname: &ast.List{Items: nameItems},+ ReturnType: typeName(retTypeName),+ }+ if len(params) > 0 {+ stmt.Parameters = &ast.List{Items: params}+ }++ var opts []ast.Node+ if language != "" {+ opts = append(opts, &ast.DefElem{Defname: "language", Arg: &ast.String{Str: language}})+ }+ if body != "" {+ opts = append(opts, &ast.DefElem{+ Defname: "as",+ Arg: &ast.List{Items: []ast.Node{&ast.String{Str: body}}},+ })+ }+ if len(opts) > 0 {+ stmt.Options = &ast.List{Items: opts}+ }+ return stmt+ }++ // Install overload 1: my_double(int4) returns int4.+ err := cat.CreateFunctionStmt(makeFuncStmt("public", "my_double",+ []string{"int4"}, "int4", "sql", "SELECT $1 * 2"))+ if err != nil {+ t.Fatalf("install my_double(int4) failed: %v", err)+ }++ // Install overload 2: my_double(int8) returns int8. Different signature,+ // must coexist (this is codex's D3 verification — sequential+ // CreateFunctionStmt calls install all overloads).+ err = cat.CreateFunctionStmt(makeFuncStmt("public", "my_double",+ []string{"int8"}, "int8", "sql", "SELECT $1 * 2"))+ if err != nil {+ t.Fatalf("install my_double(int8) failed: %v", err)+ }++ // Both overloads must be visible.+ procs := cat.LookupProcByName("my_double")+ if len(procs) < 2 {+ t.Fatalf("overload count: got %d, want >= 2", len(procs))+ }++ // Install a table and query using the function.+ if err := cat.DefineRelation(buildCreateTableStmt("public", "nums", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("n", typeName("int4"), false),+ }), 'r'); err != nil {+ t.Fatalf("install nums: %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT id, my_double(n) AS doubled FROM nums`)+ if len(q.TargetList) != 2 {+ t.Errorf("function query: got %d, want 2", len(q.TargetList))+ }+ t.Logf("function overload PoC: 2 overloads installed sequentially, analyzer picks correctly. %d procs visible under 'my_double'", len(procs))+}++// ---------- Risk scenario 14: composite type + column referencing it ----------++func TestLoaderPoC_CompositeType(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Install a composite type: address (street text, city text, zip text).+ compStmt := &ast.CompositeTypeStmt{+ Typevar: &ast.RangeVar{+ Schemaname: "public",+ Relname: "address",+ },+ Coldeflist: &ast.List{Items: []ast.Node{+ columnDef("street", typeName("text"), false),+ columnDef("city", typeName("text"), false),+ columnDef("zip", typeName("text"), false),+ }},+ }+ if err := cat.DefineCompositeType(compStmt); err != nil {+ t.Fatalf("DefineCompositeType failed: %v", err)+ }++ // Install a table with a column of the composite type.+ stmt := buildCreateTableStmt("public", "customers", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("addr", mustTypeName(t, "public.address"), false),+ })+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation with composite column failed: %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT id, addr FROM customers`)+ if len(q.TargetList) != 2 {+ t.Errorf("composite-col query: got %d, want 2", len(q.TargetList))+ }+ t.Log("composite type PoC: DefineCompositeType + DefineRelation with composite column works")+}++// ---------- Risk scenario 15: domain type + column referencing it ----------++func TestLoaderPoC_DomainType(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Install a domain: positive_int, base int4, NOT NULL.+ domainStmt := &ast.CreateDomainStmt{+ Domainname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "positive_int"},+ }},+ Typname: typeName("int4"),+ Constraints: &ast.List{Items: []ast.Node{+ &ast.Constraint{Contype: ast.CONSTR_NOTNULL},+ }},+ }+ if err := cat.DefineDomain(domainStmt); err != nil {+ t.Fatalf("DefineDomain failed: %v", err)+ }++ // Install a table with a column of the domain type.+ stmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("count", mustTypeName(t, "public.positive_int"), true),+ })+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation with domain column failed: %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT id, count FROM counters WHERE count > 0`)+ if len(q.TargetList) != 2 {+ t.Errorf("domain-col query: got %d, want 2", len(q.TargetList))+ }+ t.Log("domain type PoC: DefineDomain + DefineRelation with domain column works")+}++// ---------- Exploration: does query span actually need real types? ----------+//+// Hypothesis: for query span purposes, we only need column NAMES, not real+// column TYPES. Lineage is a name-resolution problem, not a type-resolution+// problem. If we install every column as "text" (regardless of real metadata+// type), most queries should still analyze correctly and produce identical+// lineage output.+//+// Why this matters: if true, v5 can skip typeNameFromString entirely, skip+// the user enum / domain / composite problem entirely, and reduce the+// builder function library to a trivial "table with column names" shape.+//+// This test runs a battery of queries against tables where all columns are+// declared as text, and asks: (a) does AnalyzeSelectStmt succeed, (b) is the+// lineage output the same as with correct types? A query that fails here+// would fall through to extractFallbackColumns today (same as the current+// path does on any analyzer error).++func TestLoaderPoC_TypesDontMatterForLineage(t *testing.T) {+ // Build two catalogs: one with "honest" types, one with everything as text.+ // Run the same queries against both and compare lineage.++ installHonest := func(cat *catalog.Catalog) {+ // orders (id int, amount numeric(10,2), status text, placed_at timestamptz, tags text[])+ if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),+ columnDef("status", typeName("text"), false),+ columnDef("placed_at", mustTypeName(t, "timestamp with time zone"), false),+ columnDef("tags", mustTypeName(t, "text[]"), false),+ }), 'r'); err != nil {+ t.Fatalf("honest install: %v", err)+ }+ // customers (id int, email text)+ if err := cat.DefineRelation(buildCreateTableStmt("public", "customers", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("email", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("honest install customers: %v", err)+ }+ }++ installLying := func(cat *catalog.Catalog) {+ // Everything text, everything. Even numeric/timestamp/array columns.+ // This is the minimum information the loader would need if we decided+ // bytebase metadata should not bother with per-column type strings.+ if err := cat.DefineRelation(buildCreateTableStmt("public", "orders", []*ast.ColumnDef{+ columnDef("id", typeName("text"), true),+ columnDef("amount", typeName("text"), false),+ columnDef("status", typeName("text"), false),+ columnDef("placed_at", typeName("text"), false),+ columnDef("tags", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("lying install: %v", err)+ }+ if err := cat.DefineRelation(buildCreateTableStmt("public", "customers", []*ast.ColumnDef{+ columnDef("id", typeName("text"), true),+ columnDef("email", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("lying install customers: %v", err)+ }+ }++ catHonest := catalog.New()+ catHonest.SetSearchPath([]string{"public"})+ installHonest(catHonest)++ catLying := catalog.New()+ catLying.SetSearchPath([]string{"public"})+ installLying(catLying)++ // Each test case: a query, a description, an "honest expected to pass" flag.+ // We record (for each catalog) whether AnalyzeSelectStmt succeeded and what+ // the lineage looked like, then compare.+ cases := []struct {+ name string+ query string+ // expectedHonest is whether we expect the honest catalog to analyze successfully.+ // All our cases are expected to succeed on the honest catalog.+ }{+ // Category 1: Name resolution only — no operator semantics involved+ {"plain_select", `SELECT id, status FROM orders`},+ {"select_star", `SELECT * FROM orders`},+ {"join", `SELECT o.id, c.email FROM orders o JOIN customers c ON c.id = o.id`},+ {"alias", `SELECT o.id AS order_id FROM orders o`},++ // Category 2: text-friendly operators (=, LIKE, IS NULL)+ {"text_equality", `SELECT id FROM orders WHERE status = 'open'`},+ {"text_like", `SELECT id FROM orders WHERE status LIKE 'op%'`},+ {"is_null", `SELECT id FROM orders WHERE status IS NULL`},++ // Category 3: numeric operators against text — will omni coerce or fail?+ {"numeric_compare", `SELECT id FROM orders WHERE amount > 100`},+ {"numeric_arithmetic", `SELECT id, amount + 1 AS amount_plus FROM orders`},+ {"numeric_sum", `SELECT SUM(amount) FROM orders GROUP BY status`},++ // Category 4: date/time ops against text+ {"date_compare", `SELECT id FROM orders WHERE placed_at > '2024-01-01'::date`},+ {"date_extract", `SELECT EXTRACT(year FROM placed_at::timestamp) FROM orders`},++ // Category 5: array ops against text+ {"array_contains", `SELECT id FROM orders WHERE tags @> ARRAY['urgent']`},+ {"array_unnest", `SELECT id, unnest(tags) FROM orders`},++ // Category 6: subquery + CTE (name resolution heavy)+ {"cte", `WITH active AS (SELECT id, status FROM orders WHERE status = 'open') SELECT id FROM active`},+ {"subquery_where", `SELECT id FROM orders WHERE id IN (SELECT id FROM customers)`},+ {"correlated", `SELECT id, (SELECT email FROM customers c WHERE c.id = orders.id) AS email FROM orders`},++ // Category 7: aggregations+ {"count_star", `SELECT COUNT(*) FROM orders`},+ {"count_col", `SELECT COUNT(id) FROM orders`},+ {"group_by", `SELECT status, COUNT(*) FROM orders GROUP BY status`},+ }++ type result struct {+ ok bool+ err string+ nTargets int+ names []string+ }++ runOne := func(cat *catalog.Catalog, sql string) result {+ stmts, err := ParsePg(sql)+ if err != nil {+ return result{err: "parse: " + err.Error()}+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ return result{err: "not a SelectStmt"}+ }+ q, err := cat.AnalyzeSelectStmt(sel)+ if err != nil {+ return result{err: "analyze: " + err.Error()}+ }+ r := result{ok: true, nTargets: len(q.TargetList)}+ for _, te := range q.TargetList {+ r.names = append(r.names, te.ResName)+ }+ return r+ }++ var bothOk, honestOnlyOk, lyingOnlyOk, bothFail, lineageDiverges int+ t.Log("---- query span type-free install exploration ----")+ t.Logf("%-22s %-8s %-8s %s", "case", "honest", "lying", "note")+ for _, tc := range cases {+ h := runOne(catHonest, tc.query)+ l := runOne(catLying, tc.query)++ honestMark := "FAIL"+ if h.ok {+ honestMark = "PASS"+ }+ lyingMark := "FAIL"+ if l.ok {+ lyingMark = "PASS"+ }++ note := ""+ if h.ok && l.ok {+ bothOk+++ // Compare lineage shape (target count + names).+ if h.nTargets != l.nTargets {+ note = fmt.Sprintf("TARGETS DIFFER h=%d l=%d", h.nTargets, l.nTargets)+ lineageDiverges+++ } else {+ same := true+ for i := range h.names {+ if h.names[i] != l.names[i] {+ same = false+ break+ }+ }+ if !same {+ note = fmt.Sprintf("NAMES DIFFER h=%v l=%v", h.names, l.names)+ lineageDiverges+++ } else {+ note = fmt.Sprintf("IDENTICAL (%d cols)", h.nTargets)+ }+ }+ } else if h.ok && !l.ok {+ honestOnlyOk+++ note = "lying fails: " + truncate(l.err, 60)+ } else if !h.ok && l.ok {+ lyingOnlyOk+++ note = "honest fails: " + truncate(h.err, 60)+ } else {+ bothFail+++ note = "both fail"+ }++ t.Logf("%-22s %-8s %-8s %s", tc.name, honestMark, lyingMark, note)+ }++ t.Log("")+ t.Logf("summary: both_pass=%d honest_only=%d lying_only=%d both_fail=%d diverge=%d",+ bothOk, honestOnlyOk, lyingOnlyOk, bothFail, lineageDiverges)+ t.Log("")+ t.Log("key question: for cases where both paths analyze successfully, does lineage match?")+ t.Log("key question: for cases where only honest analyzes, can fallback recover equal-or-better lineage?")+}++func truncate(s string, n int) string {+ if len(s) <= n {+ return s+ }+ return s[:n] + "..."+}++// TestLoaderPoC_UnknownPseudoType explores whether PG's 'unknown' pseudo-type+// can be used as a universal column type that accepts all operators.+func TestLoaderPoC_UnknownPseudoType(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ tries := []string{"unknown", "anyelement", "any"}+ for _, ty := range tries {+ stmt := buildCreateTableStmt("public", "t_"+ty, []*ast.ColumnDef{+ {Colname: "id", TypeName: typeName(ty)},+ })+ err := cat.DefineRelation(stmt, 'r')+ if err != nil {+ t.Logf("%-12s → DefineRelation rejected: %s", ty, truncate(err.Error(), 80))+ } else {+ t.Logf("%-12s → DefineRelation accepted", ty)+ }+ }+}++// TestLoaderPoC_UnknownTypeQueryBattery installs the same queries as+// TestLoaderPoC_TypesDontMatterForLineage but with all columns declared as+// 'unknown' instead of 'text'. The question: does the analyzer's overload+// resolution treat unknown differently from text?+func TestLoaderPoC_UnknownTypeQueryBattery(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ install := func(table string, cols []string) {+ colDefs := make([]*ast.ColumnDef, len(cols))+ for i, c := range cols {+ colDefs[i] = columnDef(c, typeName("unknown"), false)+ }+ if err := cat.DefineRelation(buildCreateTableStmt("public", table, colDefs), 'r'); err != nil {+ t.Fatalf("install %s: %v", table, err)+ }+ }+ install("orders", []string{"id", "amount", "status", "placed_at", "tags", "customer_id"})+ install("customers", []string{"id", "email", "name", "region"})++ cases := []struct {+ category, name, query string+ }{+ // ---- Baseline: name resolution only ----+ {"name_res", "plain_select", `SELECT id, status FROM orders`},+ {"name_res", "select_star", `SELECT * FROM orders`},+ {"name_res", "join", `SELECT o.id, c.email FROM orders o JOIN customers c ON c.id = o.customer_id`},+ {"name_res", "three_way_join", `SELECT o.id FROM orders o JOIN customers c ON c.id = o.customer_id JOIN orders o2 ON o2.id = o.id`},+ {"name_res", "alias_qualified", `SELECT o.id AS order_id, c.name AS customer FROM orders o, customers c WHERE c.id = o.customer_id`},++ // ---- Text-friendly operators ----+ {"text_ops", "text_equality", `SELECT id FROM orders WHERE status = 'open'`},+ {"text_ops", "text_like", `SELECT id FROM orders WHERE status LIKE 'op%'`},+ {"text_ops", "text_ilike", `SELECT id FROM orders WHERE status ILIKE 'OP%'`},+ {"text_ops", "text_in_list", `SELECT id FROM orders WHERE status IN ('open', 'closed')`},+ {"text_ops", "is_null", `SELECT id FROM orders WHERE status IS NULL`},+ {"text_ops", "is_not_null", `SELECT id FROM orders WHERE status IS NOT NULL`},++ // ---- Numeric operators (unknown auto-coerces) ----+ {"numeric_ops", "numeric_compare", `SELECT id FROM orders WHERE amount > 100`},+ {"numeric_ops", "numeric_arithmetic", `SELECT id, amount + 1 AS p FROM orders`},+ {"numeric_ops", "numeric_between", `SELECT id FROM orders WHERE amount BETWEEN 10 AND 100`},+ {"numeric_ops", "numeric_negation", `SELECT id, -amount FROM orders`},++ // ---- Date/time ops ----+ {"date_ops", "date_compare", `SELECT id FROM orders WHERE placed_at > '2024-01-01'::date`},+ {"date_ops", "date_extract", `SELECT EXTRACT(year FROM placed_at::timestamp) FROM orders`},+ {"date_ops", "interval_arith", `SELECT id FROM orders WHERE placed_at > now() - interval '7 days'`},++ // ---- Array ops ----+ {"array_ops", "array_contains", `SELECT id FROM orders WHERE tags @> ARRAY['urgent']`},+ {"array_ops", "array_any", `SELECT id FROM orders WHERE 'urgent' = ANY(tags)`},+ {"array_ops", "array_length", `SELECT id, array_length(tags, 1) FROM orders`},+ {"array_ops", "array_unnest", `SELECT id, unnest(tags) FROM orders`},++ // ---- Aggregates (the worry zone) ----+ {"aggregate", "count_star", `SELECT COUNT(*) FROM orders`},+ {"aggregate", "count_col", `SELECT COUNT(id) FROM orders`},+ {"aggregate", "count_distinct", `SELECT COUNT(DISTINCT customer_id) FROM orders`},+ {"aggregate", "sum", `SELECT SUM(amount) FROM orders`},+ {"aggregate", "avg", `SELECT AVG(amount) FROM orders`},+ {"aggregate", "min_max", `SELECT MIN(amount), MAX(amount) FROM orders`},+ {"aggregate", "string_agg", `SELECT STRING_AGG(status, ',') FROM orders`},+ {"aggregate", "array_agg", `SELECT ARRAY_AGG(id) FROM orders`},+ {"aggregate", "group_by", `SELECT status, COUNT(*) FROM orders GROUP BY status`},+ {"aggregate", "having", `SELECT status, COUNT(*) FROM orders GROUP BY status HAVING COUNT(*) > 5`},++ // ---- Aggregate with explicit cast (workaround) ----+ {"cast_workaround", "sum_with_cast", `SELECT SUM(amount::numeric) FROM orders`},+ {"cast_workaround", "avg_with_cast", `SELECT AVG(amount::numeric) FROM orders`},+ {"cast_workaround", "string_agg_cast", `SELECT STRING_AGG(status::text, ',') FROM orders`},++ // ---- Window functions ----+ {"window", "row_number", `SELECT id, ROW_NUMBER() OVER (ORDER BY placed_at) FROM orders`},+ {"window", "rank_partition", `SELECT id, RANK() OVER (PARTITION BY status ORDER BY amount) FROM orders`},+ {"window", "sum_over", `SELECT id, SUM(amount) OVER (PARTITION BY status) FROM orders`},+ {"window", "lag_col", `SELECT id, LAG(status) OVER (ORDER BY placed_at) FROM orders`},++ // ---- Control flow / null handling ----+ {"control", "case_when", `SELECT id, CASE WHEN status = 'open' THEN 1 ELSE 0 END FROM orders`},+ {"control", "coalesce", `SELECT COALESCE(status, 'unknown') FROM orders`},+ {"control", "nullif", `SELECT NULLIF(status, 'open') FROM orders`},++ // ---- Subqueries / set operations ----+ {"subquery", "exists", `SELECT id FROM orders o WHERE EXISTS (SELECT 1 FROM customers c WHERE c.id = o.customer_id)`},+ {"subquery", "scalar_subquery", `SELECT id, (SELECT MAX(amount) FROM orders) AS max_amt FROM orders`},+ {"subquery", "not_in", `SELECT id FROM orders WHERE customer_id NOT IN (SELECT id FROM customers WHERE region = 'us')`},+ {"subquery", "union", `SELECT id FROM orders UNION SELECT id FROM customers`},+ {"subquery", "intersect", `SELECT id FROM orders INTERSECT SELECT id FROM customers`},+ {"subquery", "except", `SELECT id FROM orders EXCEPT SELECT id FROM customers`},++ // ---- CTE + recursion ----+ {"cte", "simple_cte", `WITH active AS (SELECT id, status FROM orders WHERE status = 'open') SELECT id FROM active`},+ {"cte", "cte_with_agg", `WITH stats AS (SELECT status, COUNT(*) c FROM orders GROUP BY status) SELECT * FROM stats`},++ // ---- Sort / limit ----+ {"order_limit", "order_by_col", `SELECT id FROM orders ORDER BY placed_at DESC LIMIT 10`},+ {"order_limit", "order_by_expr", `SELECT id FROM orders ORDER BY amount + 1`},+ {"order_limit", "distinct_on", `SELECT DISTINCT ON (customer_id) id, customer_id FROM orders ORDER BY customer_id, placed_at DESC`},+ }++ type bucket struct{ pass, fail int }+ byCategory := map[string]*bucket{}++ pass := 0+ fail := 0+ var failed []string+ for _, tc := range cases {+ if byCategory[tc.category] == nil {+ byCategory[tc.category] = &bucket{}+ }+ stmts, err := ParsePg(tc.query)+ if err != nil {+ t.Logf("%-12s %-22s PARSE FAIL: %v", tc.category, tc.name, err)+ fail+++ byCategory[tc.category].fail+++ failed = append(failed, tc.category+"/"+tc.name+" (parse)")+ continue+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ t.Logf("%-12s %-22s NOT SELECT", tc.category, tc.name)+ fail+++ byCategory[tc.category].fail+++ continue+ }+ _, err = cat.AnalyzeSelectStmt(sel)+ if err != nil {+ t.Logf("%-12s %-22s FAIL: %s", tc.category, tc.name, truncate(err.Error(), 90))+ fail+++ byCategory[tc.category].fail+++ failed = append(failed, tc.category+"/"+tc.name)+ continue+ }+ pass+++ byCategory[tc.category].pass+++ t.Logf("%-12s %-22s PASS", tc.category, tc.name)+ }++ t.Log("")+ t.Log("----- unknown-type battery: per-category breakdown -----")+ for _, cat := range []string{"name_res", "text_ops", "numeric_ops", "date_ops", "array_ops", "aggregate", "cast_workaround", "window", "control", "subquery", "cte", "order_limit"} {+ b := byCategory[cat]+ if b == nil {+ continue+ }+ t.Logf(" %-16s %d pass / %d fail", cat, b.pass, b.fail)+ }+ t.Log("")+ t.Logf("TOTAL: %d pass, %d fail (%.0f%% pass rate)", pass, fail, 100.0*float64(pass)/float64(pass+fail))+ if len(failed) > 0 {+ t.Log("failed:")+ for _, f := range failed {+ t.Logf(" - %s", f)+ }+ }+}++// TestLoaderPoC_HybridTypeResolution explores the pragmatic middle ground:+// for each column, try to resolve the metadata type string; if it fails+// (user type not in catalog, malformed type string), fall back to text.+// This gives us "best of both" — real types where available, permissive+// text where not.+func TestLoaderPoC_HybridTypeResolution(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Hybrid column builder: try honest type, fall back to text.+ hybridCol := func(name string, typeStr string) *ast.ColumnDef {+ tn := mustTypeName(t, typeStr)+ // Check if catalog can resolve it at this moment.+ // Note: for user types to resolve, they must already be installed.+ // We're checking against the catalog AS-IS.+ resolved := false+ func() {+ defer func() { _ = recover() }() // defensive: ResolveType may not panic, but be safe+ _, _, err := cat.ResolveType(toCatalogTypeName(tn))+ resolved = (err == nil)+ }()+ if resolved {+ return &ast.ColumnDef{Colname: name, TypeName: tn}+ }+ t.Logf(" hybrid: column %q type %q unresolvable in catalog, falling back to text", name, typeStr)+ return &ast.ColumnDef{Colname: name, TypeName: typeName("text")}+ }++ // Simulate metadata where some types are resolvable, some aren't.+ // 'int4' is a builtin — resolvable.+ // 'numeric(10,2)' is a builtin — resolvable.+ // 'public.unknown_enum_xyz' is a user type we never installed — should fall back to text.+ stmt := buildCreateTableStmt("public", "mix", []*ast.ColumnDef{+ hybridCol("id", "int4"),+ hybridCol("amount", "numeric(10,2)"),+ hybridCol("tag", "public.unknown_enum_xyz"),+ })+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("hybrid install failed: %v", err)+ }++ // Queries that use the numeric column (known type) should still work+ // with real operator resolution.+ q1 := mustAnalyze(t, cat, `SELECT id, amount FROM mix WHERE amount > 100`)+ if len(q1.TargetList) != 2 {+ t.Errorf("q1 targets: got %d, want 2", len(q1.TargetList))+ }++ // Queries that touch the tag column only do name resolution — no operator+ // type-checking — should work even though tag is text-in-reality-but-+ // metadata-said-enum.+ q2 := mustAnalyze(t, cat, `SELECT id, tag FROM mix WHERE tag = 'a'`)+ if len(q2.TargetList) != 2 {+ t.Errorf("q2 targets: got %d, want 2", len(q2.TargetList))+ }++ t.Log("hybrid PoC: resolvable types use real TypeName, unresolvable types fall back to text")+ t.Log("effect: best of both — operator semantics work where types known, name resolution works where not")+}++// TestLoaderPoC_BvsCDivergence pinpoints the exact behavioral difference between+// Option B (hybrid: builtins honest, user types unknown) and Option C (all+// honest types, user types installed).+//+// Scenario: a table with ONE builtin-int column and ONE user-enum column.+// Run the same queries against two catalogs:+//+// catC: enum installed via DefineEnum, table column is the real enum type+// catB: enum NOT installed, table column is 'unknown'+//+// For each query, record (passes/fails, target count, lineage equivalent).+// The output shows exactly where B and C diverge.+func TestLoaderPoC_BvsCDivergence(t *testing.T) {+ // ----- catC: option C (all honest) -----+ catC := catalog.New()+ catC.SetSearchPath([]string{"public"})++ if err := catC.DefineEnum(&ast.CreateEnumStmt{+ TypeName: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "task_status"},+ }},+ Vals: &ast.List{Items: []ast.Node{+ &ast.String{Str: "open"},+ &ast.String{Str: "closed"},+ }},+ }); err != nil {+ t.Fatalf("catC install enum: %v", err)+ }+ if err := catC.DefineRelation(buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("status", mustTypeName(t, "public.task_status"), false),+ columnDef("created_at", mustTypeName(t, "timestamptz"), false),+ }), 'r'); err != nil {+ t.Fatalf("catC install table: %v", err)+ }++ // ----- catB: option B (hybrid — enum column as unknown, everything else honest) -----+ catB := catalog.New()+ catB.SetSearchPath([]string{"public"})+ // Note: no DefineEnum call. Enum "doesn't exist" from catB's perspective.+ if err := catB.DefineRelation(buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true), // builtin → honest+ columnDef("status", typeName("unknown"), false), // user type → unknown+ columnDef("created_at", typeName("timestamptz"), false), // builtin → honest+ }), 'r'); err != nil {+ t.Fatalf("catB install table: %v", err)+ }++ cases := []struct {+ name, query string+ }{+ // Queries that should work in BOTH (builtin columns untouched)+ {"select_id", `SELECT id FROM tasks`},+ {"select_star", `SELECT * FROM tasks`},+ {"count_star", `SELECT COUNT(*) FROM tasks`},+ {"sum_id", `SELECT SUM(id) FROM tasks`},+ {"avg_id", `SELECT AVG(id) FROM tasks`},+ {"max_created", `SELECT MAX(created_at) FROM tasks`},++ // Queries touching status column — B gets 'unknown', C gets real enum+ {"where_status_eq", `SELECT id FROM tasks WHERE status = 'open'`},+ {"where_status_in", `SELECT id FROM tasks WHERE status IN ('open', 'closed')`},+ {"select_status", `SELECT status FROM tasks`},+ {"group_by_status", `SELECT status, COUNT(*) FROM tasks GROUP BY status`},+ {"order_by_status", `SELECT id FROM tasks ORDER BY status`},+ {"distinct_status", `SELECT DISTINCT status FROM tasks`},++ // Aggregates directly on the enum column — the real test+ {"count_status", `SELECT COUNT(status) FROM tasks`},+ {"count_distinct_status", `SELECT COUNT(DISTINCT status) FROM tasks`},+ {"min_status", `SELECT MIN(status) FROM tasks`},+ {"max_status", `SELECT MAX(status) FROM tasks`},+ {"array_agg_status", `SELECT array_agg(status) FROM tasks`},+ {"string_agg_status", `SELECT string_agg(status::text, ',') FROM tasks`},++ // Mixed: status in WHERE, sum on builtin+ {"mixed_sum", `SELECT SUM(id) FROM tasks WHERE status = 'open'`},+ {"mixed_group", `SELECT status, SUM(id) FROM tasks GROUP BY status`},+ }++ type res struct {+ ok bool+ err string+ }+ runOne := func(cat *catalog.Catalog, sql string) res {+ stmts, err := ParsePg(sql)+ if err != nil {+ return res{err: "parse: " + truncate(err.Error(), 60)}+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ return res{err: "not a SelectStmt"}+ }+ if _, err := cat.AnalyzeSelectStmt(sel); err != nil {+ return res{err: truncate(err.Error(), 70)}+ }+ return res{ok: true}+ }++ var bothOk, cOnlyOk, bOnlyOk, bothFail int+ t.Logf("%-24s %-6s %-6s %s", "case", "C", "B", "divergence note")+ t.Logf("%-24s %-6s %-6s %s", "----", "-", "-", "---------------")+ for _, tc := range cases {+ c := runOne(catC, tc.query)+ b := runOne(catB, tc.query)+ cMark := "FAIL"+ if c.ok {+ cMark = "PASS"+ }+ bMark := "FAIL"+ if b.ok {+ bMark = "PASS"+ }++ note := ""+ switch {+ case c.ok && b.ok:+ bothOk+++ note = "same"+ case c.ok && !b.ok:+ cOnlyOk+++ note = "C only — B: " + b.err+ case !c.ok && b.ok:+ bOnlyOk+++ note = "B only — C: " + c.err+ default:+ bothFail+++ note = "both fail"+ }+ t.Logf("%-24s %-6s %-6s %s", tc.name, cMark, bMark, note)+ }++ t.Log("")+ t.Logf("summary: both_ok=%d C_only=%d B_only=%d both_fail=%d", bothOk, cOnlyOk, bOnlyOk, bothFail)+ t.Logf("option B loss (queries where C works but B falls back): %d / %d = %.0f%%",+ cOnlyOk, len(cases), 100.0*float64(cOnlyOk)/float64(len(cases)))+}++// TestLoaderPoC_BvsCDomainColumn shows the case where B is STRICTLY BETTER than C:+// a table with a domain-typed column where the domain is NOT in bytebase metadata+// (which is the ACTUAL state of bytebase PG sync today — domain/composite/range+// types are not synced). Option C fails at install time and blocks all queries+// on the table. Option B uses unknown and only loses type-sensitive aggregates.+func TestLoaderPoC_BvsCDomainColumn(t *testing.T) {+ // Simulate reality: bytebase sync does NOT capture the 'public.positive_int'+ // domain. The table metadata says column 'count' has type 'public.positive_int',+ // but bytebase has no DomainMetadata for it.++ // ----- catC: option C — try to honest-install everything -----+ // Closure expansion would try to load 'public.positive_int' from metadata,+ // find nothing, and either fail the table install or skip the column.+ // Let's simulate: C tries to install the table with the real type reference,+ // fails because the domain doesn't exist in the catalog.+ catC := catalog.New()+ catC.SetSearchPath([]string{"public"})+ cStmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("count", mustTypeName(t, "public.positive_int"), true),+ })+ cErr := catC.DefineRelation(cStmt, 'r')+ if cErr != nil {+ t.Logf("[C] DefineRelation failed as expected: %s", truncate(cErr.Error(), 80))+ } else {+ t.Log("[C] DefineRelation unexpectedly succeeded")+ }++ // ----- catB: option B — count column degrades to unknown -----+ catB := catalog.New()+ catB.SetSearchPath([]string{"public"})+ bStmt := buildCreateTableStmt("public", "counters", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("count", typeName("unknown"), true),+ })+ if err := catB.DefineRelation(bStmt, 'r'); err != nil {+ t.Fatalf("[B] DefineRelation failed: %v", err)+ }+ t.Log("[B] DefineRelation succeeded")++ cases := []struct {+ name, query string+ }{+ {"simple_select", `SELECT id, count FROM counters`},+ {"where_id", `SELECT id FROM counters WHERE id > 0`},+ {"where_count", `SELECT id FROM counters WHERE count > 0`},+ {"sum_id", `SELECT SUM(id) FROM counters`},+ {"sum_count", `SELECT SUM(count) FROM counters`},+ {"sum_id_cast", `SELECT SUM(count::numeric) FROM counters`},+ }++ runOne := func(cat *catalog.Catalog, sql string) string {+ stmts, err := ParsePg(sql)+ if err != nil {+ return "parse: " + err.Error()+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ return "not a SelectStmt"+ }+ if _, err := cat.AnalyzeSelectStmt(sel); err != nil {+ return truncate(err.Error(), 60)+ }+ return "PASS"+ }++ t.Log("")+ t.Log("Query availability on 'counters' table (domain column):")+ t.Logf("%-22s %-40s %s", "case", "C result", "B result")+ var cWorks, bWorks int+ for _, tc := range cases {+ var cResult string+ if cErr != nil {+ cResult = "TABLE NOT INSTALLED"+ } else {+ cResult = runOne(catC, tc.query)+ }+ bResult := runOne(catB, tc.query)+ if cResult == "PASS" {+ cWorks+++ }+ if bResult == "PASS" {+ bWorks+++ }+ t.Logf("%-22s %-40s %s", tc.name, truncate(cResult, 38), bResult)+ }+ t.Log("")+ t.Logf("domain column query survival: C=%d/%d, B=%d/%d", cWorks, len(cases), bWorks, len(cases))+}++// toCatalogTypeName converts ast.TypeName to catalog.TypeName for ResolveType.+// catalog.TypeName is the lightweight shape used by the public ResolveType API.+func toCatalogTypeName(tn *ast.TypeName) catalog.TypeName {+ var schema, name string+ if tn.Names != nil && len(tn.Names.Items) > 0 {+ items := tn.Names.Items+ if len(items) == 1 {+ if s, ok := items[0].(*ast.String); ok {+ name = s.Str+ }+ } else if len(items) >= 2 {+ if s, ok := items[0].(*ast.String); ok {+ schema = s.Str+ }+ if s, ok := items[len(items)-1].(*ast.String); ok {+ name = s.Str+ }+ }+ }+ return catalog.TypeName{Schema: schema, Name: name, TypeMod: tn.Typemod}+}++// Format helper for readable output.+var _ = fmt.Sprintf // keep fmt imported when strings.Builder not used++// ---------- Equivalence test: hand-built CreateStmt vs parsed DDL ----------+//+// Per codex review section 2B: the PoC must prove that hand-constructed AST+// produces a catalog state equivalent to parsing the equivalent DDL text.+// This test installs the same table two ways on two separate catalogs and+// asserts the resulting Relation objects are structurally identical on all+// fields reachable via public API.++func TestLoaderPoC_HandBuiltVsDDLEquivalence(t *testing.T) {+ // Catalog A: install via hand-built CreateStmt.+ catA := catalog.New()+ catA.SetSearchPath([]string{"public"})+ handBuilt := buildCreateTableStmt("public", "orders", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("customer", typeName("text"), false),+ columnDef("amount", mustTypeName(t, "numeric(10,2)"), false),+ columnDef("placed_at", mustTypeName(t, "timestamp with time zone"), false),+ })+ if err := catA.DefineRelation(handBuilt, 'r'); err != nil {+ t.Fatalf("catA install: %v", err)+ }++ // Catalog B: install via parsed DDL text.+ catB := catalog.New()+ catB.SetSearchPath([]string{"public"})+ ddl := `CREATE TABLE public.orders (+ id int NOT NULL,+ customer text,+ amount numeric(10,2),+ placed_at timestamp with time zone+ )`+ if _, err := catB.Exec(ddl, nil); err != nil {+ t.Fatalf("catB install via Exec: %v", err)+ }++ relA := catA.GetRelation("public", "orders")+ relB := catB.GetRelation("public", "orders")+ if relA == nil || relB == nil {+ t.Fatalf("relation install missing: A=%v B=%v", relA != nil, relB != nil)+ }++ // Compare structural fields. OIDs are not expected to match absolutely+ // (each catalog has its own oidGen starting point), but relative order+ // and per-column shape must match.++ if relA.RelKind != relB.RelKind {+ t.Errorf("RelKind differs: A=%c B=%c", relA.RelKind, relB.RelKind)+ }+ if len(relA.Columns) != len(relB.Columns) {+ t.Fatalf("column count: A=%d B=%d", len(relA.Columns), len(relB.Columns))+ }+ for i := range relA.Columns {+ ca, cb := relA.Columns[i], relB.Columns[i]+ if ca.Name != cb.Name {+ t.Errorf("col[%d].Name: A=%q B=%q", i, ca.Name, cb.Name)+ }+ if ca.TypeOID != cb.TypeOID {+ t.Errorf("col[%d] (%s).TypeOID: A=%d B=%d", i, ca.Name, ca.TypeOID, cb.TypeOID)+ }+ if ca.NotNull != cb.NotNull {+ t.Errorf("col[%d] (%s).NotNull: A=%v B=%v", i, ca.Name, ca.NotNull, cb.NotNull)+ }+ if ca.AttNum != cb.AttNum {+ t.Errorf("col[%d] (%s).AttNum: A=%d B=%d", i, ca.Name, ca.AttNum, cb.AttNum)+ }+ if ca.TypeMod != cb.TypeMod {+ t.Errorf("col[%d] (%s).TypeMod: A=%d B=%d", i, ca.Name, ca.TypeMod, cb.TypeMod)+ }+ }++ // Functional equivalence: analyze the same SELECT against both catalogs+ // and assert the same target list shape.+ qA := mustAnalyze(t, catA, `SELECT id, customer, amount, placed_at FROM orders WHERE amount > 0`)+ qB := mustAnalyze(t, catB, `SELECT id, customer, amount, placed_at FROM orders WHERE amount > 0`)++ if len(qA.TargetList) != len(qB.TargetList) {+ t.Errorf("query target count: A=%d B=%d", len(qA.TargetList), len(qB.TargetList))+ }+ for i := range qA.TargetList {+ if qA.TargetList[i].ResName != qB.TargetList[i].ResName {+ t.Errorf("target[%d].ResName: A=%q B=%q",+ i, qA.TargetList[i].ResName, qB.TargetList[i].ResName)+ }+ }++ t.Log("equivalence PoC: hand-built CreateStmt and parsed DDL produce structurally identical Relation + analyze the same way")+}++// ============================================================================+// Loader pseudo-install verification+// ============================================================================+//+// These tests probe whether omni accepts minimal "pseudo" forms of each schema+// object kind, so the catalog loader strategy ("install everything; on failure install a+// pseudo at the same slot to keep downstream installs cascading through") can+// work inline during catalog init rather than as a query-time retry.+//+// Each test exercises one pseudo form in isolation, then the final+// TestLoaderPoC_RootPseudoCascadePrevention combines them end-to-end.++// ---- Pseudo enum: DefineEnum with minimal vals ----++func TestLoaderPoC_PseudoEnum_EmptyVals(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Can omni accept an enum with zero values?+ stmt := &ast.CreateEnumStmt{+ TypeName: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "broken_enum"},+ }},+ Vals: &ast.List{Items: []ast.Node{}},+ }+ err := cat.DefineEnum(stmt)+ t.Logf("DefineEnum empty vals: err=%v", err)++ if err == nil {+ // If it accepted, can a table column reference it and can the query run?+ tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("status", mustTypeName(t, "public.broken_enum"), false),+ })+ if err := cat.DefineRelation(tbl, 'r'); err != nil {+ t.Logf("DefineRelation with empty-enum column: err=%v", err)+ } else {+ q, err := analyzeSafe(cat, `SELECT id, status FROM t WHERE status = 'anything'`)+ t.Logf("analyze against empty-enum column: err=%v targets=%d", err, targetsOf(q))+ }+ }+}++func TestLoaderPoC_PseudoEnum_SingleDummyVal(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Fallback: enum with single dummy value.+ stmt := &ast.CreateEnumStmt{+ TypeName: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "broken_enum"},+ }},+ Vals: &ast.List{Items: []ast.Node{+ &ast.String{Str: "__pseudo__"},+ }},+ }+ if err := cat.DefineEnum(stmt); err != nil {+ t.Fatalf("DefineEnum single dummy val failed: %v", err)+ }++ // Install a table using the pseudo enum as a column type.+ tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("status", mustTypeName(t, "public.broken_enum"), false),+ })+ if err := cat.DefineRelation(tbl, 'r'); err != nil {+ t.Fatalf("DefineRelation with pseudo-enum column failed: %v", err)+ }++ // Probe queries that would normally hit enum operators.+ queries := []string{+ `SELECT id, status FROM t`,+ `SELECT id FROM t WHERE status = 'running'`,+ `SELECT id FROM t WHERE status::text = 'running'`,+ `SELECT COUNT(*) FROM t GROUP BY status`,+ }+ for _, q := range queries {+ res, err := analyzeSafe(cat, q)+ t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))+ }+}++// ---- Pseudo domain: DefineDomain with base type text ----++func TestLoaderPoC_PseudoDomain_OverText(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ stmt := &ast.CreateDomainStmt{+ Domainname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "broken_domain"},+ }},+ Typname: typeName("text"),+ }+ if err := cat.DefineDomain(stmt); err != nil {+ t.Fatalf("DefineDomain over text failed: %v", err)+ }++ tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("code", mustTypeName(t, "public.broken_domain"), false),+ })+ if err := cat.DefineRelation(tbl, 'r'); err != nil {+ t.Fatalf("DefineRelation with pseudo-domain column failed: %v", err)+ }++ queries := []string{+ `SELECT id, code FROM t`,+ `SELECT id FROM t WHERE code = 'anything'`,+ `SELECT id FROM t WHERE code LIKE 'A%'`,+ }+ for _, q := range queries {+ res, err := analyzeSafe(cat, q)+ t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))+ }+}++// ---- Pseudo composite: DefineCompositeType with metadata field names all text ----++func TestLoaderPoC_PseudoComposite_FieldsAllText(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Pretend metadata tells us this composite has fields {street, city, zip}.+ // We preserve the names but flatten every field to text.+ stmt := &ast.CompositeTypeStmt{+ Typevar: &ast.RangeVar{+ Schemaname: "public",+ Relname: "broken_addr",+ },+ Coldeflist: &ast.List{Items: []ast.Node{+ columnDef("street", typeName("text"), false),+ columnDef("city", typeName("text"), false),+ columnDef("zip", typeName("text"), false),+ }},+ }+ if err := cat.DefineCompositeType(stmt); err != nil {+ t.Fatalf("DefineCompositeType pseudo failed: %v", err)+ }++ tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("addr", mustTypeName(t, "public.broken_addr"), false),+ })+ if err := cat.DefineRelation(tbl, 'r'); err != nil {+ t.Fatalf("DefineRelation with pseudo-composite column failed: %v", err)+ }++ queries := []string{+ `SELECT id, addr FROM t`,+ `SELECT id, (addr).city FROM t`,+ `SELECT id FROM t WHERE (addr).city = 'NYC'`,+ }+ for _, q := range queries {+ res, err := analyzeSafe(cat, q)+ t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))+ }+}++// ---- Pseudo range: DefineRange with subtype text ----++func TestLoaderPoC_PseudoRange_SubtypeText(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Try: range with subtype text. PG itself requires subtype to have btree+ // ordering, which text does. If omni accepts it, this is a viable pseudo.+ stmt := &ast.CreateRangeStmt{+ TypeName: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "broken_range"},+ }},+ Params: &ast.List{Items: []ast.Node{+ &ast.DefElem{+ Defname: "subtype",+ Arg: &ast.TypeName{+ Names: &ast.List{Items: []ast.Node{&ast.String{Str: "text"}}},+ Typemod: -1,+ },+ },+ }},+ }+ err := cat.DefineRange(stmt)+ t.Logf("DefineRange subtype=text: err=%v", err)+ if err != nil {+ t.Skip("range install failed; this is fine if bytebase metadata never has ranges")+ }++ tbl := buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("span", mustTypeName(t, "public.broken_range"), false),+ })+ if err := cat.DefineRelation(tbl, 'r'); err != nil {+ t.Logf("DefineRelation with pseudo-range column: err=%v", err)+ }++ q, err := analyzeSafe(cat, `SELECT id, span FROM t`)+ t.Logf("query on pseudo-range: err=%v targets=%d", err, targetsOf(q))+}++// ---- Pseudo view: constant SELECT body, no FROM ----++func TestLoaderPoC_PseudoView_ConstantTargetList(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Pretend metadata tells us view "v" exposes columns {id, title, total},+ // but its body is broken/unparseable. Build a pseudo SELECT body with+ // NULL::text aliased to each name.+ pseudoBody := `SELECT NULL::text AS id, NULL::text AS title, NULL::text AS total`+ sel := mustParseSelect(t, pseudoBody)++ stmt := &ast.ViewStmt{+ View: &ast.RangeVar{+ Schemaname: "public",+ Relname: "broken_view",+ Relpersistence: 'p',+ },+ Query: sel,+ }+ if err := cat.DefineView(stmt); err != nil {+ t.Fatalf("DefineView with constant-target body failed: %v", err)+ }++ rel := cat.GetRelation("public", "broken_view")+ if rel == nil {+ t.Fatal("pseudo view not installed")+ }+ t.Logf("pseudo view installed: RelKind=%c cols=%d", rel.RelKind, len(rel.Columns))+ for _, c := range rel.Columns {+ t.Logf(" col: %s", c.Name)+ }++ queries := []string{+ `SELECT id, title, total FROM broken_view`,+ `SELECT id FROM broken_view WHERE total = '0'`,+ `SELECT COUNT(*) FROM broken_view`,+ }+ for _, q := range queries {+ res, err := analyzeSafe(cat, q)+ t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))+ }+}++// ---- Pseudo function: all params/return as text ----++func TestLoaderPoC_PseudoFunction_AllText(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Pretend metadata says the real function was broken_fn(some_broken_type)+ // returning some_other_broken_type. We install a pseudo with everything+ // collapsed to text.+ stmt := &ast.CreateFunctionStmt{+ Funcname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "broken_fn"},+ }},+ Parameters: &ast.List{Items: []ast.Node{+ &ast.FunctionParameter{ArgType: typeName("text"), Mode: ast.FUNC_PARAM_IN},+ }},+ ReturnType: typeName("text"),+ Options: &ast.List{Items: []ast.Node{+ &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},+ &ast.DefElem{+ Defname: "as",+ Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},+ },+ }},+ }+ if err := cat.CreateFunctionStmt(stmt); err != nil {+ t.Fatalf("pseudo function install failed: %v", err)+ }++ procs := cat.LookupProcByName("broken_fn")+ t.Logf("pseudo function installed: %d procs under 'broken_fn'", len(procs))++ // Install a minimal table to query against.+ if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("name", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("install t: %v", err)+ }++ queries := []string{+ `SELECT id, broken_fn(name) AS result FROM t`,+ `SELECT id FROM t WHERE broken_fn(name) = 'x'`,+ // Analyzer should pick the only overload regardless of actual arg type.+ `SELECT broken_fn('literal') FROM t`,+ }+ for _, q := range queries {+ res, err := analyzeSafe(cat, q)+ t.Logf("query %q: err=%v targets=%d", q, err, targetsOf(res))+ }+}++// ---- End-to-end: root-pseudo prevents cascade ----++func TestLoaderPoC_RootPseudoCascadePrevention(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Scenario:+ // enum E — real install fails (simulated)+ // table T — has column col E+ // view V — SELECT col FROM T+ //+ // Phase 1: for each object in topo order:+ // 1. attempt real install+ // 2. on failure, install pseudo version in the same slot+ //+ // Expected: T and V install successfully because they see pseudo E in the+ // catalog. Query against V has correct lineage.++ // Step 1: simulate E real-install failure. In the real loader we would try+ // DefineEnum first and catch the error. Here we skip straight to the+ // pseudo form to simulate "real install failed".+ pseudoE := &ast.CreateEnumStmt{+ TypeName: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "task_status"},+ }},+ Vals: &ast.List{Items: []ast.Node{+ &ast.String{Str: "__pseudo__"},+ }},+ }+ if err := cat.DefineEnum(pseudoE); err != nil {+ t.Fatalf("pseudo enum install failed: %v", err)+ }++ // Step 2: install T pointing at the (pseudo) E. This is the real install,+ // not a pseudo — T's own metadata is fine.+ tStmt := buildCreateTableStmt("public", "tasks", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("status", mustTypeName(t, "public.task_status"), false),+ columnDef("title", typeName("text"), false),+ })+ if err := cat.DefineRelation(tStmt, 'r'); err != nil {+ t.Fatalf("T real install failed (cascade not prevented): %v", err)+ }++ // Step 3: install V — real install, view body references T.+ viewBody := `SELECT id, title, status FROM tasks WHERE id > 0`+ sel := mustParseSelect(t, viewBody)+ vStmt := &ast.ViewStmt{+ View: &ast.RangeVar{+ Schemaname: "public",+ Relname: "open_tasks",+ Relpersistence: 'p',+ },+ Query: sel,+ }+ if err := cat.DefineView(vStmt); err != nil {+ t.Fatalf("V real install failed (cascade not prevented): %v", err)+ }++ // Step 4: query V — should analyze successfully and return lineage.+ q := mustAnalyze(t, cat, `SELECT id, title, status FROM open_tasks`)+ if len(q.TargetList) != 3 {+ t.Errorf("query on view: got %d targets, want 3", len(q.TargetList))+ }++ // Verify lineage reaches base tables, not just view columns.+ for i, te := range q.TargetList {+ t.Logf(" target[%d]: name=%s expr=%T", i, te.ResName, te.Expr)+ }++ // Step 5: try a query that uses enum-typed column with operators that+ // would normally need enum resolution.+ res, err := analyzeSafe(cat, `SELECT id FROM open_tasks WHERE status = 'pending'`)+ t.Logf("query with WHERE on pseudo-enum column: err=%v targets=%d", err, targetsOf(res))++ t.Log("cascade prevention PoC: pseudo-enum at root allowed T and V to install as real objects")+}++// ---- Corner case 1: mixed real + pseudo overloads of the same function name ----+//+// Scenario: function fn has two overloads in metadata:+// fn(int4) → int4 (real install succeeds)+// fn(my_broken_type) → my_broken_type (real install fails because type is broken)+//+// catalog loader strategy: keep fn(int4) real, install pseudo fn(text) for the broken one.+// Verify: calls to fn(int_col) still pick fn(int4); calls to fn(text_col) pick+// the pseudo fn(text); calls to fn(literal_string) pick the pseudo.++func TestLoaderPoC_MixedRealAndPseudoOverloads(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ makeFuncStmt := func(name string, argType, retType string) *ast.CreateFunctionStmt {+ return &ast.CreateFunctionStmt{+ Funcname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: name},+ }},+ Parameters: &ast.List{Items: []ast.Node{+ &ast.FunctionParameter{ArgType: typeName(argType), Mode: ast.FUNC_PARAM_IN},+ }},+ ReturnType: typeName(retType),+ Options: &ast.List{Items: []ast.Node{+ &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},+ &ast.DefElem{+ Defname: "as",+ Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},+ },+ }},+ }+ }++ // Install real fn(int4).+ if err := cat.CreateFunctionStmt(makeFuncStmt("fn", "int4", "int4")); err != nil {+ t.Fatalf("real fn(int4) install failed: %v", err)+ }++ // Install pseudo fn(text) — pretending the real fn(broken_type) failed.+ if err := cat.CreateFunctionStmt(makeFuncStmt("fn", "text", "text")); err != nil {+ t.Fatalf("pseudo fn(text) install failed: %v", err)+ }++ procs := cat.LookupProcByName("fn")+ t.Logf("fn overloads after mixed install: %d", len(procs))+ if len(procs) < 2 {+ t.Errorf("expected >= 2 fn overloads, got %d", len(procs))+ }++ // Install a table with one int and one text column.+ if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("n", typeName("int4"), false),+ columnDef("s", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("install t: %v", err)+ }++ // Each query exercises a different resolution path.+ queries := []struct {+ name string+ sql string+ }{+ {"fn(int_col) → should pick real fn(int4)", `SELECT id, fn(n) AS r FROM t`},+ {"fn(text_col) → should pick pseudo fn(text)", `SELECT id, fn(s) AS r FROM t`},+ {"fn(literal_int) → picks fn(int4) or via implicit cast", `SELECT fn(42) FROM t`},+ {"fn(literal_text) → picks fn(text)", `SELECT fn('hello') FROM t`},+ }+ for _, q := range queries {+ res, err := analyzeSafe(cat, q.sql)+ t.Logf("%s: err=%v targets=%d", q.name, err, targetsOf(res))+ }+}++// ---- Corner case 2: pseudo install after a real install has truly failed ----+//+// Scenario: real DefineDomain fails because the base type does not exist.+// The loader must then install a pseudo in the same slot. Verify:+// 1. real install actually returns an error (otherwise this test is vacuous)+// 2. catalog does not hold partial state for the failed name+// 3. pseudo install at the same name succeeds+// 4. a table column using that pseudo name resolves correctly++func TestLoaderPoC_PseudoAfterFailedReal(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ // Step 1: try a real DefineDomain with a non-existent base type.+ badStmt := &ast.CreateDomainStmt{+ Domainname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "status_code"},+ }},+ Typname: typeName("nonexistent_base_type_xyz"),+ }+ err := cat.DefineDomain(badStmt)+ t.Logf("real DefineDomain with bogus base type: err=%v", err)++ realFailed := err != nil++ // Step 2: check catalog state. If the real install created a partial entry,+ // the pseudo install at the same name may conflict.+ //+ // Probe by trying to look up the name via a constructed type reference.+ // There's no direct "is this type defined?" API, but we can try the pseudo+ // install and see if it errors with a duplicate.++ // Step 3: install pseudo domain over text at the same name.+ pseudoStmt := &ast.CreateDomainStmt{+ Domainname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "status_code"},+ }},+ Typname: typeName("text"),+ }+ err = cat.DefineDomain(pseudoStmt)+ t.Logf("pseudo DefineDomain (base=text) after real failure: err=%v", err)++ if err != nil {+ t.Errorf("pseudo install after real failure should succeed; got %v", err)+ return+ }++ // Step 4: use the pseudo in a table column.+ if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("code", mustTypeName(t, "public.status_code"), false),+ }), 'r'); err != nil {+ t.Fatalf("table with pseudo-domain column install failed: %v", err)+ }++ q := mustAnalyze(t, cat, `SELECT id, code FROM t WHERE code = 'ok'`)+ if len(q.TargetList) != 2 {+ t.Errorf("query on pseudo-domain column: got %d targets, want 2", len(q.TargetList))+ }++ if !realFailed {+ t.Log("note: real DefineDomain with bogus base type DID NOT fail — " ++ "omni's validation is looser than expected. Test still exercises " ++ "the pseudo install path but the 'recovery after failure' premise " ++ "is weaker. Consider whether the loader needs to handle this case.")+ } else {+ t.Log("pseudo recovery PoC: real install failed cleanly, pseudo at same " ++ "name installed successfully, downstream table install works")+ }+}++// ---- helpers for loader tests ----++// ---- Install cost benchmarks: catalog-loader hand-built vs Exec(DDL text) ----+//+// Validates the plan's "5-10× faster than current" claim empirically.+// Generates synthetic schemas of varying size with representative shapes+// (tables with mixed-type columns) and times both install paths.+//+// Run with:+// go test -bench='BenchmarkLoader_Install' -benchmem -run='^$' -count=3 \+// github.com/bytebase/bytebase/backend/plugin/parser/pg++func makeSyntheticTables(n int) []*ast.CreateStmt {+ tables := make([]*ast.CreateStmt, n)+ for i := range n {+ tables[i] = buildCreateTableStmt("public", fmt.Sprintf("t_%d", i), []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("name", typeName("text"), false),+ columnDef("created_at", typeName("timestamp"), false),+ columnDef("amount", typeName("int8"), false),+ columnDef("status", typeName("text"), false),+ })+ }+ return tables+}++func makeSyntheticDDL(n int) string {+ var sb strings.Builder+ sb.Grow(n * 200)+ for i := range n {+ fmt.Fprintf(&sb, "CREATE TABLE public.t_%d (id int4 NOT NULL, name text, created_at timestamp, amount int8, status text);\n", i)+ }+ return sb.String()+}++func benchmarkE3Install(b *testing.B, n int) {+ tables := makeSyntheticTables(n)+ b.ResetTimer()+ for range b.N {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ for _, stmt := range tables {+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ b.Fatalf("DefineRelation: %v", err)+ }+ }+ }+}++func benchmarkDDLInstall(b *testing.B, n int) {+ ddl := makeSyntheticDDL(n)+ b.ResetTimer()+ for range b.N {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ if _, err := cat.Exec(ddl, &catalog.ExecOptions{ContinueOnError: true}); err != nil {+ b.Fatalf("Exec: %v", err)+ }+ }+}++func BenchmarkLoader_Install_E3_100(b *testing.B) { benchmarkE3Install(b, 100) }+func BenchmarkLoader_Install_E3_500(b *testing.B) { benchmarkE3Install(b, 500) }+func BenchmarkLoader_Install_E3_2000(b *testing.B) { benchmarkE3Install(b, 2000) }+func BenchmarkLoader_Install_DDL_100(b *testing.B) { benchmarkDDLInstall(b, 100) }+func BenchmarkLoader_Install_DDL_500(b *testing.B) { benchmarkDDLInstall(b, 500) }+func BenchmarkLoader_Install_DDL_2000(b *testing.B) { benchmarkDDLInstall(b, 2000) }++// ---- Overload selection assertions: prove analyzer picks the right overload ----+//+// Builds on TestLoaderPoC_MixedRealAndPseudoOverloads. That test logs outcomes+// but does not assert which overload got picked. Here we install two overloads+// with DISTINCT return types (int4 vs text) and assert that the analyzer picks+// the right one by checking the result column's TypeOID.+//+// Well-known PG OIDs: int4=23, text=25 (stable; used throughout omni tests).++func TestLoaderPoC_OverloadSelectionAsserted(t *testing.T) {+ const (+ oidInt4 uint32 = 23+ oidText uint32 = 25+ )++ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ mkFn := func(arg, ret string) *ast.CreateFunctionStmt {+ return &ast.CreateFunctionStmt{+ Funcname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: "fn"},+ }},+ Parameters: &ast.List{Items: []ast.Node{+ &ast.FunctionParameter{ArgType: typeName(arg), Mode: ast.FUNC_PARAM_IN},+ }},+ ReturnType: typeName(ret),+ Options: &ast.List{Items: []ast.Node{+ &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},+ &ast.DefElem{+ Defname: "as",+ Arg: &ast.List{Items: []ast.Node{&ast.String{Str: "SELECT $1"}}},+ },+ }},+ }+ }++ // Real fn(int4) → int4. Pseudo fn(text) → text (simulated: pretend the+ // real fn(broken_type) failed and we installed a text-backed pseudo).+ if err := cat.CreateFunctionStmt(mkFn("int4", "int4")); err != nil {+ t.Fatalf("fn(int4) install failed: %v", err)+ }+ if err := cat.CreateFunctionStmt(mkFn("text", "text")); err != nil {+ t.Fatalf("fn(text) install failed: %v", err)+ }++ if err := cat.DefineRelation(buildCreateTableStmt("public", "t", []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("n", typeName("int4"), false),+ columnDef("s", typeName("text"), false),+ }), 'r'); err != nil {+ t.Fatalf("table install: %v", err)+ }++ exprResultType := func(q *catalog.Query, targetIdx int) uint32 {+ t.Helper()+ if targetIdx >= len(q.TargetList) {+ t.Fatalf("target index %d out of range (have %d)", targetIdx, len(q.TargetList))+ }+ te := q.TargetList[targetIdx]+ fce, ok := te.Expr.(*catalog.FuncCallExpr)+ if !ok {+ t.Fatalf("target[%d] expected FuncCallExpr, got %T", targetIdx, te.Expr)+ }+ return fce.ResultType+ }++ // Case 1: fn(int_col) must pick fn(int4) → returns int4+ q := mustAnalyze(t, cat, `SELECT fn(n) FROM t`)+ if got := exprResultType(q, 0); got != oidInt4 {+ t.Errorf("fn(int_col): expected ResultType=%d (int4), got %d", oidInt4, got)+ }++ // Case 2: fn(text_col) must pick fn(text) → returns text+ q = mustAnalyze(t, cat, `SELECT fn(s) FROM t`)+ if got := exprResultType(q, 0); got != oidText {+ t.Errorf("fn(text_col): expected ResultType=%d (text), got %d", oidText, got)+ }++ // Case 3: fn(42) — integer literal — must pick fn(int4)+ q = mustAnalyze(t, cat, `SELECT fn(42) FROM t`)+ if got := exprResultType(q, 0); got != oidInt4 {+ t.Errorf("fn(literal_int): expected ResultType=%d (int4), got %d", oidInt4, got)+ }++ // Case 4: fn('hello') — string literal — must pick fn(text)+ q = mustAnalyze(t, cat, `SELECT fn('hello') FROM t`)+ if got := exprResultType(q, 0); got != oidText {+ t.Errorf("fn(literal_text): expected ResultType=%d (text), got %d", oidText, got)+ }++ t.Log("overload selection asserted: analyzer picks the right overload by arg type; " ++ "mixed real+pseudo overloads resolve correctly")+}++// ---- REAL cascade prevention: force a real install failure, then install pseudo ----+//+// The previous cascade test simulated failure by skipping the real install+// entirely. This test actually runs a real install that fails at the omni+// level, then proves the loader pattern "catch error → install pseudo at+// same slot" lets downstream installs use real AST.+//+// Scenario:+// Domain D (base=nonexistent_type_xyz) — real DefineDomain ERRORS+// Table T (col D, title text) — real DefineRelation must succeed against pseudo D+// View V (SELECT col, title FROM T) — real DefineView must succeed against real T+// Query SELECT col, title FROM V — must analyze with lineage to T.{col, title}++func TestLoaderPoC_RealCascadeWithGenuineFailure(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})++ installOne := func(kind, name string, installReal func() error, installPseudo func() error) (degraded bool) {+ if err := installReal(); err != nil {+ t.Logf("[%s %q] real install failed: %v", kind, name, err)+ if pErr := installPseudo(); pErr != nil {+ t.Fatalf("[%s %q] pseudo install also failed: %v", kind, name, pErr)+ }+ t.Logf("[%s %q] pseudo installed at same slot", kind, name)+ return true+ }+ t.Logf("[%s %q] real install succeeded", kind, name)+ return false+ }++ // Step 1: install D. Real fails (nonexistent base), pseudo succeeds.+ dName := "d_status"+ dDegraded := installOne("domain", dName,+ func() error {+ return cat.DefineDomain(&ast.CreateDomainStmt{+ Domainname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: dName},+ }},+ Typname: typeName("nonexistent_base_type_for_test_xyz"),+ })+ },+ func() error {+ return cat.DefineDomain(&ast.CreateDomainStmt{+ Domainname: &ast.List{Items: []ast.Node{+ &ast.String{Str: "public"},+ &ast.String{Str: dName},+ }},+ Typname: typeName("text"),+ })+ },+ )+ if !dDegraded {+ t.Fatal("expected D real install to fail (test premise broken)")+ }++ // Step 2: install T. This is a REAL install (T's metadata is fine).+ // The column references the pseudo D, so DefineRelation must find it.+ tName := "records"+ tDegraded := installOne("table", tName,+ func() error {+ return cat.DefineRelation(buildCreateTableStmt("public", tName, []*ast.ColumnDef{+ columnDef("id", typeName("int4"), true),+ columnDef("col", mustTypeName(t, "public."+dName), false),+ columnDef("title", typeName("text"), false),+ }), 'r')+ },+ func() error {+ return errors.Errorf("should not reach pseudo for table %s", tName)+ },+ )+ if tDegraded {+ t.Fatal("T should install as REAL against pseudo D — cascade prevention violated")+ }++ // Step 3: install V. REAL install. View body references T (which is real).+ vName := "records_view"+ vDegraded := installOne("view", vName,+ func() error {+ sel := mustParseSelect(t, `SELECT id, col, title FROM records WHERE id > 0`)+ return cat.DefineView(&ast.ViewStmt{+ View: &ast.RangeVar{+ Schemaname: "public",+ Relname: vName,+ Relpersistence: 'p',+ },+ Query: sel,+ })+ },+ func() error {+ return errors.Errorf("should not reach pseudo for view %s", vName)+ },+ )+ if vDegraded {+ t.Fatal("V should install as REAL against real T — cascade prevention violated")+ }++ // Step 4: query V. Verify lineage makes it all the way to base table columns.+ q := mustAnalyze(t, cat, `SELECT id, col, title FROM records_view`)+ if len(q.TargetList) != 3 {+ t.Fatalf("view query: got %d targets, want 3", len(q.TargetList))+ }+ for i, te := range q.TargetList {+ if _, isVar := te.Expr.(*catalog.VarExpr); !isVar {+ t.Errorf("target[%d] (%s): expected VarExpr (resolved lineage), got %T",+ i, te.ResName, te.Expr)+ }+ }++ // Step 5: query with WHERE on the pseudo-backed column — text semantics+ // should let this analyze successfully.+ q2, err := analyzeSafe(cat, `SELECT id FROM records_view WHERE col = 'active'`)+ if err != nil {+ t.Errorf("WHERE on pseudo-domain column should analyze; got err=%v", err)+ } else if len(q2.TargetList) != 1 {+ t.Errorf("WHERE query: got %d targets, want 1", len(q2.TargetList))+ }++ t.Log("REAL cascade PoC: genuine DefineDomain failure → pseudo at same slot → " ++ "downstream table and view install as real → query lineage intact")+}++// analyzeSafe runs AnalyzeSelectStmt but returns (nil, err) instead of failing+// the test — probe tests need to record whether analyze succeeded.+func analyzeSafe(cat *catalog.Catalog, sql string) (*catalog.Query, error) {+ stmts, err := ParsePg(sql)+ if err != nil {+ return nil, errors.Wrap(err, "parse")+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ return nil, errors.Errorf("not a SelectStmt: %T", stmts[0].AST)+ }+ return cat.AnalyzeSelectStmt(sel)+}++func targetsOf(q *catalog.Query) int {+ if q == nil {+ return -1+ }+ return len(q.TargetList)+}diff --git a/backend/plugin/parser/pg/query_span_loader_pseudo.go b/backend/plugin/parser/pg/query_span_loader_pseudo.gonew file mode 100644index 0000000000..90a8b2fd50--- /dev/null+++ b/backend/plugin/parser/pg/query_span_loader_pseudo.go@@ -0,0 +1,272 @@+package pg++import (+ "strings"++ "github.com/pkg/errors"++ "github.com/bytebase/omni/pg/ast"+)++// This file builds "pseudo" AST nodes used by the Catalog loader when a real+// install fails. Every pseudo form is backed by built-in types only (text,+// primarily) so pseudo install itself cannot cascade into further failures.+//+// Pseudo forms preserve object names and — where metadata allows — column+// names. Type fidelity is intentionally dropped. For query span this is+// sufficient in the vast majority of cases: lineage is a name-resolution+// problem, not a type-resolution problem.+//+// Every form in this file corresponds to a PoC test in+// query_span_v5_poc_test.go (TestLoaderPoC_Pseudo*).++// pseudoTextTypeName is the single pseudo column/parameter/return type omni+// accepts uniformly. Every pseudo form ends up routing to this shape.+func pseudoTextTypeName() *ast.TypeName {+ return &ast.TypeName{+ Names: &ast.List{Items: []ast.Node{&ast.String{Str: "text"}}},+ Typemod: -1,+ }+}++// pseudoCreateEnumStmt builds a minimal enum with zero values. omni's+// DefineEnum accepts an empty Vals list.+func pseudoCreateEnumStmt(schema, name string) *ast.CreateEnumStmt {+ return &ast.CreateEnumStmt{+ TypeName: qualifiedNameList(schema, name),+ Vals: &ast.List{Items: []ast.Node{}},+ }+}++// pseudoCreateDomainStmt builds a domain over text with no constraints.+// Query span does not inspect domain constraints, so dropping them is safe.+func pseudoCreateDomainStmt(schema, name string) *ast.CreateDomainStmt {+ return &ast.CreateDomainStmt{+ Domainname: qualifiedNameList(schema, name),+ Typname: pseudoTextTypeName(),+ }+}++// pseudoCompositeTypeStmt builds a composite type with a single text field+// named "_broken". Without CompositeTypeMetadata in storepb we cannot+// preserve real field names, so (col).field access against a pseudo composite+// will fall through to extractFallbackColumns.+func pseudoCompositeTypeStmt(schema, name string) *ast.CompositeTypeStmt {+ return &ast.CompositeTypeStmt{+ Typevar: &ast.RangeVar{+ Schemaname: schema,+ Relname: name,+ },+ Coldeflist: &ast.List{Items: []ast.Node{+ &ast.ColumnDef{+ Colname: "_broken",+ TypeName: pseudoTextTypeName(),+ },+ }},+ }+}++// pseudoCreateRangeStmt builds a range over text. text has btree ordering so+// omni accepts it as a valid subtype.+func pseudoCreateRangeStmt(schema, name string) *ast.CreateRangeStmt {+ return &ast.CreateRangeStmt{+ TypeName: qualifiedNameList(schema, name),+ Params: &ast.List{Items: []ast.Node{+ &ast.DefElem{+ Defname: "subtype",+ Arg: pseudoTextTypeName(),+ },+ }},+ }+}++// pseudoCreateTableStmt builds a table whose columns are all text-typed but+// retain their metadata names. This is the fallback used when real install+// fails because of unresolved column types (broken enum, unknown composite,+// etc.).+func pseudoCreateTableStmt(schema, name string, columnNames []string) *ast.CreateStmt {+ items := make([]ast.Node, 0, len(columnNames))+ for _, col := range columnNames {+ if col == "" {+ continue+ }+ items = append(items, &ast.ColumnDef{+ Colname: col,+ TypeName: pseudoTextTypeName(),+ })+ }+ return &ast.CreateStmt{+ Relation: &ast.RangeVar{+ Schemaname: schema,+ Relname: name,+ Relpersistence: 'p',+ },+ TableElts: &ast.List{Items: items},+ }+}++// pseudoViewStmt builds a view whose body is a constant SELECT target list+// exposing each metadata column name as a nullable text expression. The body+// has no FROM clause.+//+// Returns an error if the body cannot be parsed; callers should treat this+// as a "truly broken" object.+func pseudoViewStmt(schema, name string, columnNames []string) (*ast.ViewStmt, error) {+ sel, err := pseudoConstantSelect(columnNames)+ if err != nil {+ return nil, err+ }+ return &ast.ViewStmt{+ View: &ast.RangeVar{+ Schemaname: schema,+ Relname: name,+ Relpersistence: 'p',+ },+ Query: sel,+ }, nil+}++// pseudoCreateTableAsStmt builds a pseudo materialized view using the same+// constant-target-list body as pseudoViewStmt.+func pseudoCreateTableAsStmt(schema, name string, columnNames []string) (*ast.CreateTableAsStmt, error) {+ sel, err := pseudoConstantSelect(columnNames)+ if err != nil {+ return nil, err+ }+ return &ast.CreateTableAsStmt{+ Query: sel,+ Objtype: ast.OBJECT_MATVIEW,+ Into: &ast.IntoClause{+ Rel: &ast.RangeVar{+ Schemaname: schema,+ Relname: name,+ Relpersistence: 'p',+ },+ },+ }, nil+}++// pseudoCreateFunctionStmt builds a function whose signature collapses every+// parameter to text and whose return type is text. argCount is taken from the+// real function's signature so analyzer overload resolution by arity still+// matches against the pseudo.+//+// The function body is `SELECT $1` when argCount > 0, `SELECT NULL::text`+// otherwise — always a valid SQL-language body.+func pseudoCreateFunctionStmt(schema, name string, argCount int) *ast.CreateFunctionStmt {+ stmt := &ast.CreateFunctionStmt{+ Funcname: qualifiedNameList(schema, name),+ ReturnType: pseudoTextTypeName(),+ }+ if argCount > 0 {+ params := make([]ast.Node, argCount)+ for i := range argCount {+ params[i] = &ast.FunctionParameter{+ ArgType: pseudoTextTypeName(),+ Mode: ast.FUNC_PARAM_IN,+ }+ }+ stmt.Parameters = &ast.List{Items: params}+ }+ body := "SELECT NULL::text"+ if argCount > 0 {+ body = "SELECT $1::text"+ }+ stmt.Options = &ast.List{Items: []ast.Node{+ &ast.DefElem{Defname: "language", Arg: &ast.String{Str: "sql"}},+ &ast.DefElem{+ Defname: "as",+ Arg: &ast.List{Items: []ast.Node{&ast.String{Str: body}}},+ },+ }}+ return stmt+}++// pseudoConstantSelect parses "SELECT NULL::text AS col1, NULL::text AS col2 ..."+// and returns the resulting *ast.SelectStmt. Columns with empty names are+// skipped; if the resulting target list is empty, a single anonymous column+// is emitted so the SELECT is well-formed.+func pseudoConstantSelect(columnNames []string) (*ast.SelectStmt, error) {+ var targets []string+ for _, col := range columnNames {+ if col == "" {+ continue+ }+ targets = append(targets, "NULL::text AS "+quoteIdent(col))+ }+ if len(targets) == 0 {+ targets = []string{"NULL::text"}+ }+ sql := "SELECT " + strings.Join(targets, ", ")+ stmts, err := ParsePg(sql)+ if err != nil {+ return nil, errors.Wrap(err, "parse pseudo constant select")+ }+ if len(stmts) != 1 {+ return nil, errors.Errorf("pseudo constant select: expected 1 statement, got %d", len(stmts))+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ return nil, errors.Errorf("pseudo constant select: expected SelectStmt, got %T", stmts[0].AST)+ }+ return sel, nil+}++// functionArgCountFromSignature parses a signature string like+// "my_func(integer, text)" and returns the number of arguments. A signature+// with no parentheses or an empty argument list returns 0.+func functionArgCountFromSignature(signature string) int {+ openIdx := strings.Index(signature, "(")+ if openIdx < 0 {+ return 0+ }+ closeIdx := strings.LastIndex(signature, ")")+ if closeIdx < 0 || closeIdx <= openIdx {+ return 0+ }+ inner := strings.TrimSpace(signature[openIdx+1 : closeIdx])+ if inner == "" {+ return 0+ }+ // Split on top-level commas. Type names inside parens (rare, e.g.+ // numeric(10,2)) must not count as separators.+ depth := 0+ count := 1+ for _, r := range inner {+ switch r {+ case '(':+ depth+++ case ')':+ if depth > 0 {+ depth--+ }+ case ',':+ if depth == 0 {+ count+++ }+ default:+ // other runes contribute nothing to arg-count accounting+ }+ }+ return count+}++// ---------------- small AST helpers shared with builders.go ----------------++// qualifiedNameList builds an omni List of String nodes representing a+// schema-qualified object name. An empty schema emits only the name.+func qualifiedNameList(schema, name string) *ast.List {+ items := make([]ast.Node, 0, 2)+ if schema != "" {+ items = append(items, &ast.String{Str: schema})+ }+ items = append(items, &ast.String{Str: name})+ return &ast.List{Items: items}+}++// quoteIdent wraps an identifier in double quotes, doubling any internal+// double quotes. Used when a metadata column name may contain special chars+// or collide with SQL reserved words.+func quoteIdent(name string) string {+ return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`+}diff --git a/backend/plugin/parser/pg/query_span_loader_test.go b/backend/plugin/parser/pg/query_span_loader_test.gonew file mode 100644index 0000000000..82152f4456--- /dev/null+++ b/backend/plugin/parser/pg/query_span_loader_test.go@@ -0,0 +1,912 @@+package pg++import (+ "context"+ "strings"+ "testing"++ "github.com/pkg/errors"++ "github.com/bytebase/omni/pg/ast"+ "github.com/bytebase/omni/pg/catalog"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// ---------------- typeNameFromString ----------------++func TestLoaderTypeNameFromString(t *testing.T) {+ cases := []struct {+ in string+ wantErr bool+ }{+ {"int4", false},+ {"text", false},+ {"numeric(10,2)", false},+ {"timestamp(3) with time zone", false},+ {"character varying(255)", false},+ {"text[]", false},+ {"public.task_status", false},+ {`"public"."task_status"`, false},+ {"", true}, // empty cannot parse as SELECT NULL::+ }+ for _, c := range cases {+ tn, err := typeNameFromString(c.in)+ if c.wantErr {+ if err == nil {+ t.Errorf("typeNameFromString(%q): expected error, got tn=%+v", c.in, tn)+ }+ continue+ }+ if err != nil {+ t.Errorf("typeNameFromString(%q): unexpected error: %v", c.in, err)+ continue+ }+ if tn == nil {+ t.Errorf("typeNameFromString(%q): nil TypeName with no error", c.in)+ }+ }+}++// ---------------- extractUserTypeRefs ----------------++func TestLoaderExtractUserTypeRefs(t *testing.T) {+ cases := []struct {+ in string+ want []UserTypeRef+ }{+ // Built-in scalars.+ {"integer", nil},+ {"int4", nil},+ {"text", nil},+ {"bigint", nil},+ {"boolean", nil},+ {"json", nil},+ {"jsonb", nil},+ {"uuid", nil},+ {"date", nil},+ {"interval", nil},++ // Built-in with length / precision / timezone.+ {"numeric(10,2)", nil},+ {"numeric(8)", nil},+ {"decimal(10,2)", nil},+ {"character varying(255)", nil},+ {"character(10)", nil},+ {"varchar(255)", nil},+ {"bit(8)", nil},+ {"bit varying(8)", nil},+ {"timestamp(3) with time zone", nil},+ {"time(6) without time zone", nil},+ {"timestamp without time zone", nil},++ // USER-DEFINED.+ {"public.task_status", []UserTypeRef{{Schema: "public", Name: "task_status"}}},+ {"myschema.my_domain", []UserTypeRef{{Schema: "myschema", Name: "my_domain"}}},++ // Quoted qualified name.+ {`"MySchema"."MyType"`, []UserTypeRef{{Schema: "MySchema", Name: "MyType"}}},++ // PG internal array form.+ {"_text", nil},+ {"_int4", nil},+ {"_task_status", nil}, // sync.go:834 drops the schema; we do not topo-sort it.++ // System schemas.+ {"pg_catalog.int4", nil},+ {"information_schema.cardinal_number", nil},+ {"pg_toast.pg_toast_2619", nil},++ // Empty.+ {"", nil},++ // Whitespace.+ {" text ", nil},+ }+ for _, c := range cases {+ got := extractUserTypeRefs(c.in)+ if !userTypeRefsEqual(got, c.want) {+ t.Errorf("extractUserTypeRefs(%q): got %+v, want %+v", c.in, got, c.want)+ }+ }+}++func userTypeRefsEqual(a, b []UserTypeRef) bool {+ if len(a) != len(b) {+ return false+ }+ for i := range a {+ if a[i] != b[i] {+ return false+ }+ }+ return true+}++// ---------------- helpers ----------------++func TestLoaderStripTypeModifiers(t *testing.T) {+ cases := []struct {+ in, want string+ }{+ {"numeric(10,2)", "numeric"},+ {"timestamp(3) with time zone", "timestamp"},+ {"timestamp without time zone", "timestamp"},+ {"character varying(255)", "character varying"},+ {" text ", "text"},+ {"public.task_status", "public.task_status"},+ {"", ""},+ }+ for _, c := range cases {+ if got := stripTypeModifiers(c.in); got != c.want {+ t.Errorf("stripTypeModifiers(%q): got %q, want %q", c.in, got, c.want)+ }+ }+}++func TestLoaderSplitQualifiedName(t *testing.T) {+ cases := []struct {+ in string+ wantSchema string+ wantName string+ wantOk bool+ }{+ {"public.foo", "public", "foo", true},+ {`"weird name"."another"`, "weird name", "another", true},+ {"foo", "", "", false},+ {"", "", "", false},+ {"a.b.c", "", "", false}, // too many parts+ }+ for _, c := range cases {+ s, n, ok := splitQualifiedName(c.in)+ if s != c.wantSchema || n != c.wantName || ok != c.wantOk {+ t.Errorf("splitQualifiedName(%q): got (%q,%q,%v), want (%q,%q,%v)",+ c.in, s, n, ok, c.wantSchema, c.wantName, c.wantOk)+ }+ }+}++func TestLoaderIsSystemSchema(t *testing.T) {+ // Sanity check that extractUserTypeRefs routes system schemas to nil via+ // IsSystemSchema. The underlying predicate is tested in system_objects.go.+ cases := map[string]bool{+ "pg_catalog.foo": true,+ "information_schema.bar": true,+ "pg_toast.baz": true,+ "pg_temp_1.qux": true,+ "public.foo": false,+ "myschema.bar": false,+ }+ for in, wantFiltered := range cases {+ got := extractUserTypeRefs(in)+ if wantFiltered && got != nil {+ t.Errorf("extractUserTypeRefs(%q): expected filtered, got %+v", in, got)+ }+ if !wantFiltered && got == nil {+ t.Errorf("extractUserTypeRefs(%q): expected non-nil, got nil", in)+ }+ }+}++// ---------------- classifyAnalyzeError ----------------++func TestLoaderClassifyAnalyzeError(t *testing.T) {+ cases := []struct {+ name string+ err error+ want fallbackReason+ }{+ {"nil", nil, reasonNone},+ {+ "undefined function",+ &catalog.Error{Code: catalog.CodeUndefinedFunction, Message: "function fn(text) does not exist"},+ reasonExpectedPseudoSemantic,+ },+ {+ "ambiguous function",+ &catalog.Error{Code: catalog.CodeAmbiguousFunction, Message: "function fn is not unique"},+ reasonExpectedPseudoSemantic,+ },+ {+ "datatype mismatch",+ &catalog.Error{Code: catalog.CodeDatatypeMismatch, Message: "UNION types text and integer cannot be matched"},+ reasonExpectedPseudoSemantic,+ },+ {+ "feature not supported",+ &catalog.Error{Code: catalog.CodeFeatureNotSupported, Message: "not supported"},+ reasonExpectedPseudoSemantic,+ },+ {+ "ambiguous column",+ &catalog.Error{Code: catalog.CodeAmbiguousColumn, Message: "column x is ambiguous"},+ reasonExpectedPseudoSemantic,+ },+ {+ "undefined table",+ &catalog.Error{Code: catalog.CodeUndefinedTable, Message: `relation "foo" does not exist`},+ reasonUndefinedReference,+ },+ {+ "undefined column",+ &catalog.Error{Code: catalog.CodeUndefinedColumn, Message: `column "x" does not exist`},+ reasonUndefinedReference,+ },+ {+ "undefined object",+ &catalog.Error{Code: catalog.CodeUndefinedObject, Message: `type "t" does not exist`},+ reasonUndefinedReference,+ },+ {+ "undefined schema",+ &catalog.Error{Code: catalog.CodeUndefinedSchema, Message: `schema "s" does not exist`},+ reasonUndefinedReference,+ },+ {+ "other catalog error",+ &catalog.Error{Code: catalog.CodeDuplicateTable, Message: "duplicate"},+ reasonAnalyzerUnsupported,+ },+ {+ "plain error",+ errors.New("unsupported node type"),+ reasonAnalyzerUnsupported,+ },+ {+ "wrapped catalog error",+ errors.Wrap(&catalog.Error{Code: catalog.CodeDatatypeMismatch, Message: "mismatch"}, "ctx"),+ reasonExpectedPseudoSemantic,+ },+ }+ for _, c := range cases {+ if got := classifyAnalyzeError(c.err); got != c.want {+ t.Errorf("%s: classifyAnalyzeError got %v, want %v", c.name, got, c.want)+ }+ }+}++// ---------------- pseudo builders ----------------++func TestLoaderPseudoEnum(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ if err := cat.DefineEnum(pseudoCreateEnumStmt("public", "broken")); err != nil {+ t.Fatalf("DefineEnum: %v", err)+ }+ // Reference it from a table column and analyze.+ tn, err := typeNameFromString("public.broken")+ if err != nil {+ t.Fatalf("typeNameFromString: %v", err)+ }+ tbl := &ast.CreateStmt{+ Relation: &ast.RangeVar{Schemaname: "public", Relname: "t", Relpersistence: 'p'},+ TableElts: &ast.List{Items: []ast.Node{+ &ast.ColumnDef{Colname: "s", TypeName: tn},+ }},+ }+ if err := cat.DefineRelation(tbl, 'r'); err != nil {+ t.Fatalf("DefineRelation with pseudo enum column: %v", err)+ }+}++func TestLoaderPseudoDomain(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ if err := cat.DefineDomain(pseudoCreateDomainStmt("public", "d")); err != nil {+ t.Fatalf("DefineDomain: %v", err)+ }+}++func TestLoaderPseudoComposite(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ if err := cat.DefineCompositeType(pseudoCompositeTypeStmt("public", "c")); err != nil {+ t.Fatalf("DefineCompositeType: %v", err)+ }+}++func TestLoaderPseudoRange(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ if err := cat.DefineRange(pseudoCreateRangeStmt("public", "r")); err != nil {+ t.Fatalf("DefineRange: %v", err)+ }+}++func TestLoaderPseudoTable(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ stmt := pseudoCreateTableStmt("public", "t", []string{"id", "name", "email"})+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation: %v", err)+ }+ rel := cat.GetRelation("public", "t")+ if rel == nil {+ t.Fatal("relation not installed")+ }+ if len(rel.Columns) != 3 {+ t.Errorf("column count: got %d, want 3", len(rel.Columns))+ }+}++func TestLoaderPseudoView(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ stmt, err := pseudoViewStmt("public", "v", []string{"id", "label"})+ if err != nil {+ t.Fatalf("build pseudo view: %v", err)+ }+ if err := cat.DefineView(stmt); err != nil {+ t.Fatalf("DefineView: %v", err)+ }+ rel := cat.GetRelation("public", "v")+ if rel == nil {+ t.Fatal("view not installed")+ }+ if len(rel.Columns) != 2 {+ t.Errorf("view column count: got %d, want 2", len(rel.Columns))+ }+}++func TestLoaderPseudoMatView(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ stmt, err := pseudoCreateTableAsStmt("public", "m", []string{"x", "y"})+ if err != nil {+ t.Fatalf("build pseudo matview: %v", err)+ }+ if err := cat.ExecCreateTableAs(stmt); err != nil {+ t.Fatalf("ExecCreateTableAs: %v", err)+ }+}++func TestLoaderPseudoFunction(t *testing.T) {+ cases := []struct {+ name string+ argCount int+ }{+ {"zero args", 0},+ {"one arg", 1},+ {"three args", 3},+ }+ for _, c := range cases {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ stmt := pseudoCreateFunctionStmt("public", "fn", c.argCount)+ if err := cat.CreateFunctionStmt(stmt); err != nil {+ t.Errorf("%s: CreateFunctionStmt: %v", c.name, err)+ continue+ }+ procs := cat.LookupProcByName("fn")+ if len(procs) == 0 {+ t.Errorf("%s: no procs installed", c.name)+ }+ }+}++func TestLoaderFunctionArgCountFromSignature(t *testing.T) {+ cases := map[string]int{+ "": 0,+ "fn()": 0,+ "fn(integer)": 1,+ "fn(integer, text)": 2,+ "fn(integer, text, uuid)": 3,+ "fn(numeric(10,2))": 1, // inner paren must not split+ "fn(numeric(10,2), int)": 2,+ }+ for in, want := range cases {+ if got := functionArgCountFromSignature(in); got != want {+ t.Errorf("functionArgCountFromSignature(%q): got %d, want %d", in, got, want)+ }+ }+}++func TestLoaderQualifiedNameList(t *testing.T) {+ list := qualifiedNameList("public", "foo")+ if list == nil || len(list.Items) != 2 {+ t.Fatalf("qualifiedNameList(public, foo): got %+v, want 2 items", list)+ }+ bare := qualifiedNameList("", "foo")+ if bare == nil || len(bare.Items) != 1 {+ t.Fatalf("qualifiedNameList(\"\", foo): got %+v, want 1 item", bare)+ }+}++// ---------------- real builders ----------------++func TestLoaderBuildCreateEnumStmt(t *testing.T) {+ stmt := buildCreateEnumStmt("public", &storepb.EnumTypeMetadata{+ Name: "task_status",+ Values: []string{"pending", "running", "done"},+ })+ if stmt == nil || stmt.Vals == nil || len(stmt.Vals.Items) != 3 {+ t.Fatalf("unexpected: %+v", stmt)+ }+ cat := catalog.New()+ if err := cat.DefineEnum(stmt); err != nil {+ t.Fatalf("DefineEnum: %v", err)+ }+}++func TestLoaderBuildCreateStmt(t *testing.T) {+ stmt, err := buildCreateStmt("public", &storepb.TableMetadata{+ Name: "t",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4", Nullable: false},+ {Name: "name", Type: "text", Nullable: true},+ {Name: "amount", Type: "numeric(10,2)", Nullable: true},+ },+ })+ if err != nil {+ t.Fatalf("buildCreateStmt: %v", err)+ }+ if stmt == nil || stmt.TableElts == nil || len(stmt.TableElts.Items) != 3 {+ t.Fatalf("unexpected: %+v", stmt)+ }+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ if err := cat.DefineRelation(stmt, 'r'); err != nil {+ t.Fatalf("DefineRelation: %v", err)+ }+ rel := cat.GetRelation("public", "t")+ if rel == nil || len(rel.Columns) != 3 {+ t.Fatalf("rel: %+v", rel)+ }+}++func TestLoaderBuildCreateStmt_BadTypeErrors(t *testing.T) {+ _, err := buildCreateStmt("public", &storepb.TableMetadata{+ Name: "t",+ Columns: []*storepb.ColumnMetadata{+ {Name: "bad", Type: "this is not a type at all;;"},+ },+ })+ if err == nil {+ t.Fatal("expected error for bad column type")+ }+}++func TestLoaderBuildCreateStmt_EmptyTypeErrors(t *testing.T) {+ _, err := buildCreateStmt("public", &storepb.TableMetadata{+ Name: "t",+ Columns: []*storepb.ColumnMetadata{{Name: "x", Type: ""}},+ })+ if err == nil {+ t.Fatal("expected error for empty column type")+ }+}++func TestLoaderBuildViewStmt(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ base, err := buildCreateStmt("public", &storepb.TableMetadata{+ Name: "orders",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "total", Type: "int4"},+ },+ })+ if err != nil {+ t.Fatalf("build base: %v", err)+ }+ if err := cat.DefineRelation(base, 'r'); err != nil {+ t.Fatalf("install base: %v", err)+ }+ stmt, err := buildViewStmt("public", &storepb.ViewMetadata{+ Name: "v",+ Definition: "SELECT id, total FROM orders",+ })+ if err != nil {+ t.Fatalf("buildViewStmt: %v", err)+ }+ if err := cat.DefineView(stmt); err != nil {+ t.Fatalf("DefineView: %v", err)+ }+}++func TestLoaderBuildViewStmt_EmptyDefinition(t *testing.T) {+ _, err := buildViewStmt("public", &storepb.ViewMetadata{Name: "v"})+ if err == nil {+ t.Fatal("expected error for empty definition")+ }+}++func TestLoaderBuildCreateTableAsStmt(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ base, err := buildCreateStmt("public", &storepb.TableMetadata{+ Name: "orders",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ },+ })+ if err != nil {+ t.Fatalf("build base: %v", err)+ }+ if err := cat.DefineRelation(base, 'r'); err != nil {+ t.Fatalf("install base: %v", err)+ }+ stmt, err := buildCreateTableAsStmt("public", &storepb.MaterializedViewMetadata{+ Name: "m",+ Definition: "SELECT id FROM orders",+ })+ if err != nil {+ t.Fatalf("buildCreateTableAsStmt: %v", err)+ }+ if err := cat.ExecCreateTableAs(stmt); err != nil {+ t.Fatalf("ExecCreateTableAs: %v", err)+ }+}++func TestLoaderBuildCreateFunctionStmt(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ stmt, err := buildCreateFunctionStmt("public", &storepb.FunctionMetadata{+ Name: "fn",+ Signature: "fn(integer, text)",+ })+ if err != nil {+ t.Fatalf("buildCreateFunctionStmt: %v", err)+ }+ if stmt.Parameters == nil || len(stmt.Parameters.Items) != 2 {+ t.Fatalf("expected 2 params, got %+v", stmt.Parameters)+ }+ if err := cat.CreateFunctionStmt(stmt); err != nil {+ t.Fatalf("CreateFunctionStmt: %v", err)+ }+}++func TestLoaderBuildCreateFunctionStmt_ZeroArgs(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ stmt, err := buildCreateFunctionStmt("public", &storepb.FunctionMetadata{+ Name: "fn",+ Signature: "fn()",+ })+ if err != nil {+ t.Fatalf("buildCreateFunctionStmt: %v", err)+ }+ if err := cat.CreateFunctionStmt(stmt); err != nil {+ t.Fatalf("CreateFunctionStmt: %v", err)+ }+}++func TestLoaderParseFunctionSignatureArgTypes(t *testing.T) {+ cases := map[string][]string{+ "fn()": nil,+ "fn(integer)": {"integer"},+ "fn(integer, text)": {"integer", "text"},+ "fn(numeric(10,2), text)": {"numeric(10,2)", "text"},+ "fn( int4 , text )": {"int4", "text"},+ "fn": nil,+ }+ for in, want := range cases {+ got, err := parseFunctionSignatureArgTypes(in)+ if err != nil {+ t.Errorf("%q: unexpected error %v", in, err)+ continue+ }+ if !stringSlicesEqual(got, want) {+ t.Errorf("%q: got %v, want %v", in, got, want)+ }+ }+}++func stringSlicesEqual(a, b []string) bool {+ if len(a) != len(b) {+ return false+ }+ for i := range a {+ if a[i] != b[i] {+ return false+ }+ }+ return true+}++func TestLoaderParseSelectBody(t *testing.T) {+ sel, err := parseSelectBody("SELECT 1 AS x")+ if err != nil {+ t.Fatalf("parseSelectBody: %v", err)+ }+ if sel == nil {+ t.Fatal("nil sel")+ }+ if _, err := parseSelectBody(""); err == nil {+ t.Error("expected error for empty body")+ }+ if _, err := parseSelectBody("SELECT 1; SELECT 2"); err == nil {+ t.Error("expected error for multi-statement body")+ }+}++// ---------------- loader ----------------++func TestLoader_HappyPath(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ meta := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ EnumTypes: []*storepb.EnumTypeMetadata{+ {Name: "task_status", Values: []string{"pending", "running"}},+ },+ Tables: []*storepb.TableMetadata{{+ Name: "tasks",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ {Name: "status", Type: "public.task_status"},+ {Name: "title", Type: "text"},+ },+ }},+ Views: []*storepb.ViewMetadata{{+ Name: "open_tasks",+ Definition: "SELECT id, title, status FROM tasks",+ DependencyColumns: []*storepb.DependencyColumn{+ {Schema: "public", Table: "tasks", Column: "id"},+ {Schema: "public", Table: "tasks", Column: "title"},+ {Schema: "public", Table: "tasks", Column: "status"},+ },+ }},+ }},+ }+ loader := newCatalogLoader(cat, meta)+ if err := loader.Load(context.Background()); err != nil {+ t.Fatalf("Load: %v", err)+ }+ if len(loader.degraded) != 0 {+ t.Errorf("expected zero degraded, got %+v", loader.degraded)+ }+ if len(loader.trulyBroken) != 0 {+ t.Errorf("expected zero truly broken, got %+v", loader.trulyBroken)+ }+ if cat.GetRelation("public", "tasks") == nil {+ t.Error("tasks not installed")+ }+ if cat.GetRelation("public", "open_tasks") == nil {+ t.Error("open_tasks not installed")+ }+}++func TestLoader_BrokenEnumCascadesToPseudo(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ meta := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Tables: []*storepb.TableMetadata{{+ Name: "t",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ // Reference a user type that is not in metadata. The+ // real install will fail; pseudo should take over.+ {Name: "status", Type: "public.nonexistent_enum"},+ },+ }},+ }},+ }+ loader := newCatalogLoader(cat, meta)+ if err := loader.Load(context.Background()); err != nil {+ t.Fatalf("Load: %v", err)+ }+ key := "rel:public.t"+ if _, ok := loader.degraded[key]; !ok {+ t.Errorf("expected public.t to be degraded, got %+v", loader.degraded)+ }+ if _, ok := loader.trulyBroken[key]; ok {+ t.Errorf("unexpected truly broken: %+v", loader.trulyBroken)+ }+ rel := cat.GetRelation("public", "t")+ if rel == nil {+ t.Fatal("pseudo t not installed")+ }+ if len(rel.Columns) != 2 {+ t.Errorf("pseudo t: got %d cols, want 2", len(rel.Columns))+ }+}++func TestLoader_TopoOrder_DependencyBeforeUse(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ meta := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ // View is listed BEFORE its base table in metadata. The loader+ // must reorder so the table is installed first; otherwise the+ // view body analyzes against an empty catalog.+ Views: []*storepb.ViewMetadata{{+ Name: "v",+ Definition: "SELECT id FROM base",+ DependencyColumns: []*storepb.DependencyColumn{+ {Schema: "public", Table: "base", Column: "id"},+ },+ }},+ Tables: []*storepb.TableMetadata{{+ Name: "base",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "int4"},+ },+ }},+ }},+ }+ loader := newCatalogLoader(cat, meta)+ if err := loader.Load(context.Background()); err != nil {+ t.Fatalf("Load: %v", err)+ }+ if loader.degraded["rel:public.v"] != nil {+ t.Errorf("view should install real, got degraded: %v", loader.degraded["rel:public.v"])+ }+ if loader.degraded["rel:public.base"] != nil {+ t.Errorf("base should install real, got degraded: %v", loader.degraded["rel:public.base"])+ }+}++func TestLoader_CycleBreaking(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ // Two views referencing each other — a metadata-level cycle.+ meta := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Views: []*storepb.ViewMetadata{+ {+ Name: "v_alpha",+ Definition: "SELECT id FROM v_beta",+ DependencyColumns: []*storepb.DependencyColumn{+ {Schema: "public", Table: "v_beta", Column: "id"},+ },+ },+ {+ Name: "v_beta",+ Definition: "SELECT id FROM v_alpha",+ DependencyColumns: []*storepb.DependencyColumn{+ {Schema: "public", Table: "v_alpha", Column: "id"},+ },+ },+ },+ }},+ }+ loader := newCatalogLoader(cat, meta)+ if err := loader.Load(context.Background()); err != nil {+ t.Fatalf("Load: %v", err)+ }+ // Both views should be in the catalog (real or pseudo). At least one+ // should be degraded (the lex-first member of the SCC).+ if cat.GetRelation("public", "v_alpha") == nil {+ t.Error("v_alpha missing")+ }+ if cat.GetRelation("public", "v_beta") == nil {+ t.Error("v_beta missing")+ }+ if len(loader.degraded) == 0 {+ t.Error("expected at least one degraded in SCC")+ }+ // v_alpha is lex-first by sortKey, so it should be the pseudo root.+ if _, ok := loader.degraded["rel:public.v_alpha"]; !ok {+ t.Errorf("lex-first v_alpha should be degraded, got %+v", loader.degraded)+ }+}++func TestLoader_LoaderObjectsTracksEverything(t *testing.T) {+ cat := catalog.New()+ meta := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ EnumTypes: []*storepb.EnumTypeMetadata{+ {Name: "e", Values: []string{"a"}},+ },+ Tables: []*storepb.TableMetadata{{+ Name: "t",+ Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "int4"}},+ }},+ }},+ }+ loader := newCatalogLoader(cat, meta)+ _ = loader.Load(context.Background())+ if !loader.loaderObjects["schema:public"] {+ t.Error("schema:public not tracked")+ }+ if !loader.loaderObjects["type:public.e"] {+ t.Error("type:public.e not tracked")+ }+ if !loader.loaderObjects["rel:public.t"] {+ t.Error("rel:public.t not tracked")+ }+}++func TestLoader_NilMetaIsNoop(t *testing.T) {+ cat := catalog.New()+ loader := newCatalogLoader(cat, nil)+ if err := loader.Load(context.Background()); err != nil {+ t.Errorf("Load(nil meta): %v", err)+ }+}++func TestLoader_NilCatalogErrors(t *testing.T) {+ loader := newCatalogLoader(nil, &storepb.DatabaseSchemaMetadata{})+ if err := loader.Load(context.Background()); err == nil {+ t.Error("expected error for nil catalog")+ }+}++// TestLoader_FunctionOverloadsAllInstalled guards against a regression where+// multiple FunctionMetadata entries sharing a name collapsed into a single+// loader object because the loader key didn't include the signature. Both+// overloads must end up in the catalog under the same name.+func TestLoader_FunctionOverloadsAllInstalled(t *testing.T) {+ cat := catalog.New()+ cat.SetSearchPath([]string{"public"})+ meta := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{{+ Name: "public",+ Functions: []*storepb.FunctionMetadata{+ {+ Name: "fn",+ Signature: "fn(integer)",+ Definition: "CREATE OR REPLACE FUNCTION public.fn(x int) RETURNS int LANGUAGE sql AS $$ SELECT $1 $$;",+ },+ {+ Name: "fn",+ Signature: "fn(text)",+ Definition: "CREATE OR REPLACE FUNCTION public.fn(x text) RETURNS text LANGUAGE sql AS $$ SELECT $1 $$;",+ },+ },+ }},+ }+ loader := newCatalogLoader(cat, meta)+ if err := loader.Load(context.Background()); err != nil {+ t.Fatalf("Load: %v", err)+ }+ if len(loader.degraded) != 0 {+ t.Errorf("expected zero degraded, got %+v", loader.degraded)+ }+ procs := cat.LookupProcByName("fn")+ if len(procs) < 2 {+ t.Errorf("expected both overloads installed, got %d procs", len(procs))+ }+ fnEntries := 0+ for k := range loader.loaderObjects {+ if strings.HasPrefix(k, "func:public.fn|") {+ fnEntries+++ }+ }+ if fnEntries != 2 {+ t.Errorf("expected 2 distinct fn loader-object keys, got %d: %+v",+ fnEntries, loader.loaderObjects)+ }+}++func TestLoaderTopoSort_Determinism(t *testing.T) {+ // Build a few fixtures and ensure topoSortObjects returns the same order+ // for the same input.+ objects := []*objectEntry{+ {kind: kindSchema, schema: "public", name: "public"},+ {kind: kindTable, schema: "public", name: "b", tableMeta: &storepb.TableMetadata{Name: "b"}},+ {kind: kindTable, schema: "public", name: "a", tableMeta: &storepb.TableMetadata{Name: "a"}},+ {kind: kindEnum, schema: "public", name: "e", enumMeta: &storepb.EnumTypeMetadata{Name: "e"}},+ }+ first := topoSortObjects(objects)+ second := topoSortObjects(objects)+ if len(first) != len(second) {+ t.Fatal("length mismatch")+ }+ for i := range first {+ if first[i].key() != second[i].key() {+ t.Errorf("non-deterministic: [%d] %s vs %s", i, first[i].key(), second[i].key())+ }+ }+}++func TestLoaderFallbackReasonString(t *testing.T) {+ cases := map[fallbackReason]string{+ reasonNone: "none",+ reasonExpectedPseudoSemantic: "expected_pseudo_semantic",+ reasonUndefinedReference: "undefined_reference",+ reasonAnalyzerUnsupported: "analyzer_unsupported",+ }+ for r, want := range cases {+ if got := r.String(); got != want {+ t.Errorf("fallbackReason(%d).String(): got %q, want %q", r, got, want)+ }+ }+}diff --git a/backend/plugin/parser/pg/query_span_loader_type_name.go b/backend/plugin/parser/pg/query_span_loader_type_name.gonew file mode 100644index 0000000000..8e85623c19--- /dev/null+++ b/backend/plugin/parser/pg/query_span_loader_type_name.go@@ -0,0 +1,279 @@+package pg++import (+ "strings"++ "github.com/pkg/errors"++ "github.com/bytebase/omni/pg/ast"+)++// UserTypeRef names a user-defined type referenced by a column, function+// parameter, or function return type. Extracted from type strings produced by+// the PG sync layer (sync.go:820-872).+type UserTypeRef struct {+ Schema string+ Name string+}++// typeNameFromString parses a PG type string into an *ast.TypeName by running+// it through omni's SELECT parser via the "SELECT NULL::<type>" trick. This+// reuses the stable SELECT parse path and avoids the DDL parse path that+// breaks on BYT-9215 / BYT-9261 class inputs.+//+// Callers must only pass values originating from bytebase sync metadata. The+// guardrails (single-statement, single-target) reject most malformed inputs+// but are not a sanitization layer for arbitrary user input.+func typeNameFromString(typeStr string) (*ast.TypeName, error) {+ stmts, err := ParsePg("SELECT NULL::" + typeStr)+ if err != nil {+ return nil, errors.Wrapf(err, "parse type %q", typeStr)+ }+ if len(stmts) != 1 {+ return nil, errors.Errorf("type %q: expected 1 statement, got %d", typeStr, len(stmts))+ }+ sel, ok := stmts[0].AST.(*ast.SelectStmt)+ if !ok {+ return nil, errors.Errorf("type %q: expected SelectStmt, got %T", typeStr, stmts[0].AST)+ }+ if sel.TargetList == nil || len(sel.TargetList.Items) != 1 {+ return nil, errors.Errorf("type %q: expected 1 target", typeStr)+ }+ rt, ok := sel.TargetList.Items[0].(*ast.ResTarget)+ if !ok {+ return nil, errors.Errorf("type %q: expected ResTarget, got %T", typeStr, sel.TargetList.Items[0])+ }+ cast, ok := rt.Val.(*ast.TypeCast)+ if !ok {+ return nil, errors.Errorf("type %q: expected TypeCast, got %T", typeStr, rt.Val)+ }+ return cast.TypeName, nil+}++// extractUserTypeRefs returns the user-defined type references embedded in a+// PG type string. Built-in types, PG internal array forms (_name), and+// system-schema-qualified types return nil.+//+// Soundness rule (hard contract C5): false negatives are acceptable (the+// loader's pseudo fallback catches them); false positives are not, because+// they invent edges in the dependency graph.+func extractUserTypeRefs(typeStr string) []UserTypeRef {+ if typeStr == "" {+ return nil+ }+ base := stripTypeModifiers(typeStr)+ if base == "" || isBuiltinType(base) {+ return nil+ }+ if strings.HasPrefix(base, "_") {+ // PG internal array form. The element may be a user type but the+ // schema is not recoverable from this representation — see the+ // sync.go:834 note in the plan.+ return nil+ }+ schema, name, ok := splitQualifiedName(base)+ if !ok {+ return nil+ }+ if IsSystemSchema(schema) {+ return nil+ }+ return []UserTypeRef{{Schema: schema, Name: name}}+}++// stripTypeModifiers normalizes a PG type string to its bare base-type form+// for allow-list lookup. It strips:+// - parenthesized type modifiers: "numeric(10,2)" -> "numeric"+// - trailing time-zone suffixes: "timestamp(3) with time zone" -> "timestamp"+// - surrounding whitespace+//+// The result may still be a qualified name like "public.task_status"; that is+// the caller's job to split.+func stripTypeModifiers(typeStr string) string {+ s := strings.TrimSpace(typeStr)+ s = stripTimeZoneSuffix(s)+ s = stripParens(s)+ return strings.TrimSpace(s)+}++// stripTimeZoneSuffix removes a trailing "with time zone" / "without time+// zone" clause. Idempotent on inputs that do not carry the suffix.+func stripTimeZoneSuffix(s string) string {+ lower := strings.ToLower(s)+ for _, suffix := range []string{" with time zone", " without time zone"} {+ if strings.HasSuffix(lower, suffix) {+ return s[:len(s)-len(suffix)]+ }+ }+ return s+}++// stripParens removes the first parenthesized chunk from a type string.+// "numeric(10,2)" -> "numeric"+// "timestamp(3)" -> "timestamp"+// "character varying(255)" -> "character varying"+func stripParens(s string) string {+ before, _, found := strings.Cut(s, "(")+ if !found {+ return s+ }+ return strings.TrimSpace(before)+}++// splitQualifiedName splits a dotted type name into (schema, name). Quoted+// identifiers are supported.+// "public.task_status" -> ("public", "task_status", true)+// "myschema.my_domain" -> ("myschema", "my_domain", true)+// `"MySchema"."MyType"` -> ("MySchema", "MyType", true)+// "task_status" -> ("", "", false)+func splitQualifiedName(s string) (schema, name string, ok bool) {+ parts, err := splitDottedIdentifier(s)+ if err != nil || len(parts) != 2 {+ return "", "", false+ }+ return parts[0], parts[1], true+}++// splitDottedIdentifier splits "a.b.c" into ["a", "b", "c"], handling+// double-quoted identifiers like `"weird name"."another"`.+func splitDottedIdentifier(s string) ([]string, error) {+ var parts []string+ var cur strings.Builder+ inQuote := false+ for i := 0; i < len(s); i++ {+ c := s[i]+ switch {+ case c == '"':+ inQuote = !inQuote+ case c == '.' && !inQuote:+ parts = append(parts, cur.String())+ cur.Reset()+ default:+ cur.WriteByte(c)+ }+ }+ if inQuote {+ return nil, errors.Errorf("unbalanced quote in %q", s)+ }+ parts = append(parts, cur.String())+ return parts, nil+}++// isBuiltinType reports whether a bare (modifier-stripped) type name is a+// PostgreSQL built-in type. Membership is exact, case-insensitive.+//+// The list is hand-maintained. New entries must come with a golden test in+// query_span_e3_test.go.+func isBuiltinType(s string) bool {+ _, ok := pgBuiltinTypes[strings.ToLower(s)]+ return ok+}++// pgBuiltinTypes is the allow-list of PostgreSQL built-in type base names+// (without type modifiers, time-zone suffixes, or array brackets).+//+// Sources: PostgreSQL data types reference + the transforms in+// backend/plugin/db/pg/sync.go:820-872 for the specific surface bytebase+// currently emits.+var pgBuiltinTypes = map[string]struct{}{+ // Numeric.+ "smallint": {},+ "integer": {},+ "int": {},+ "int2": {},+ "int4": {},+ "int8": {},+ "bigint": {},+ "decimal": {},+ "numeric": {},+ "real": {},+ "double precision": {},+ "float4": {},+ "float8": {},+ "smallserial": {},+ "serial": {},+ "bigserial": {},+ "money": {},++ // Character / binary.+ "character": {},+ "character varying": {},+ "char": {},+ "varchar": {},+ "text": {},+ "bytea": {},+ "bpchar": {},+ "name": {},++ // Bit.+ "bit": {},+ "bit varying": {},+ "varbit": {},++ // Date / time.+ "date": {},+ "time": {},+ "timetz": {},+ "timestamp": {},+ "timestamptz": {},+ "interval": {},++ // Boolean and friends.+ "boolean": {},+ "bool": {},++ // Structured.+ "json": {},+ "jsonb": {},+ "xml": {},+ "uuid": {},++ // Geometric.+ "point": {},+ "line": {},+ "lseg": {},+ "box": {},+ "path": {},+ "polygon": {},+ "circle": {},++ // Network.+ "cidr": {},+ "inet": {},+ "macaddr": {},+ "macaddr8": {},++ // Text search.+ "tsvector": {},+ "tsquery": {},++ // PG pseudo / system.+ "void": {},+ "record": {},+ "anyelement": {},+ "anyarray": {},+ "anynonarray": {},+ "anyenum": {},+ "anyrange": {},+ "any": {},+ "trigger": {},+ "event_trigger": {},+ "cstring": {},+ "internal": {},+ "language_handler": {},+ "fdw_handler": {},+ "index_am_handler": {},+ "tsm_handler": {},+ "pg_lsn": {},+ "oid": {},+ "regclass": {},+ "regproc": {},+ "regprocedure": {},+ "regoper": {},+ "regoperator": {},+ "regtype": {},+ "regconfig": {},+ "regdictionary": {},+ "regnamespace": {},+ "regrole": {},+}diff --git a/backend/plugin/parser/pg/query_span_omni.go b/backend/plugin/parser/pg/query_span_omni.goindex 2cae12e256..65a2cda3b5 100644--- a/backend/plugin/parser/pg/query_span_omni.go+++ b/backend/plugin/parser/pg/query_span_omni.go@@ -11,10 +11,8 @@ import ("github.com/bytebase/omni/pg/catalog"omniparser "github.com/bytebase/omni/pg/parser"- "github.com/bytebase/bytebase/backend/common/log"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"- "github.com/bytebase/bytebase/backend/plugin/schema""github.com/bytebase/bytebase/backend/store/model")@@ -47,6 +45,20 @@ type omniQuerySpanExtractor struct {// fallbackCTEAliases holds CTE column aliases (e.g., WITH t1(cc1, cc2) AS (...)).// Keyed by lowercase CTE name.fallbackCTEAliases map[string][]string++ // Catalog loader state — populated by initCatalog.+ // degraded records objects whose real install failed and fell back to+ // pseudo. Keyed by loader object key (schema.name with kind prefix).+ degraded map[string]error+ // trulyBroken records objects whose pseudo install also failed.+ trulyBroken map[string]error+ // loaderObjects records every object collected from metadata. Used by+ // tests to distinguish loader bugs from metadata gaps.+ loaderObjects map[string]bool+ // lastFallbackReason is the classifier's verdict for the most recent+ // AnalyzeSelectStmt error at any of the three fallback sites. Used by+ // tests; not consumed by production code.+ lastFallbackReason fallbackReason}// newOmniQuerySpanExtractor creates a new omni-based query span extractor.@@ -79,10 +91,12 @@ func (e *omniQuerySpanExtractor) getDatabaseMetadata(database string) (*model.Dareturn meta, nil}-// initCatalog initializes the omni catalog from the database metadata.-// It uses schemapg.GetDatabaseDefinition to produce complete DDL (including-// CREATE TYPE, CREATE SEQUENCE, dependency ordering, etc.), the same-// approach used by WalkThroughOmni.+// initCatalog initializes the omni catalog from the database metadata using+// the Catalog loader. Every metadata object is installed in dependency order,+// with an inline pseudo fallback at any failed slot (see query_span_e3_*.go).+//+// The DDL render + batch-parse path used before this refactor is deleted: BYT-9215 and+// BYT-9261 are unreachable by construction here.func (e *omniQuerySpanExtractor) initCatalog() error {meta, err := e.getDatabaseMetadata(e.defaultDatabase)if err != nil {@@ -96,33 +110,32 @@ func (e *omniQuerySpanExtractor) initCatalog() error {return nil}- ddl, err := schema.GetDatabaseDefinition(- storepb.Engine_POSTGRES,- schema.GetDefinitionContext{},- meta.GetProto(),- )- if err != nil {- return errors.Wrapf(err, "failed to generate schema DDL for catalog init")+ loader := newCatalogLoader(e.cat, meta.GetProto())+ if err := loader.Load(e.ctx); err != nil {+ return errors.Wrapf(err, "catalog loader failed for %q", e.defaultDatabase)}- if ddl != "" {- if _, err := e.cat.Exec(ddl, &catalog.ExecOptions{ContinueOnError: true}); err != nil {- slog.Error("failed to parse schema DDL for catalog init",- slog.String("database", e.defaultDatabase),- log.BBError(err),- slog.String("context", ddlErrorContext(ddl, err)),- )- return errors.Wrapf(err, "failed to load schema DDL into catalog")- }+ e.degraded = loader.degraded+ e.trulyBroken = loader.trulyBroken+ e.loaderObjects = loader.loaderObjects++ if len(loader.degraded) > 0 || len(loader.trulyBroken) > 0 {+ slog.Debug("catalog loader degraded objects",+ slog.String("database", e.defaultDatabase),+ slog.Int("degraded", len(loader.degraded)),+ slog.Int("truly_broken", len(loader.trulyBroken)),+ )}- // Store original (non-stubbed) function definitions for body analysis.+ // Populate original (non-stubbed) function definitions for PL/pgSQL body+ // analysis. This is independent of catalog loading and unchanged by the loader.for _, s := range meta.GetProto().GetSchemas() {for _, f := range s.GetFunctions() {- if f.Definition != "" {- origBody := extractFuncBodyFromDef(f.Definition)- if origBody != "" {- e.funcOrigDefs[strings.ToLower(f.Name)] = origBody- }+ if f.Definition == "" {+ continue+ }+ origBody := extractFuncBodyFromDef(f.Definition)+ if origBody != "" {+ e.funcOrigDefs[strings.ToLower(f.Name)] = origBody}}}@@ -130,29 +143,6 @@ func (e *omniQuerySpanExtractor) initCatalog() error {return nil}-// ddlErrorContext extracts a snippet around the error position from the DDL.-// If err is a *omniparser.ParseError with a position, it shows ~200 bytes-// around that position. Otherwise returns the first 500 bytes.-func ddlErrorContext(ddl string, err error) string {- var parseErr *omniparser.ParseError- if errors.As(err, &parseErr) && parseErr.Position > 0 {- pos := parseErr.Position- start := pos - 100- if start < 0 {- start = 0- }- end := pos + 100- if end > len(ddl) {- end = len(ddl)- }- return ddl[start:end]- }- if len(ddl) > 500 {- return ddl[:500] + "...(truncated)"- }- return ddl-}-// extractFuncBodyFromDef extracts the dollar-quoted function body from a// CREATE FUNCTION definition string.func extractFuncBodyFromDef(definition string) string {@@ -275,6 +265,10 @@ func (e *omniQuerySpanExtractor) getQuerySpan(ctx context.Context, stmt string)query, err := e.cat.AnalyzeSelectStmt(selStmt)if err != nil {+ // Record the classifier's verdict for test inspection. This does not+ // gate behavior — the fallback below always runs on analyzer errors.+ e.lastFallbackReason = classifyAnalyzeError(err)+// Before falling back, try to handle user-defined table-returning functions// that omni's AnalyzeSelectStmt can't resolve (e.g., RETURNS TABLE functions// used as table sources: SELECT * FROM func()).diff --git a/backend/plugin/parser/pg/query_span_omni_plpgsql.go b/backend/plugin/parser/pg/query_span_omni_plpgsql.goindex 4411081745..f203d4e357 100644--- a/backend/plugin/parser/pg/query_span_omni_plpgsql.go+++ b/backend/plugin/parser/pg/query_span_omni_plpgsql.go@@ -100,6 +100,10 @@ func (e *omniQuerySpanExtractor) analyzeSQLBody(body string) ([]base.SourceColum}query, err := e.cat.AnalyzeSelectStmt(selStmt)if err != nil {+ // Record the classifier verdict for test inspection; fallback+ // below always runs.+ e.lastFallbackReason = classifyAnalyzeError(err)+// Fall back to parse-tree extraction when analysis fails// (e.g., nested function calls, parameter references).fallbackResults := e.extractFallbackColumns(selStmt)@@ -424,6 +428,10 @@ func (a *plpgsqlAnalyzer) analyzeEmbeddedSQL(sql string) []base.SourceColumnSetquery, err := a.extractor.cat.AnalyzeSelectStmt(selStmt)if err != nil {+ // Record the classifier verdict for test inspection; fallback below+ // always runs.+ a.extractor.lastFallbackReason = classifyAnalyzeError(err)+// When full analysis fails (e.g., type mismatches with aggregate functions),// fall back to extracting column references from the parse tree and resolving// them against known tables in the catalog.diff --git a/docs/plans/2026-04-15-query-span-eager-pseudo-loader.md b/docs/plans/2026-04-15-query-span-eager-pseudo-loader.mdnew file mode 100644index 0000000000..6044a1feae--- /dev/null+++ b/docs/plans/2026-04-15-query-span-eager-pseudo-loader.md@@ -0,0 +1,519 @@+# Query Span Catalog Loader: Eager Per-Object Install with Inline Root-Pseudo Fallback++**Status:** Draft v2.2 (post codex round-2 + main-line re-alignment)+**Author:** junyi+**Date:** 2026-04-15+**Supersedes:** v5 (lazy per-query closure), v1/v2/v3 of this document+**PoC:** `backend/plugin/parser/pg/query_span_v5_poc_test.go` — 30 tests + 6 benchmarks, all passing+**Related issues:** BYT-9215, BYT-9261, BYT-9021, BYT-9117 (mysql reuse)++---++## One-sentence summary++Install every schema object eagerly at catalog init in topological order, using hand-constructed AST nodes fed directly to omni's `DefineX` API. When a real install fails, immediately install a text-backed pseudo version at the same slot so downstream objects install as real. The DDL render path is deleted.++## Problem++`query_span_omni.go:86` calls `schema.GetDatabaseDefinition(POSTGRES, meta)` to render the full schema as a single DDL blob, then `cat.Exec(ddl, ContinueOnError)`. omni's `catalog.Exec` at `omni/pg/catalog/exec.go:33` calls `pgparser.Parse(sql)` on the whole blob first — batch parse. If any one statement has a construct pgparser rejects (BYT-9215: `::` in quoted identifier, BYT-9261: `->>` in index expression), the entire parse fails and `Exec` returns a top-level error before any `ContinueOnError` branch is reached.++Effect: one bad object in one schema makes query span fail for **every** query on that database. Masking silently degrades to `extractFallbackColumns`, which produces empty-source results that bypass most masking rules.++## Why closure is hard (and why the loader sidesteps it)++A previous iteration (v5) tried to solve this lazily: parse the query, extract its object references, install only those, avoid the full schema cost. The investigation showed accurate closure is not computable:++- Function and operator overloads require knowing argument types, which require loading the closure (chicken and egg).+- Composite `(col).field` access, domain base types, inheritance children, partition children — none visible from query syntax alone.+- Dynamic SQL in PL/pgSQL function bodies — fundamentally unknowable.++v5's escape hatch was "install all candidates by name, not signature" — a conservative superset. But this layered on metadata-assisted closure expansion, retry loops, and several other under-specified mechanisms.++the loader abandons the closure problem entirely: **install everything**. The analyzer reads what it needs; the rest is idle catalog weight. Install cost is O(schema), but per-object hand-built install is fast enough (measured 1.5-2.3× faster than current DDL path) that this is not a regression.++## Core insight: cascade through dependency order, pseudo at the root++Consider enum `E` with a broken definition, table `T` with column of type `E`, view `V` over `T`.++**Naive skip-on-failure** (what v5 fail-fast would do):+- `E` install fails → skip+- `T` install fails (can't resolve `E`) → skip+- `V` install fails (can't resolve `T`) → skip+- Any query touching anything in the chain fails.++**the loader inline pseudo**:+- `E` real install fails → immediately install `pseudo(E)` (text-backed enum) at the same slot → `degraded[E] = err`+- `T` real install: references `pseudo(E)` in the catalog, succeeds as **real** object with correct column names+- `V` real install: references real `T`, succeeds as **real** object+- Query on `V` runs with full lineage to base-table columns; the only degradation is that `E`-typed columns behave like `text` for type resolution.++**For cascade prevention to work we need two things**:+1. Topological order — process `E` before `T` before `V`. Otherwise `T`'s failure is indistinguishable from `T` itself being broken.+2. Inline pseudo — fail real, immediately install pseudo at the same slot, in the same loop. No retry, no second pass.++Empirical validation: `TestLoaderPoC_RealCascadeWithGenuineFailure` forces a genuine `DefineDomain` SQLSTATE 42704 failure, installs a text-backed pseudo at the same name, then installs downstream table and view as real `DefineRelation` / `DefineView` calls, queries the view, and asserts every target is `*catalog.VarExpr` (full lineage to base-table columns). Not a simulation.++## Proposal++### Phase 1 — eager loader++```go+func (l *catalogLoader) Load(ctx context.Context, meta *storepb.DatabaseSchemaMetadata) error {+ objects := l.collectObjects(meta) // flatten metadata into []ObjectEntry+ sccs := l.tarjanSCC(objects) // condense cycles+ sorted := l.topoSort(sccs) // SCC-level topo sort+ for _, sccGroup := range sorted {+ for _, obj := range lexSort(sccGroup) { // intra-SCC: deterministic lex order+ if err := l.installReal(obj); err != nil {+ l.degraded[obj.Key()] = err+ if pErr := l.installPseudo(obj); pErr != nil {+ l.trulyBroken[obj.Key()] = pErr+ }+ }+ }+ }+ return nil+}+```++Single forward sweep. When `Load` returns, the catalog contains every metadata object as either real (degraded may still be false if nothing failed downstream) or pseudo. `trulyBroken` is for objects whose even-pseudo install failed — vanishingly rare, captured for observability.++### Phase 2 — analyze++```go+func (e *omniQuerySpanExtractor) ExtractQuerySpan(ctx context.Context) (*base.QuerySpan, error) {+ if err := e.initCatalog(ctx); err != nil { return nil, err }+ stmt := parseQuery(...)+ analyzed, err := e.cat.AnalyzeSelectStmt(stmt)+ if err != nil {+ return e.extractFallbackSpan(stmt, err) // classified, see §Fallback classifier+ }+ return e.extractLineage(analyzed, stmt, e.degraded)+}+```++`extractLineage` consults the `degraded` set to mark lineage entries whose source is a pseudo object.++### Phase 3 — what the loader does not do++No closure discovery. No `ExtractCatalogReferences`. No install-retry loop. No lazy install. These were v5 concepts. the loader deletes them.++## Fallback classifier (pure function, used by tests)++The classifier is kept because tests need to assert which reason a given query hit. It is **not** wired into runtime telemetry, metrics, or a flip gate. It is a pure function returning an enum.++```go+type fallbackReason int++const (+ reasonNone fallbackReason = iota+ reasonExpectedPseudoSemantic // pseudo-induced operator / type mismatch+ reasonUndefinedReference // analyzer says object not found+ reasonAnalyzerUnsupported // non-*catalog.Error internal analyzer failure+)++func classifyAnalyzeError(err error) fallbackReason {+ var cErr *catalog.Error+ if !errors.As(err, &cErr) {+ return reasonAnalyzerUnsupported+ }+ switch cErr.Code {+ case catalog.CodeUndefinedFunction, catalog.CodeAmbiguousFunction,+ catalog.CodeDatatypeMismatch, catalog.CodeFeatureNotSupported,+ catalog.CodeAmbiguousColumn:+ return reasonExpectedPseudoSemantic+ case catalog.CodeUndefinedTable, catalog.CodeUndefinedColumn,+ catalog.CodeUndefinedObject, catalog.CodeUndefinedSchema:+ return reasonUndefinedReference+ default:+ return reasonAnalyzerUnsupported+ }+}+```++Three buckets are enough for test assertions. We dropped the `reasonLoaderBugSuspect` / `reasonTrulyBroken` / `reasonUserOrSyncLag` distinction from v3 because it was introduced to feed a runtime flip gate we no longer have. Tests that want to distinguish "loader bug" from "user query referenced non-existent object" do so by constructing the fixture they know the answer to.++The classifier lives in `query_span_e3_classify.go` and is called from all three fallback sites (see §Fallback call-site audit) to give tests a consistent assertion surface. The extractor stores the last classified reason as `e.lastFallbackReason` for tests to check.++## Fallback call-site audit++Three sites today fall through to `extractFallbackColumns` / `fallbackExtractLineage` on any analyzer error, unconditionally:++1. `query_span_omni.go:276-295` — top-level `AnalyzeSelectStmt` in `ExtractQuerySpan`.+2. `query_span_omni_plpgsql.go:85-115` — `analyzeSQLBody` (SQL function body inlining).+3. `query_span_omni_plpgsql.go:405-450` — `analyzeEmbeddedSQL` (embedded SQL in PL/pgSQL blocks).++All three adopt `classifyAnalyzeError` as a pre-fallback step. The fallback itself still runs; the reason is recorded for test inspection. PL/pgSQL is not "zero changes" — it is a shared touchpoint, ~30 LOC per site.++`tryUserFuncTableSource` (`query_span_omni.go:1350+`) remains as a pre-fallback path for RETURNS TABLE functions.++## Dependency graph from metadata++Topological order derives from `storepb.DatabaseSchemaMetadata`. Dependency edges:++| Object | Depends on |+|---|---|+| Schema | none |+| Enum | Schema |+| Domain | Schema |+| Composite type | Schema |+| Range type | Schema |+| Table | Schema; each column's user-defined type reference (extracted from `columns[i].type` string) |+| View | Schema; every entry in `dependency_columns.{schema,table}` |+| MatView | Same as view |+| Function | Schema; each parameter type *if* user-defined; return type *if* user-defined |++**Unavailable metadata** doesn't block the loader:++- **Inheritance**: `sync.go:777` uses `INFORMATION_SCHEMA.COLUMNS`, which returns parent-merged columns for child tables. The child's `columns[]` is complete. the loader installs each child as a standalone table.+- **Partition**: same reasoning. Query span doesn't care about partition bounds.+- **Composite / Domain / Range internal type refs**: `storepb` doesn't carry these. the loader treats them as leaves in the dep graph. If real install fails because of an unresolved internal reference, pseudo catches it.++### Cycle handling — Tarjan SCC + lex ordering++Metadata-level cycles are rare (mutually recursive views via `CREATE OR REPLACE`). the loader:++1. Tarjan SCC on the dep graph.+2. Topo sort the condensed DAG of SCCs.+3. For each SCC of size > 1:+ - Sort members by `(schema, name)` lexicographically.+ - Process in order. The first member's real install fails (its body references the SCC's other members) → pseudo fallback.+ - Subsequent members reference the first (now pseudo), so their real installs succeed.++For an SCC of size N, exactly one member ends up pseudo; the other N-1 are real with degraded lineage back to the cycle root. Strictly better than "all SCC members become pseudo."++Lex ordering guarantees determinism for test reproducibility.++## Type string grammar++`column.Type` in `storepb.ColumnMetadata` is produced by `sync.go:820-872`. Full surface:++| # | Form | Example | Extractor output |+|---|---|---|---|+| 1 | Built-in scalar | `integer`, `text`, `boolean`, `json`, `jsonb`, `uuid`, `date`, `interval` | `[]` |+| 2 | Built-in with length | `character(10)`, `character varying(255)`, `bit(8)`, `bit varying(8)` | `[]` |+| 3 | Built-in numeric | `numeric(10,2)`, `decimal(8)`, `numeric(5)` | `[]` |+| 4 | Built-in time | `timestamp(3) with time zone`, `time(6) without time zone` | `[]` |+| 5 | USER-DEFINED | `public.task_status`, `myschema.my_domain` | `[{public, task_status}]` |+| 6 | ARRAY of built-in | `_text`, `_int4` (raw `udt_name`, PG internal form) | `[]` |+| 7 | ARRAY of user-defined | `_task_status` (schema stripped by sync.go:834) | `[]` (cannot topo-sort; installs fail → pseudo) |+| 8 | System-schema-qualified | `pg_catalog.int4`, `information_schema.cardinal_number` | `[]` |++Form 7 is an existing sync issue. the loader is resilient to it (pseudo catches). the loader does not fix it.++### Extractor++```go+func extractUserTypeRefs(typeStr string) []UserTypeRef {+ if typeStr == "" { return nil }+ base := stripParens(typeStr) // "numeric(10,2)" → "numeric"+ base = stripTimeZoneSuffix(base) // "timestamp(3) with time zone" → "timestamp"+ if isBuiltin(base) { return nil } // allow-list+ if strings.HasPrefix(base, "_") { return nil }// PG internal array form+ if schema, name, ok := splitQualified(base); ok {+ if isSystemSchema(schema) { return nil }+ return []UserTypeRef{{Schema: schema, Name: name}}+ }+ return nil+}+```++### Soundness rule++False negatives (missing a user type ref) are acceptable — pseudo catches. False positives (marking a built-in as user type) are not — they corrupt the topo order.++### Golden corpus (used in tests)++```+"integer" → []+"text" → []+"bigint" → []+"numeric(10,2)" → []+"character varying(255)" → []+"timestamp(3) with time zone" → []+"json" / "jsonb" / "uuid" / "interval" → []+"public.task_status" → [{public, task_status}]+"myschema.my_domain" → [{myschema, my_domain}]+"_text" → []+"_task_status" → []+"pg_catalog.int4" → []+"information_schema.cardinal_number" → []+"" → []+```++Built-in allow-list: ~50 entries, hardcoded from PG standard types. Additions require a golden test.++## Pseudo install forms (PoC-verified)++Every pseudo form is covered by a green test in `query_span_v5_poc_test.go`.++| Object | Pseudo AST | PoC test |+|---|---|---|+| Enum | `CreateEnumStmt{Vals: empty}` | `TestLoaderPoC_PseudoEnum_EmptyVals` |+| Domain | `CreateDomainStmt{Typname: text}` | `TestLoaderPoC_PseudoDomain_OverText` |+| Composite | `CompositeTypeStmt{Coldeflist: [_broken text]}` | `TestLoaderPoC_PseudoComposite_FieldsAllText` |+| Range | `CreateRangeStmt{subtype: text}` | `TestLoaderPoC_PseudoRange_SubtypeText` |+| Table | `CreateStmt{cols: [(name, text) for name in metadata]}` | Implicit; pseudo is `DefineRelation` with all-text columns |+| View | `ViewStmt{Query: SELECT NULL::text AS c1, NULL::text AS c2 ...}` | `TestLoaderPoC_PseudoView_ConstantTargetList` |+| MatView | `CreateTableAsStmt{Query: SELECT NULL::text AS ...}` | Variant of pseudo view |+| Function | `CreateFunctionStmt{params: [text], returns: text, body: SELECT $1}` | `TestLoaderPoC_PseudoFunction_AllText`; selection asserted by `TestLoaderPoC_OverloadSelectionAsserted` |++**Cascade prevention end-to-end**: `TestLoaderPoC_RealCascadeWithGenuineFailure`. Real `DefineDomain` 42704 failure → pseudo at same slot → downstream table + view install as real objects → query on view has full lineage to base-table columns.++**Composite field access limitation**: `storepb` doesn't carry composite field names, so pseudo composite uses a single `_broken` field. Queries using `(col).field` fall back to `extractFallbackColumns`. This is no worse than the current path (which also fails BYT-9215-class queries). Medium-term follow-up: add `CompositeTypeMetadata` to proto + sync.++## `storepb` gaps and how the loader handles them++| Gap | the loader handling | Fidelity impact |+|---|---|---|+| `DomainMetadata` missing | Pseudo domain over text | Domain constraints not visible (query span doesn't use them) |+| `RangeTypeMetadata` missing | Pseudo range subtype=text | Range-specific operators degrade to fallback |+| `CompositeTypeMetadata` missing | Pseudo composite with single field | `(col).field` falls back; top-level `SELECT col` works |+| `TableMetadata.inherits_parent` missing | Child's columns already parent-merged by sync | Zero impact |++Only composite field access is a real fidelity loss, and it is no worse than today's `initCatalog` failure on BYT-9215/9261.++## `initCatalog` shrinks++Before (`query_span_omni.go` around line 86):+```go+func (e *omniQuerySpanExtractor) initCatalog(ctx context.Context) error {+ ddl := schema.GetDatabaseDefinition(POSTGRES, e.meta)+ _, err := e.cat.Exec(ddl, &catalog.ExecOptions{ContinueOnError: true})+ // fallback handling when err is non-nil+}+```++After:+```go+func (e *omniQuerySpanExtractor) initCatalog(ctx context.Context) error {+ e.cat = catalog.New()+ e.cat.SetSearchPath(e.searchPath)+ loader := newCatalogLoader(e.cat, e.meta)+ if err := loader.Load(ctx); err != nil {+ return err // only catastrophic errors, e.g. ctx cancellation+ }+ e.degraded = loader.degraded+ e.trulyBroken = loader.trulyBroken+ return nil+}+```++The DDL render path is deleted outright. BYT-9215 and BYT-9261 cannot recur by construction: no DDL text is parsed during install. The only SQL parser invocations in Phase 1 are for view/matview bodies (SELECT statements on a stable code path).++## Search path (out of scope)++Two pre-existing search-path issues were scoped into v2/v3 drafts via codex review iterations, but they are orthogonal to BYT-9215 / BYT-9261 and not caused or worsened by the loader. They are moved to §Follow-ups:++- `ExtractAccessTablesOption.DefaultSchema` is a single string, forcing callers to pass `e.searchPath[0]` and drop the rest of the search path.+- Fallback path `query_span_omni.go:963` resolves unqualified relations with `searchPath[0]` only.++Neither is a regression the loader introduces, neither blocks BYT-9215 / BYT-9261 remediation, and fixing either would pull unrelated call sites (including `resource_change.go`) into the the loader PR. the loader preserves current search-path behavior unchanged.++`$user` expansion is also unchanged: `query_span.go:26` calls `meta.GetSearchPath()` (user-less variant at `backend/store/model/database.go:222`). Plumbing `currentUser` through `GetQuerySpanContext` is a separate product decision.++## File layout++New files under `backend/plugin/parser/pg/`:++```+query_span_e3_type_name.go — typeNameFromString (cheat parser) + extractUserTypeRefs + allow-list+query_span_e3_builders.go — buildCreateStmt, buildViewStmt, buildCreateEnumStmt, ...+query_span_e3_pseudo.go — pseudoCreateStmt, pseudoViewStmt, pseudoEnum, ...+query_span_e3_loader.go — catalogLoader{} + Load() + collectObjects + Tarjan SCC + topoSort+query_span_e3_classify.go — classifyAnalyzeError + fallbackReason+query_span_e3_test.go — loader, topo, cycle, pseudo, classifier, type-grammar unit tests+query_span_e3_integration_test.go — BYT repros + cascade + search path + PL/pgSQL integration tests+```++Modified:+```+query_span_omni.go — initCatalog body replaced; fallback at :276 uses classifier+query_span_omni_plpgsql.go — analyzeSQLBody and analyzeEmbeddedSQL fallbacks use classifier+```++Kept unchanged:+- `query_span_v5_poc_test.go` — 30 tests + 6 benchmarks, locked in as regression baseline+- `extractLineage` core — operates on the same analyzer output+- `base.QuerySpan` contract (additive only: new `Degraded` bit on `SourceColumn`)++## The `Degraded` field on result columns++`base.QuerySpan.Results[i].SourceColumns[j]` gains a boolean `Degraded` bit:++- `Degraded=false`: fully resolved to a real table column.+- `Degraded=true`: source is a pseudo object; column name is correct but type-level fidelity reduced.++Masking policy: conservative — `Degraded=true` sources are still subject to masking (treated as "real enough"). This is a strict improvement over today, where catalog failures produce empty sources that silently bypass masking.++## Key decisions++**D1. Eager per-object install, not lazy closure.** Sound by construction; closure discovery is heuristic. Measured 1.5-2.3× faster than current DDL path on synthetic schemas.++**D2. Topological order via Tarjan SCC + intra-SCC lex ordering.** Makes skip-on-failure composable with downstream resolution. Cycles handled by pseudo-ing one member (lex-first), rest install real.++**D3. Inline pseudo at the failed slot, not downstream or at query time.** Preserves catalog state invariant: after processing obj X's slot, X is in the catalog (real or pseudo).++**D4. Pseudo types are text-based.** text has the most permissive operator resolution and implicit-cast surface in PG. PoC validates every tested query pattern.++**D5. No catalog cache.** Current path is per-query O(schema); the loader is the same shape but faster. Caching is a post-the loader optimization if profiling demands it.++**D6. Ignore inheritance and partition parent metadata.** `sync.go:777` flattens.++**D7. Composite field access is a known degradation.** `storepb` lacks composite field names today. Medium-term follow-up: add `CompositeTypeMetadata`.++**D8. No proto changes in the the loader PR.** Ship the blast-radius fix independent of metadata enrichment.++**D9. No omni PR required.** v2/v3 briefly considered a prerequisite omni `ErrorIdent` PR. Dropped: the value it unlocked (runtime flip gate precision) is not on the main line now that shipping is by test-driven PR review, not automated gate.++**D10. MySQL reuse (BYT-9117).** Same loader + pseudo pattern, different `build*Stmt` functions. Scheduled as follow-up.++**D11. Classifier is a pure function for test assertions.** Not a runtime gate, not a telemetry sink, not wired into counters. Three reason buckets are enough for tests to distinguish expected degradation from genuine bugs.++## Tests++**The test matrix replaces the shadow-diff harness.** No runtime comparison of two code paths. Instead, a comprehensive set of unit + integration tests exercises every the loader path and every known BYT-issue class. Any regression surfaces in CI, not in production.++### Retention++- All 30 existing PoC tests + 6 benchmarks in `query_span_v5_poc_test.go`. Regression baseline.+- Existing `query_span_test.go` cases run unchanged; they are now covered by the the loader path.+- Existing testcontainer tests (`query_span_ddl_debug_testcontainer_test.go`, `query_span_typecast_testcontainer_test.go`) run unchanged.++### New unit tests (`query_span_e3_test.go`)++- **Loader**:+ - `collectObjects` extracts the expected `ObjectEntry` set from fixture `DatabaseSchemaMetadata`.+ - `tarjanSCC` finds cycles and returns SCC groups in deterministic order.+ - `topoSort` of SCCs produces a valid topological order.+ - `installReal` + `installPseudo` interaction on a fixture with one broken object.+- **Classifier**: each SQLSTATE case in `classifyAnalyzeError` returns the expected reason. Non-`*catalog.Error` → `reasonAnalyzerUnsupported`.+- **Type grammar**: every row in §Type string grammar golden corpus is asserted against `extractUserTypeRefs`.+- **Pseudo builders**: each `pseudoXStmt` produces an AST that installs via its corresponding `DefineX` call without error. Structural assertions on resulting catalog state.++### New integration tests (`query_span_e3_integration_test.go`)++- **BYT-9215 repro**: construct `DatabaseSchemaMetadata` with a table name `"'lib"."address"`. Assert the loader loader installs successfully (real or pseudo), a query on an unrelated table succeeds with full lineage, and a query on the broken object degrades gracefully.+- **BYT-9261 repro**: table with a `->>` index expression in metadata. Same assertion pattern.+- **Broken enum cascade**: `E` with invalid metadata, `T(col E)`, `V` over `T`. After load, `E` is pseudo, `T` and `V` are real. Query on `V` has `Degraded=true` on the `E`-typed column only.+- **Cyclic view definitions**: two mutually recursive views. After load, one is pseudo (the lex-first), the other is real. Query on either returns lineage; the one referencing the pseudo has `Degraded=true`.+- **Pseudo function selection**: table with int column + text column; pseudo function `fn(text)` coexists with real `fn(int4)`. Assert query `fn(int_col)` picks the int overload, `fn(text_col)` picks the pseudo.+- **Search path — full path resolution**: table `foo` only in second schema of a two-element search path. Assert primary path finds it.+- **Search path — fallback path consistency**: same table under a deliberate analyzer failure. Assert fallback also finds it (`query_span_omni.go:963` walks full path).+- **PL/pgSQL body with broken dependency**: function body references a pseudo-composite. Assert `analyzeSQLBody` routes to `reasonExpectedPseudoSemantic` and returns fallback lineage.+- **PL/pgSQL embedded SQL with type mismatch**: same pattern for `analyzeEmbeddedSQL`.++### Benchmarks++- `BenchmarkLoader_Install_*` remain in place (kernel benchmarks).+- Optional additions once full loader exists: benchmark the full `Load()` (including topo, classify, pseudo, PL/pgSQL) on a fixture schema.++## Rollout++Simple:++1. Write plan v2.2 (this document). Done.+2. Implement loader PR(s) per §File layout. Single PR or split into "builders + pseudo + classifier" + "loader + integration" — reviewer's call.+3. All tests in §Tests green.+4. PR review.+5. Merge.+6. Monitor user reports. Fix forward if regressions surface.++No feature flag. No shadow mode. No flip gate. `git revert` is the rollback mechanism if needed. Standard bytebase shipping practice.++### Follow-ups (independent PRs)++- `CompositeTypeMetadata` in proto + sync → recovers `(col).field` lineage fidelity.+- `DomainMetadata` / `RangeTypeMetadata` → when data shows they matter.+- `$user` search-path plumbing → product decision.+- BYT-9117 mysql reuse.+- Four latent `base.QuerySpan` fields (`PredicateColumns`, `PredicatePaths`, `NotFoundError`, `FunctionNotSupportedError`) → tracked independently; orthogonal to load path.++## Risks++| Risk | Severity | Mitigation |+|---|---|---|+| Pseudo composite loses `(col).field` lineage | Medium | Falls back to `extractFallbackColumns`. No worse than current path. |+| `extractUserTypeRefs` false positives corrupt topo | Medium | Golden tests on full grammar (§Type string grammar). Hard contract C5. |+| `extractUserTypeRefs` false negatives miss user types | Low | Install fails → pseudo catches. |+| Function signature parsing (overloads by signature string) misparses | Medium | Golden tests on common forms. |+| Tarjan SCC non-determinism | Low | Intra-SCC lex ordering. Unit test asserts order. |+| Install cost on huge schemas (>10k objects) | Low | Measured 2000 tables at ~4 ms. Extrapolates to ~20 ms at 10k. |+| `transformFuncCall` / `transformAExpr` drop schema qualification | Medium | the loader installs all function/operator candidates globally; drop is masked at query time. File omni tracking issue separately. |+| PL/pgSQL analyzer picks up degraded state unexpectedly | Low | Same `cat`; integration test covers function body using a degraded table. |+| Cycle semantics untested against real mutually recursive views | Medium | Integration fixture with two mutually recursive views asserts one pseudo, one real. |+| Inherited columns not actually complete in metadata | Low | Add one integration test with an inheritance chain. |+| the loader introduces a regression that reaches main | Medium | Test matrix is the safety net (unit + integration + BYT repros + testcontainer). Fix forward if it slips. |++## Alternatives considered++**A. v5 lazy closure + DefineX.** Requires `ExtractCatalogReferences` + closure expansion + retry loop. Four layers gone in the loader. Superseded.++**B. ANTLR-approach manual walker.** ~3000 LOC of hand-written walker replacing `AnalyzeSelectStmt`. Overkill: the broken path is `initCatalog`, not the analyzer. Discarded.++**C. Alt D stopgap (per-statement Exec).** Valid as immediate stopgap if the loader will take >1 week, but with the loader starting immediately it adds a PR and a commit target that the loader will obsolete. Skipped.++**D. Catalog cache.** Optimization on top of the loader. Deferred until profiling demands it.++**E. Pseudo the cascade victims instead of the root.** Larger degraded set. Rejected in favor of root-pseudo (D3).++**F. All SCC members as pseudo.** Cycle size N → N members pseudo. Rejected in favor of one-pseudo-rest-real (D2).++**G. Runtime feature flag + shadow diff harness + automated flip gate.** v1/v2/v3 iteration path. Turned out to be over-engineered for bytebase's shipping practice. Replaced by test matrix + standard PR review + fix-forward.++**H. omni prerequisite PR adding structured `ErrorIdent`.** Useful only for automated flip gates. Dropped with the flip gate. Can be proposed independently later if classifier accuracy for operational diagnosis warrants it.++## Hard contracts++**C1. View/matview body must be `*ast.SelectStmt`, not `*ast.RawStmt`.** omni's `DefineView` / `ExecCreateTableAs` type-assert directly.++**C2. Partitioned tables pass relkind `'r'` to `DefineRelation`.** omni flips to `'p'` internally when `Partspec` is set.++**C3. Function overloads install sequentially.** Each overload is one `CreateFunctionStmt`. New signatures append; duplicate signatures are rejected.++**C4. `typeNameFromString` input must be from trusted metadata.**++**C5. `extractUserTypeRefs` must be sound, not precise.** False negatives acceptable (pseudo catches). False positives are not (corrupt dep graph).++**C6. Pseudo install must never depend on user objects.** Every pseudo form is built from built-in types only.++**C7. Loader must hold no long-lived state.** Fresh `catalog.New()` per extractor.++**C8. All 17 type string shapes in §Type string grammar have golden tests.** Including the PG internal array form `_name` and system-schema qualified forms.++**C9. Classifier is shared.** `classifyAnalyzeError` is one function called from three fallback sites. Never inline-classify at a call site.++**C10. Tarjan SCC intra-SCC ordering is lexicographic on `(schema, name)`.** Determinism required for test reproducibility.++## Complexity honesty++| Dimension | v5 | **the loader** |+|---|---|---|+| Per-query install cost | O(closure) | O(schema), measured 1.5-2.3× faster than current |+| Loader code | `ExtractCatalogReferences` + `expandClosure` + `installClosure` + retry | `collectObjects` + `tarjanSCC` + `topoSort` + `installReal`/`installPseudo` |+| Implementation LOC estimate | ~1000 | ~1000 (loader + classifier + search path unification + type-string extractor) |+| Fidelity regression risk | Medium (missed refs) | Low (only composite `(col).field`) |+| PL/pgSQL coupling | Fallback not narrowed | Classifier at 3 sites, observability only |+| Metadata proto changes | None | None |+| omni PR required | None | None |+| Runtime machinery | Fallback flag, shadow mode | None |+| Rollout | Multi-week soak | Merge when tests green |++## Appendix: metadata audit++Performed 2026-04-15 via exploration of `proto/store/store/database.proto` and `backend/plugin/db/pg/sync.go`.++| Field | Exists? | Location | Notes |+|---|---|---|---|+| `TableMetadata.columns[i].type` | ✅ | `database.proto:510` | String; user types as `"schema.name"` per `sync.go:1019` |+| `ViewMetadata.dependency_columns` | ✅ | `database.proto:587` | `{schema, table, column}` per view col |+| `MaterializedViewMetadata.dependency_columns` | ✅ | `database.proto:625` | Same shape |+| `EnumTypeMetadata` | ✅ | `database.proto:124-134` | `{name, values[]}` |+| `FunctionMetadata` | ✅ | `database.proto:645-672` | Per signature |+| `TableMetadata.partitions` | ✅ (parent→children) | `database.proto:360-361` | Reverse direction from topo needs; irrelevant |+| `TableMetadata.inherits_parent` | ❌ | — | Not needed: sync flattens |+| `CompositeTypeMetadata` | ❌ | — | Pseudo composite with `_broken` field |+| `DomainMetadata` | ❌ | — | Pseudo domain over text |+| `RangeTypeMetadata` | ❌ | — | Pseudo range subtype=text |++Of the four missing metadata kinds, only `CompositeTypeMetadata` causes query span fidelity loss (`(col).field` access), and it is fully recoverable via an independent metadata PR.diff --git a/go.mod b/go.modindex 2f877f975f..769a7c533d 100644--- a/go.mod+++ b/go.mod@@ -34,7 +34,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0- github.com/bytebase/omni v0.0.0-20260415065217-37977c187be4+ github.com/bytebase/omni v0.0.0-20260415084539-4a524e137c4agithub.com/caarlos0/env/v11 v11.4.0github.com/cenkalti/backoff/v5 v5.0.3github.com/cockroachdb/cockroachdb-parser v0.25.2diff --git a/go.sum b/go.sumindex 713a2926e6..1b28ae71f4 100644--- a/go.sum+++ b/go.sum@@ -251,8 +251,8 @@ github.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888 h1:E8tz8maxpih/vtgithub.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888/go.mod h1:IgLUOjyiHehdE0G3C92IjGYu9lJQgkPwf2mdNSvPWq8=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/omni v0.0.0-20260415065217-37977c187be4 h1:SiFNAGNjzUEL8EkJz7jP3doRqGLwg5Xvyp0Lq2QcN7c=-github.com/bytebase/omni v0.0.0-20260415065217-37977c187be4/go.mod h1:AT+iLKCu7NyxTaYaOLzETRFBVurjRaqUU3YmFron9pw=+github.com/bytebase/omni v0.0.0-20260415084539-4a524e137c4a h1:/13ZmyR9mpq1PjptpJmasQlJ3S5h6drkeL5yZFH5Zr8=+github.com/bytebase/omni v0.0.0-20260415084539-4a524e137c4a/go.mod h1:AT+iLKCu7NyxTaYaOLzETRFBVurjRaqUU3YmFron9pw=github.com/bytebase/parser v0.0.0-20260324092042-1d52e5412f50 h1:Kq5wlhZ586Rch39y/IQV9MnrEzHob4iiXBG6zrbvSMU=github.com/bytebase/parser v0.0.0-20260324092042-1d52e5412f50/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=