The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
feat: crazy amount of work to get around parquet not supporting uuid with clickhouse
dduxad0320161104
diff --git a/backend/app/chdb/embedded.go b/backend/app/chdb/embedded.goindex f0277fc2..bf53174b 100644--- a/backend/app/chdb/embedded.go+++ b/backend/app/chdb/embedded.go@@ -3,26 +3,44 @@package chdbimport (- "github.com/tracewayapp/traceway/backend/app/config""context""database/sql"+ "encoding/json""fmt""log"+ "strconv""strings""time"+ "github.com/tracewayapp/traceway/backend/app/config"+"github.com/ClickHouse/clickhouse-go/v2/lib/column""github.com/ClickHouse/clickhouse-go/v2/lib/driver"+ chdblib "github.com/chdb-io/chdb-go/chdb"_ "github.com/chdb-io/chdb-go/chdb/driver""github.com/google/uuid")type embeddedConn struct {- db *sql.DB+ session *chdblib.Session}func initEmbedded() error {path := config.Config.ClickhousePath++ // Create session first (global singleton in chdb-go)+ var session *chdblib.Session+ var err error+ if path != "" {+ session, err = chdblib.NewSession(path)+ } else {+ session, err = chdblib.NewSession()+ }+ if err != nil {+ return fmt.Errorf("failed to create chdb session: %w", err)+ }++ // Also open sql.DB for migrations (reuses the same session singleton)dsn := ""if path != "" {dsn = "session=" + path@@ -32,60 +50,142 @@ func initEmbedded() error {return fmt.Errorf("failed to open embedded clickhouse: %w", err)}EmbeddedDB = db- Conn = &embeddedConn{db: db}++ Conn = &embeddedConn{session: session}log.Println("Initialized embedded ClickHouse (chdb)")return nil}+// Query uses the native session API with JSONCompactStrings format,+// bypassing the Parquet driver that cannot serialize UUID columns.func (c *embeddedConn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) {- rows, err := c.db.QueryContext(ctx, query, sanitizeArgs(args)...)+ interpolated := interpolateQuery(query, args)+ result, err := c.session.Query(interpolated, "JSONCompactStrings")if err != nil {return nil, err}- return &embeddedRows{rows: rows}, nil++ buf := result.Buf()+ if len(buf) == 0 {+ result.Free()+ return &embeddedRows{}, nil+ }++ var parsed jsonResult+ if err := json.Unmarshal(buf, &parsed); err != nil {+ result.Free()+ return nil, fmt.Errorf("failed to parse query result: %w", err)+ }+ result.Free()++ cols := make([]string, len(parsed.Meta))+ for i, m := range parsed.Meta {+ cols[i] = m.Name+ }++ return &embeddedRows{columns: cols, data: parsed.Data}, nil}func (c *embeddedConn) QueryRow(ctx context.Context, query string, args ...any) driver.Row {- row := c.db.QueryRowContext(ctx, query, sanitizeArgs(args)...)- return &embeddedRow{row: row}+ interpolated := interpolateQuery(query, args)+ result, err := c.session.Query(interpolated, "JSONCompactStrings")+ if err != nil {+ return &embeddedRow{err: err}+ }++ buf := result.Buf()+ if len(buf) == 0 {+ result.Free()+ return &embeddedRow{err: sql.ErrNoRows}+ }++ var parsed jsonResult+ if err := json.Unmarshal(buf, &parsed); err != nil {+ result.Free()+ return &embeddedRow{err: fmt.Errorf("failed to parse query result: %w", err)}+ }+ result.Free()++ if len(parsed.Data) == 0 {+ return &embeddedRow{err: sql.ErrNoRows}+ }++ return &embeddedRow{data: parsed.Data[0]}}func (c *embeddedConn) PrepareBatch(ctx context.Context, query string, opts ...driver.PrepareBatchOption) (driver.Batch, error) {- return &embeddedBatch{db: c.db, query: query}, nil+ return &embeddedBatch{session: c.session, query: query}, nil}func (c *embeddedConn) Exec(ctx context.Context, query string, args ...any) error {- _, err := c.db.ExecContext(ctx, query, sanitizeArgs(args)...)- return err+ interpolated := interpolateQuery(query, args)+ result, err := c.session.Query(interpolated)+ if err != nil {+ return err+ }+ if result != nil {+ result.Free()+ }+ return nil}-func sanitizeArgs(args []any) []any {- out := make([]any, len(args))- for i, arg := range args {- switch v := arg.(type) {- case time.Time:- out[i] = v.UTC().Format("2006-01-02 15:04:05")- default:- out[i] = arg- }+// --- Query interpolation ---++func interpolateQuery(query string, args []any) string {+ if len(args) == 0 {+ return query+ }+ for _, arg := range args {+ query = strings.Replace(query, "?", formatValue(arg), 1)}- return out+ return query+}++// --- JSON result types ---++type jsonResult struct {+ Meta []jsonMeta `json:"meta"`+ Data [][]*string `json:"data"`+ Rows int `json:"rows"`+}++type jsonMeta struct {+ Name string `json:"name"`+ Type string `json:"type"`}// --- Rows adapter ---type embeddedRows struct {- rows *sql.Rows+ columns []string+ data [][]*string+ index int+}++func (r *embeddedRows) Next() bool {+ r.index+++ return r.index <= len(r.data)}-func (r *embeddedRows) Next() bool { return r.rows.Next() }-func (r *embeddedRows) Scan(dest ...any) error { return r.rows.Scan(dest...) }-func (r *embeddedRows) Close() error { return r.rows.Close() }-func (r *embeddedRows) Err() error { return r.rows.Err() }-func (r *embeddedRows) Columns() []string {- cols, _ := r.rows.Columns()- return cols+func (r *embeddedRows) Scan(dest ...any) error {+ if r.index < 1 || r.index > len(r.data) {+ return fmt.Errorf("no current row")+ }+ row := r.data[r.index-1]+ if len(dest) != len(row) {+ return fmt.Errorf("scan: expected %d destinations, got %d", len(row), len(dest))+ }+ for i, d := range dest {+ if err := scanValue(d, row[i]); err != nil {+ return fmt.Errorf("column %d: %w", i, err)+ }+ }+ return nil}++func (r *embeddedRows) Close() error { return nil }+func (r *embeddedRows) Err() error { return nil }+func (r *embeddedRows) Columns() []string { return r.columns }func (r *embeddedRows) ScanStruct(dest any) error { return fmt.Errorf("ScanStruct not supported in embedded mode") }func (r *embeddedRows) ColumnTypes() []driver.ColumnType { return nil }func (r *embeddedRows) Totals(dest ...any) error { return nil }@@ -93,20 +193,157 @@ func (r *embeddedRows) Totals(dest ...any) error { return nil }// --- Row adapter ---type embeddedRow struct {- row *sql.Row+ data []*string+ err error}-func (r *embeddedRow) Scan(dest ...any) error { return r.row.Scan(dest...) }-func (r *embeddedRow) Err() error { return r.row.Err() }+func (r *embeddedRow) Scan(dest ...any) error {+ if r.err != nil {+ return r.err+ }+ if len(dest) != len(r.data) {+ return fmt.Errorf("scan: expected %d destinations, got %d", len(r.data), len(dest))+ }+ for i, d := range dest {+ if err := scanValue(d, r.data[i]); err != nil {+ return fmt.Errorf("column %d: %w", i, err)+ }+ }+ return nil+}++func (r *embeddedRow) Err() error { return r.err }func (r *embeddedRow) ScanStruct(dest any) error { return fmt.Errorf("ScanStruct not supported in embedded mode") }+// --- Scan helper: convert JSON string values to Go types ---++func scanValue(dest any, src *string) error {+ if src == nil {+ return nil // NULL — leave dest as zero value+ }+ val := *src++ switch d := dest.(type) {+ case *string:+ *d = val+ case *uuid.UUID:+ id, err := uuid.Parse(val)+ if err != nil {+ return fmt.Errorf("parse UUID %q: %w", val, err)+ }+ *d = id+ case **uuid.UUID:+ id, err := uuid.Parse(val)+ if err != nil {+ return fmt.Errorf("parse UUID %q: %w", val, err)+ }+ *d = &id+ case *time.Time:+ for _, layout := range []string{+ "2006-01-02 15:04:05.000000000",+ "2006-01-02 15:04:05.000000",+ "2006-01-02 15:04:05.000",+ "2006-01-02 15:04:05",+ } {+ if t, err := time.Parse(layout, val); err == nil {+ *d = t+ return nil+ }+ }+ return fmt.Errorf("parse time %q: no matching layout", val)+ case *time.Duration:+ v, err := strconv.ParseInt(val, 10, 64)+ if err != nil {+ return err+ }+ *d = time.Duration(v)+ case *bool:+ *d = val == "true" || val == "1"+ case *int:+ v, err := strconv.Atoi(val)+ if err != nil {+ return err+ }+ *d = v+ case *int8:+ v, err := strconv.ParseInt(val, 10, 8)+ if err != nil {+ return err+ }+ *d = int8(v)+ case *int16:+ v, err := strconv.ParseInt(val, 10, 16)+ if err != nil {+ return err+ }+ *d = int16(v)+ case *int32:+ v, err := strconv.ParseInt(val, 10, 32)+ if err != nil {+ return err+ }+ *d = int32(v)+ case *int64:+ v, err := strconv.ParseInt(val, 10, 64)+ if err != nil {+ return err+ }+ *d = v+ case *uint:+ v, err := strconv.ParseUint(val, 10, 64)+ if err != nil {+ return err+ }+ *d = uint(v)+ case *uint8:+ v, err := strconv.ParseUint(val, 10, 8)+ if err != nil {+ return err+ }+ *d = uint8(v)+ case *uint16:+ v, err := strconv.ParseUint(val, 10, 16)+ if err != nil {+ return err+ }+ *d = uint16(v)+ case *uint32:+ v, err := strconv.ParseUint(val, 10, 32)+ if err != nil {+ return err+ }+ *d = uint32(v)+ case *uint64:+ v, err := strconv.ParseUint(val, 10, 64)+ if err != nil {+ return err+ }+ *d = v+ case *float32:+ v, err := strconv.ParseFloat(val, 32)+ if err != nil {+ return err+ }+ *d = float32(v)+ case *float64:+ v, err := strconv.ParseFloat(val, 64)+ if err != nil {+ return err+ }+ *d = v+ default:+ return fmt.Errorf("unsupported scan type %T", dest)+ }+ return nil+}+// --- Batch adapter ---type embeddedBatch struct {- db *sql.DB- query string- rows [][]any- sent bool+ session *chdblib.Session+ query string+ rows [][]any+ sent bool}func (b *embeddedBatch) Append(v ...any) error {@@ -138,10 +375,14 @@ func (b *embeddedBatch) Send() error {sb.WriteString(")")}- _, err := b.db.Exec(sb.String())- if err == nil {- b.sent = true+ result, err := b.session.Query(sb.String())+ if err != nil {+ return err+ }+ if result != nil {+ result.Free()}+ b.sent = truereturn err}@@ -154,7 +395,7 @@ func (b *embeddedBatch) Rows() int { return len(b.rows) }func (b *embeddedBatch) Columns() []column.Interface { return nil }func (b *embeddedBatch) Close() error { return nil }-// --- Value formatting for batch INSERTs ---+// --- Value formatting for batch INSERTs and query interpolation ---func formatValue(v any) string {if v == nil {